diff --git a/AGENTS.md b/AGENTS.md index 7ec8a92359..80c78a7412 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,6 +164,26 @@ remains a separate MEMORY/OFF/EXCLUSIVE database file. prefixed names, such as dsh's `DSH_*`, maps them itself. Linux still makes Store calls in the caller's process, and the worker's rules above (loopback, secret, one attempt, no replay) are unchanged. +- **VM placement (`@layerfs/runtime`, owner 2026-10-31, #515).** A LayerFS + home may be placed in Linux VMs on this machine instead of in the host's + Docker: `LAYERFS_PLACEMENT=vm:` (or the runtime's `placement` + setting), with `local` the default and unchanged. **One Linux VM per Store** + (V1): a project's VM holds that project's Store on its own ext4 disk, its + Sandboxes are Linux namespaces inside it + (`LAYERFS_SANDBOX_BACKEND=namespaces`, documented in + [the agent guide](api/docs/agent.md#namespace-sandboxes-linux)), and its + worker runs the real SDK as root with its own secret and its own + loopback-published port; the host reaches only each VM's worker, with one + attempt per call and no replay, the same rules as the macOS and Windows + routes. Store calls therefore leave the caller's process on Linux too, under + `vm:` placement only, and a process routes them per project + (`useWorker((project) => endpoint)`, or that project's + `LAYERFS_WORKER_ENDPOINT`). `project.init` makes the project's VM and + `project.remove` deletes it with its Store disk; a bare VM delete is refused + while the project exists, because that Store is on that disk only and no + remote Store push exists (#506). Implemented and verified providers: + `microsandbox` (`msb`) and `apple-container` (`container` 1.6.1). + `@layerfs/runtime`'s guide and `api/runtime/src/platform/vm/` hold the details. - **One global database, one writer, short holds (owner decision 2026-10-18).** All committed state lives in one SQLite Store file; no sharding, second database, concurrent-writer engine or objects outside it. Writes go one at a diff --git a/api/agent/src/client.js b/api/agent/src/client.js index 7e1ae74a98..8455d1cb28 100644 --- a/api/agent/src/client.js +++ b/api/agent/src/client.js @@ -17,8 +17,9 @@ export const BACKEND = /^(backend|runtime)\./; /** Runs one operation once and returns its value or throws its refusal; * `options` are the client's settings, whose `trace` selects its trace file; - * `transport` is the client's binding (default: the one a new client would use). */ -export async function invoke(op, args, options, transport = binding()) { + * `transport` is the client's binding (default: the one a new client of + * `args.project` would use). */ +export async function invoke(op, args, options, transport = binding(args.project)) { return outcome(await traced(options, op, args, (scope) => { if (BACKEND.test(op)) throw new LayerFSError("UnknownMethod", "backend and runtime methods are @layerfs/backend's and @layerfs/runtime's", null); return transport.call(op, observed(args, scope)); @@ -26,7 +27,7 @@ export async function invoke(op, args, options, transport = binding()) { } /** One call on the binding's backend table (`backend.*`, `runtime.*`). */ -export async function invokeBackend(op, args, options, transport = binding()) { +export async function invokeBackend(op, args, options, transport = binding(args?.project)) { return outcome(await traced(options, op, args, (scope) => transport.backendCall(op, observed(args, scope)))); } @@ -60,9 +61,19 @@ export function sandboxArg(sandbox) { /** Deployment settings from the client options, else the environment * (LAYERFS_). A Workspace's Sandbox needs - * every one but the container prefix and daemon host. */ + * every one but the container prefix and daemon host. With `sandbox` + * (LAYERFS_SANDBOX) `native` (Linux or macOS), a Sandbox is a host daemon: + * Linux uses FUSE with a root daemon and nonroot commands (501:20); + * macOS uses loopback NFS and this user. It needs the daemon and + * `localDir` (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is + * Docker; any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || undefined; + const sandbox = options.sandbox ?? env("SANDBOX"); + if (sandbox === "native") return nativeDeployment(options, env, sandbox); + if (sandbox !== undefined && sandbox !== "docker") { + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not ${sandbox}`, null); + } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", image: options.image ?? env("IMAGE"), @@ -80,6 +91,35 @@ export function deployment(options = {}) { return deploy; } +/** A native Sandbox's settings: the daemon, the directory its Sandboxes + * live in, the optional name prefix, and the backend inside that directory + * (`sandboxBackend`, LAYERFS_SANDBOX_BACKEND: `native` when unset, or + * `namespaces` on Linux). The namespaces backend gives each Sandbox a root + * filesystem from an image, so it also needs `image` (LAYERFS_IMAGE, the + * same digest Docker deploys) and `imageArchive` (LAYERFS_IMAGE_ARCHIVE, the + * docker-archive the bundle carries). */ +function nativeDeployment(options, env, sandbox) { + const deploy = { + sandbox, + daemon: options.daemon ?? env("DAEMON"), + localDir: options.localDir ?? env("LOCAL_DIR"), + containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), + }; + const backend = options.sandboxBackend ?? env("SANDBOX_BACKEND"); + if (backend !== undefined) deploy.sandboxBackend = backend; + const required = [["daemon", "DAEMON"], ["localDir", "LOCAL_DIR"]]; + if (backend === "namespaces") { + deploy.image = options.image ?? env("IMAGE"); + deploy.imageArchive = options.imageArchive ?? env("IMAGE_ARCHIVE"); + required.push(["image", "IMAGE"], ["imageArchive", "IMAGE_ARCHIVE"]); + } + for (const [name, setting] of required) { + if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); + } + if (!deploy.containerPrefix) delete deploy.containerPrefix; + return deploy; +} + /** Refuses every call on a closed client (or one no factory opened). */ export function live(client) { if (client?.[STATE]?.closed !== false) { diff --git a/api/agent/src/index.js b/api/agent/src/index.js index 446b59a534..1dc96b3c7e 100644 --- a/api/agent/src/index.js +++ b/api/agent/src/index.js @@ -100,7 +100,15 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON; a * Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. + * container it starts `-`. On Linux or macOS, + * LAYERFS_SANDBOX=native starts each Sandbox as a host daemon in a directory + * under LAYERFS_LOCAL_DIR. Linux requires root for FUSE and runs commands + * as 501:20; macOS uses loopback NFS and the caller's identity. No Docker + * setting is read; native provides no isolation from the host. + * LAYERFS_SANDBOX_BACKEND picks the backend inside that directory: `native` + * (unset) or, on Linux, `namespaces` (namespaces, cgroups, nonroot commands). + * It is read for every call, so a Sandbox made with one backend is never + * reopened as another. * LAYERFS_DAEMON_HOST is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on @@ -129,13 +137,18 @@ export class LayerFS { volume: env("VOLUME"), daemon: env("DAEMON"), ...(env("CONTAINER_PREFIX") ? { containerPrefix: env("CONTAINER_PREFIX") } : {}), + ...(env("SANDBOX") ? { sandbox: env("SANDBOX") } : {}), + ...(env("SANDBOX_BACKEND") ? { sandboxBackend: env("SANDBOX_BACKEND") } : {}), + ...(env("LOCAL_DIR") ? { localDir: env("LOCAL_DIR") } : {}), + ...(env("IMAGE_ARCHIVE") ? { imageArchive: env("IMAGE_ARCHIVE") } : {}), ...(env("DAEMON_HOST") ? { daemonHost: env("DAEMON_HOST") } : {}), }, storePolicy: env("STORE_POLICY"), storePath: env("STORE_PATH"), }; - // not open until a factory has opened its Store; the binding is chosen now, once - this[STATE] = { closed: null, transport: binding() }; + // not open until a factory has opened its Store; the binding is chosen now, once, + // for this client's project (a VM placement has one worker per project) + this[STATE] = { closed: null, transport: binding(this.projectName) }; this.fs = bind(this, fsMethods); this.project = bind(this, { list_branches, list_sandboxes }); this.branch = bind(this, { list_commits, diff_commits, diff_branches, fork, merge, publish }); diff --git a/api/agent/src/transport/index.js b/api/agent/src/transport/index.js index 68d9e32760..5301914202 100644 --- a/api/agent/src/transport/index.js +++ b/api/agent/src/transport/index.js @@ -8,13 +8,14 @@ // // The worker route is open to any integration that names an endpoint // (AGENTS.md, owner 2026-10-24): @layerfs/runtime starts the worker on macOS -// and Windows, where a process cannot open the Store itself. LayerFS names no -// integration here. +// and Windows, where a process cannot open the Store itself, and in a +// `vm:` home, where each project's Store is in that project's VM +// (one endpoint per project). LayerFS names no integration here. import { LayerFSError } from "../errors.js"; import { inprocess } from "./inprocess.js"; import { workerClient } from "./worker_client.js"; -/** The worker `useWorker` handed over: { endpoint }, else null. */ +/** The worker `useWorker` handed over: { endpoint }, { resolve } or null. */ let handed = null; let local = null; /** Worker clients by endpoint file. */ @@ -24,18 +25,28 @@ const workers = new Map(); export const setting = (name) => process.env[`LAYERFS_${name}`] || undefined; /** Internal, for @layerfs/runtime: clients made from now on in this process - * use the worker at `endpoint` (an endpoint file). Takes precedence over the - * environment. One worker per process: another endpoint is refused. */ + * use the worker at `endpoint` (an endpoint file), or, when `endpoint` is a + * function, the worker that function names for a client's project — one VM + * per Store has one endpoint each. Takes precedence over the environment. + * One worker per process: another endpoint is refused, and so is a second + * resolver. */ export function useWorker(endpoint) { - if (typeof endpoint !== "string" || !endpoint) throw new LayerFSError("InvalidInput", "a worker endpoint file is required", null); - if (handed && handed.endpoint !== endpoint) throw new LayerFSError("InvalidInput", `this process already uses the worker at ${handed.endpoint}`, null); - handed = { endpoint }; + const next = typeof endpoint === "function" ? { resolve: endpoint } + : typeof endpoint === "string" && endpoint ? { endpoint } + : null; + if (!next) throw new LayerFSError("InvalidInput", "a worker endpoint file is required", null); + if (handed && (handed.endpoint !== next.endpoint || handed.resolve !== next.resolve)) { + throw new LayerFSError("InvalidInput", `this process already uses the worker at ${handed.endpoint ?? "one endpoint per project"}`, null); + } + handed = next; } -/** The binding a client made now uses: the handed-over worker, else - * LAYERFS_WORKER_ENDPOINT's, else the addon in this process. */ -export function binding() { - const endpoint = handed?.endpoint ?? setting("WORKER_ENDPOINT"); +/** The binding a client made now uses: the handed-over worker (by project, or + * the one endpoint), else LAYERFS_WORKER_ENDPOINT's, else the addon in this + * process. A call without a project (an in-process backend call) uses the + * environment's endpoint, never a project's resolver. */ +export function binding(project) { + const endpoint = handed?.resolve && typeof project === "string" ? handed.resolve(project) : handed?.endpoint ?? setting("WORKER_ENDPOINT"); if (!endpoint) return (local ??= inprocess()); let client = workers.get(endpoint); if (!client) workers.set(endpoint, (client = workerClient(endpoint))); diff --git a/api/agent/test/native_deployment.mjs b/api/agent/test/native_deployment.mjs new file mode 100644 index 0000000000..4b0d951e30 --- /dev/null +++ b/api/agent/test/native_deployment.mjs @@ -0,0 +1,143 @@ +// Which runtime a Sandbox uses. No real addon, Store or Docker (the +// stand-in binding answers the client's calls): +// node api/agent/test/native_deployment.mjs +// LAYERFS_SANDBOX unset or `docker` gives exactly the Docker deployment it +// always gave; `native` (macOS) needs only the daemon and LAYERFS_LOCAL_DIR +// (no Docker setting); any other value, the former `local` included, is +// refused rather than taken as either. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { deployment } from "../src/client.js"; + +const FAKE = fileURLToPath(new URL("./fixture/fake-native.cjs", import.meta.url)); +const native = createRequire(import.meta.url)(FAKE); + +const clear = () => { + for (const name of Object.keys(process.env)) { + if (name.startsWith("LAYERFS_") || name.startsWith("DSH_LAYERFS_")) delete process.env[name]; + } + process.env.LAYERFS_NATIVE = FAKE; +}; +const docker = { docker: "/run/docker.sock", image: `sha256:${"a".repeat(64)}`, volume: "lfs-v", daemon: "/opt/layerfs-daemon" }; +const setDocker = () => { + process.env.LAYERFS_DOCKER = docker.docker; + process.env.LAYERFS_IMAGE = docker.image; + process.env.LAYERFS_VOLUME = docker.volume; + process.env.LAYERFS_DAEMON = docker.daemon; +}; +const store = () => { + process.env.LAYERFS_HOME = "/private/local-home"; + process.env.LAYERFS_STORE_PATH = "/fake/store.sqlite"; +}; + +// unset: the Docker deployment, field for field +clear(); +setDocker(); +assert.deepEqual(deployment(), docker); +// `docker` is the same deployment +process.env.LAYERFS_SANDBOX = "docker"; +assert.deepEqual(deployment(), docker); +// the Docker settings are still required there +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_IMAGE/.test(e.message)); + +// native: daemon + localDir, no Docker setting read or required +clear(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +assert.deepEqual(deployment(), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// an image (sandbox.create's option) does not change a native deployment +assert.deepEqual(deployment({ image: docker.image }), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// the backend inside the directory is native unless named; namespaces is +// passed through (the binding refuses it off Linux, and an unknown value) +assert.equal(deployment().sandboxBackend, undefined); +const archive = `/bundle/images/${"a".repeat(64)}.tar`; +process.env.LAYERFS_SANDBOX_BACKEND = "namespaces"; +process.env.LAYERFS_IMAGE = docker.image; +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// a namespaces Sandbox's root is an image: the same digest Docker deploys +// and the docker-archive the bundle carries +assert.deepEqual(deployment(), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "namespaces", + image: docker.image, + imageArchive: archive, +}); +// each of those is named when it is missing +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE\)/.test(e.message)); +process.env.LAYERFS_IMAGE = docker.image; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE_ARCHIVE\)/.test(e.message)); +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// the native backend inside the same directory reads neither of them +assert.deepEqual(deployment({ sandboxBackend: "native" }), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "native", +}); +delete process.env.LAYERFS_SANDBOX_BACKEND; +delete process.env.LAYERFS_IMAGE; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +// the container prefix names native Sandbox directories too +process.env.LAYERFS_CONTAINER_PREFIX = "ci-"; +assert.equal(deployment().containerPrefix, "ci-"); +delete process.env.LAYERFS_CONTAINER_PREFIX; +// each required setting is named when missing +delete process.env.LAYERFS_LOCAL_DIR; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_LOCAL_DIR/.test(e.message)); +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +delete process.env.LAYERFS_DAEMON; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_DAEMON/.test(e.message)); +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +// a DSH_LAYERFS_NFS_ name is not mapped (an integration maps its own names) +process.env.DSH_LAYERFS_NFS_MOUNT = "/bin/true"; +deployment(); +assert.equal(process.env.LAYERFS_NFS_MOUNT, undefined); +// since #428 the SDK reads LAYERFS_ names only: a DSH_LAYERFS_ twin selects nothing +clear(); +setDocker(); +process.env.DSH_LAYERFS_SANDBOX = "native"; +process.env.DSH_LAYERFS_LOCAL_DIR = "/l"; +assert.deepEqual(deployment(), docker); + +// anything else, the former `local` included, is refused, never silently Docker +for (const typo of ["local", "Native", "nativ", "NATIVE", "podman"]) { + clear(); + setDocker(); + process.env.LAYERFS_SANDBOX = typo; + assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && e.message.includes(`not ${typo}`)); +} + +// a client reads LAYERFS_SANDBOX and LAYERFS_LOCAL_DIR once, and +// sandbox.create sends the native deployment (no Docker setting) +clear(); +store(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +const { LayerFS } = await import("../src/index.js"); +const client = await LayerFS.open("website"); +assert.equal(client.options.deploy.sandbox, "native"); +assert.equal(client.options.deploy.localDir, "/Users/me/lfs"); +const before = native.deploys.length; +await client.sandbox.create(); +assert.deepEqual(native.deploys.slice(before), [{ sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }]); +// an image given to sandbox.create does not turn it into a Docker deployment +await client.sandbox.create({ image: docker.image }); +assert.deepEqual(native.deploys.at(-1), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +await client.close(); +// unset: the client carries no new key +clear(); +store(); +setDocker(); +const plain = await LayerFS.open("website"); +assert.equal("sandbox" in plain.options.deploy, false); +assert.equal("localDir" in plain.options.deploy, false); +await plain.close(); +console.log("native_deployment: ok"); diff --git a/api/agent/test/packaging.mjs b/api/agent/test/packaging.mjs index 496a616b95..114ed0ad3d 100644 --- a/api/agent/test/packaging.mjs +++ b/api/agent/test/packaging.mjs @@ -1,7 +1,7 @@ // @layerfs/agent's surface and client lifecycle (#428). Each case is one // command; every case but `dispatch` uses the stand-in binding and creates no // Store: -// node api/agent/test/packaging.mjs surface|lifecycle|copy|sandbox|worker-route|transport-order|dispatch +// node api/agent/test/packaging.mjs surface|lifecycle|copy|sandbox|worker-route|worker-per-project|transport-order|dispatch // `dispatch` loads the real addon (LAYERFS_NATIVE or engine/crates/layerfs-node/layerfs.node). import assert from "node:assert/strict"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; @@ -173,6 +173,32 @@ if (selected === "surface") { const calls = native.calls.length; await assert.rejects(LayerFS.open("website"), { message: /nonexistent/ }); assert.equal(native.calls.length, calls, "the worker route never reaches the addon"); +} else if (selected === "worker-per-project") { + // one process, one worker per project (`vm:`: one VM per Store) + const { fakeWorker } = await import("./fixture/fake-worker.mjs"); + const { useWorker } = await import("@layerfs/agent/internal"); + const shopWorker = await fakeWorker(), blogWorker = await fakeWorker(); + const dir = mkdtempSync(join(tmpdir(), "layerfs-per-project-")); + try { + const files = { shop: join(dir, "shop.json"), blog: join(dir, "blog.json") }; + shopWorker.endpoint(files.shop, "/private/shop"); + blogWorker.endpoint(files.blog, "/private/blog"); + // a resolver instead of one endpoint: this process's route is one VM per project + useWorker((project) => files[project]); + assert.throws(() => useWorker((project) => `${project}.json`), { code: "InvalidInput", message: /already uses/ }); + assert.throws(() => useWorker(files.shop), { code: "InvalidInput", message: /already uses/ }); + const shop = await LayerFS.open("shop"), blog = await LayerFS.open("blog"); + await shop.fs.read("main", "/README.md"); + await blog.fs.read("main", "/README.md"); + await shop.close(); + await blog.close(); + assert.deepEqual(shopWorker.calls.map((c) => c.op), ["project.list_branches", "fs.read"], "shop's calls reach shop's VM"); + assert.deepEqual(blogWorker.calls.map((c) => c.op), ["project.list_branches", "fs.read"], "blog's calls reach blog's VM"); + } finally { + shopWorker.close(); + blogWorker.close(); + rmSync(dir, { recursive: true, force: true }); + } } else if (selected === "transport-order") { // each client keeps the binding it was made with, whatever the call order const { fakeWorker } = await import("./fixture/fake-worker.mjs"); diff --git a/api/docs/agent.md b/api/docs/agent.md index 330fcab914..7dc9c7b565 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -45,7 +45,7 @@ async and return a ready client, and `new LayerFS()` is refused `InvalidInput`. **There is no options argument:** every setting comes from the environment, read once when the client is made, by its `LAYERFS_` name only: `LAYERFS_HOME`, `_STORE_PATH`, `_DOCKER`, `_IMAGE`, -`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_TRACE`, +`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_SANDBOX`, `_LOCAL_DIR`, `_TRACE`, `_STORE_POLICY` (and `_TRACE_STEP`, read at each call). An object passed where the options went (`LayerFS.open(name, {…})`, or an object as `LayerFS.init`'s third argument) is refused `InvalidInput`, never ignored. `open` opens the project's Store and makes one project lookup @@ -105,7 +105,9 @@ the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere else opening fails `Unsupported`, and without a `storePath` it fails `InvalidInput`. On macOS and Windows, any process that names its LayerFS home's worker (`LAYERFS_WORKER_ENDPOINT`, or the endpoint `@layerfs/runtime` -hands over with `useWorker`) sends the calls there instead. The route is open to any integration (owner, +hands over with `useWorker` — or that project's endpoint, on a +`vm:` home where each project's Store is in its own VM) sends the +calls there instead. The route is open to any integration (owner, 2026-10-24); LayerFS names none. A client keeps the binding it was made with: the in-process addon, or the worker if one was set when it was made, so a client made before `@layerfs/runtime` hands over its worker stays in-process and @@ -141,10 +143,13 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional path of the real addon; defaults to `engine/crates/layerfs-node/layerfs.node`. | -| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows: the endpoint file of the LayerFS home's worker, which then makes every binding call. Any integration may set it. | +| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows, and a `vm:` project VM: the endpoint file of the worker that makes this project's binding calls. Any integration may set it. | | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | +| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `native` (macOS only): a Sandbox is a daemon process of this user, with no container, no root and no Docker setting read; Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Elsewhere `sandbox.create` refuses `native` as `Unsupported`. Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=native`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each native Sandbox gets its own directory under it. | +| `LAYERFS_SANDBOX_BACKEND` | With `LAYERFS_SANDBOX=native` on Linux, optional: `native` (unset) or `namespaces`. `namespaces` puts each Sandbox's daemon in its own mount, PID and IPC namespaces and its own cgroup v2 directory, and runs commands as the sandbox's uid:gid with no capabilities and no-new-privs; see [Namespace Sandboxes (Linux)](#namespace-sandboxes-linux). It needs root, cgroup v2, `/dev/fuse` and `user.max_user_namespaces=0`, and is refused `Unsupported` elsewhere or without them. Any other value is refused `InvalidInput` before anything is made. It is read when the client is made and sent with every call, so a Sandbox made with one backend is never reopened as the other. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | @@ -244,6 +249,125 @@ Daemon/container restart recovery and replay of uncertain operations are exclude Background process ownership is in the native executor, including persisted process identity; the filesystem daemon launches and supervises no commands. +### Native Sandboxes (macOS) + +`LAYERFS_SANDBOX=native` is the native runtime for macOS, with no root, no +Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon +runs as a process of this user and serves its Workspaces over NFSv3 on +127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them +(`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands +run as this user, in this machine's namespaces: **it isolates nothing** +beyond what the caller's own sandbox (for instance Seatbelt) imposes. A +command starts with this process's `PATH` (so `node`, Homebrew's tools and +the like are found as in the caller's own shell) and its own environment, +nothing else of this process's. Docker stays the default, and Linux keeps +the Docker route only. + +It is the in-process route: the client loads the macOS addon and opens the +Store itself, so no worker endpoint (`LAYERFS_WORKER_ENDPOINT`) is set. It +needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned by +this user, 0755; the directories above it need not be traversable by others) +and `LAYERFS_STORE_PATH` on a local apfs or hfs volume; a Store on any other +filesystem is refused `Unsupported`. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` +and `_DAEMON_HOST` are not read, and an `image` given to `sandbox.create` is +ignored. Everything else above holds: numbered Sandboxes of up to 16 +Workspaces, `sandbox.create` as the only start, `sandbox.close`, idle close +after 2 minutes, reclaim, records under `LAYERFS_HOME//sandboxes//` +and start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME`) and +`daemon.log` (the daemon's output, what `backend.daemon.sandbox.get_log` +reads; a failed start's report carries its last 20 lines). The daemon listens +on 127.0.0.1 only. Its records name the daemon by process id and start time, +so a reused id is never taken for it. `sandbox.create`, the mount result and +`project.list_sandboxes()` give a native Sandbox's `image` as `local`. A +record's `runtime` is `native`; it is used only by a client whose deployment +is `native` with the same `LAYERFS_LOCAL_DIR` and prefix, so it puts that +Sandbox exactly where the record says; any other is refused `InvalidInput` +and nothing is deleted. A record of the former `local` runtime (an earlier, +unmerged build) is never used: it is refused the same way and nothing it +names is deleted. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next `sandbox.create` +or `runtime.sandbox.reclaim` reclaims it, forcing off any mount the dead +daemon left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. `@layerfs/runtime`'s Docker-side +`runtime.sandbox.list` and `sweep` do not see native Sandboxes. Before a Commit the daemon +writes back what the client buffered; after a Commit or merge installs +another root, it makes the client drop the replaced files' cached attributes +at once. + +`LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's +helper commands (`{port}` and `{dir}` are filled in). The default mount is +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client +then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are +read; an integration that can set only its own prefixed names maps them +itself. + +Limits: background processes still read `/proc` and are not available on +macOS; many small writes cost about six times APFS's time; checked on Apple +silicon with macOS 27.2. + +### Namespace Sandboxes (Linux) + +`LAYERFS_SANDBOX_BACKEND=namespaces` keeps the native deployment (no Docker, no +image) and adds isolation inside the project's VM: each Sandbox's daemon runs +in its own mount, PID and IPC namespaces and its own cgroup v2 directory +(`/sys/fs/cgroup/layerfs/`), so two Sandboxes on one Store cannot +see each other's mounts, processes or files, and closing one ends everything +it started. + +It needs root (mount and PID namespaces, `/dev/fuse`), the unified cgroup v2 +hierarchy with `cgroup.kill`, and `user.max_user_namespaces=0`: without the +last, a command could unshare a user namespace and keep its own copy of the +Workspace mount, outside the daemon's unmount. The runtime never sets that +limit; the VM's owner does (`sysctl -w user.max_user_namespaces=0`). A host +that fails any of these is refused `Unsupported` before anything is made, and +the message names the missing condition. + +Sandbox `n` lives where a native Sandbox does — `LAYERFS_LOCAL_DIR/-/` +with `config/daemon.setup`, `overlay/`, `workspaces/`, `tmp/` and +`daemon.log` — plus `bin/layerfs-daemon` (the daemon executable, copied in +0755) and its sibling `-.root/`, the private root its init +stage mounts a tmpfs on and pivots to. The private root holds: the VM's own +userland (`/usr`, `/bin`, `/sbin`, `/lib`, `/lib64`, `/etc`) bound read-only, +so there is no image and no package restoration; a private `/proc` (only the +Sandbox's processes), `/dev` (the basic devices, `/dev/fuse`, a private +`devpts` and `shm`), `/tmp`; the Sandbox's own directory bound at its own path, +so `config/`, `overlay/` and `workspaces/` keep the same names on both +sides; and the Store's directory bound read-write, so the daemon's SQLite WAL +and the writers' directory lock see the same inode as every other opener on +the kernel. The daemon listens on the VM's `127.0.0.1`, as a native one does, +and `127.0.0.1:0` in its setup means the same thing. + +Commands run through `layerfs-sandbox-helper`, which is looked up at +`LAYERFS_SANDBOX_HELPER` when set and otherwise beside the daemon executable. +Each command enters the Sandbox's namespaces and runs as its uid:gid with no +effective or permitted capabilities and no-new-privs; it cannot mount, cannot +unmount the Workspace and cannot create a user namespace. A background process +it leaves behind (even after `setsid`) is ended by the Sandbox's close, which +is one `cgroup.kill` write followed by waiting for the group to be empty; +Delete then removes the cgroup and the directory, so a closed Sandbox leaves no +process, mount or cgroup behind. No graceful filesystem drain exists. + +A record's `runtime` is `namespaces`, with the same fields as a native record; +it is used only by a client whose `LAYERFS_SANDBOX_BACKEND` is `namespaces` +with the same `LAYERFS_LOCAL_DIR` and prefix. Another backend's record — and a +`namespaces` record seen by a native deployment — is refused `InvalidInput` +and nothing is deleted. Reopen checks the recorded init process (id, start +time, boot) and its cgroup membership, so a reused id, or a process outside +the Sandbox's cgroup, is never taken for it. Everything else above holds: +numbered Sandboxes, up to 16 Workspaces each, one daemon per Sandbox, idle +close after 2 minutes, reclaim, and `image` reported as `local`. + +Limits: the network namespace is shared with the VM, so a command can reach +another Sandbox's daemon port on `127.0.0.1`; `/sys` is not mounted; the +userland is the VM's, not a pinned image; and the private root's bind mounts +are read-only at their top only, as this kernel's remount is not recursive. + ### Exec output `exec` returns `{ kind, exitCode, stdout, stderr, message, truncated }` when the diff --git a/api/docs/runtime.md b/api/docs/runtime.md index 6ce3e1e341..418096d6a3 100644 --- a/api/docs/runtime.md +++ b/api/docs/runtime.md @@ -15,7 +15,7 @@ Agents don't import it; a host that runs projects for its users does. import { LayerFSRuntime } from "@layerfs/runtime"; const runtime = await LayerFSRuntime.open({ home, image, daemon, bundle, docker, containerPrefix }); try { - await runtime.status(); // { platform, missing, docker, worker, disk, sweep } + await runtime.status(); // { platform, placement, missing, docker, worker, vm, disk, sweep } const job = await runtime.project.start_init("shop", "/src/shop", { timeout }); // timeout optional: LayerFS.init's 300 s default await runtime.project.get_init(job.id); // { step, scanned, totals, imported, project, error, ended } await runtime.project.list(); // [{ name, volume, image, source, created, initialized }] @@ -34,6 +34,19 @@ try { } ``` +A `vm:` home has one call group more (`vm`), names a project in +`worker.start`/`worker.restart` (one VM per Store, see +[Placement](#placement-one-linux-vm-per-store)), and reports each project's VM: + +```js + await runtime.worker.start("shop"); // this project's VM: create/start, bundle, worker, endpoint + await runtime.worker.restart("shop", { confirm: "restart" }); + await runtime.vm.status(); // { provider, placement, vms: [{ project, name, state, worker, disk }] } + await runtime.vm.status("shop"); // that project's VM record + await runtime.vm.stop("shop"); // stops it; its Store disk and its Store stay + await runtime.vm.delete("shop", { confirm: "delete", discardStore: false }); // the VM record only, unless discardStore +``` + ## Opening and settings The runtime opens on a LayerFS home, not a project, so it works before the @@ -44,7 +57,8 @@ on macOS Docker Desktop's `~/.docker/run/docker.sock` when only that one exists, checked once at `open`), `volumes` (Linux), `traceDir` (default `/traces`), `containerPrefix`, `stateDir` (default `/.runtime`, the runtime's project records), `sweep` -(default true) and on Windows `wsl: { distribution, user, node, home }`. A +(default true), `placement` (default `local`; `vm:` places the home +in one Linux VM, below) and on Windows `wsl: { distribution, user, node, home }`. A missing setting does not refuse `open`: `status().missing` lists it. ## Init @@ -109,7 +123,10 @@ lines from now on, or from `since`, each with `actor`, its trace file's name. ## Platforms and the worker Each platform folder implements one interface (`endpoint`, `status`, -`start`, `restart`, `init`, `path` and the Docker, home and disk helpers): +`start`, `restart`, `init`, `path`, the Store hooks `makeStore`, +`storeHolders`, `releaseStoreHolders` and `removeStore`, and the Docker, home +and disk helpers); `platform/vm/` implements the same interface over a VM +(see [Placement](#placement-one-linux-vm-per-home)). - **Linux** runs Store calls in this process. There is no worker: `worker.start` and `worker.restart` are refused `InvalidInput`. Init @@ -128,27 +145,206 @@ Each platform folder implements one interface (`endpoint`, `status`, `wslpath`. This process owns the worker through its stdin: `close()` drains and ends it. -The runtime starts a worker only on macOS and Windows. The worker route is -open to any integration (owner, 2026-10-24): on those platforms the runtime -hands its endpoint to `@layerfs/agent` (`useWorker(endpoint)`, without -changing the environment), so every client made after `open` in its process -uses the worker; a client made before keeps its in-process binding. One -worker per process: a second runtime of another home is refused. Another -process, such as an agent's command, uses the worker when its -`LAYERFS_WORKER_ENDPOINT` names the endpoint file; the integration that runs -it sets that. Such a process also needs the project's settings: make the -project with `project.start_init`, which creates and records its volume, then -set `LAYERFS_VOLUME` to the volume `project.list` gives, -`LAYERFS_STORE_PATH` to `/var/lib/docker/volumes//_data/global/store.sqlite`, -and `LAYERFS_IMAGE` and `LAYERFS_DAEMON` for its Sandboxes (on macOS, the -bundle's `layerfs-daemon`). A volume made outside the runtime works for Store -calls, but `project.list` shows the project with `volume: null` and -`volume.list` marks the volume `unknown`. The endpoint is `/worker/endpoint.json`, `{ home, port, -secret, build, daemonHost?, docker?, init? }`, readable by its user only. On macOS -`docker` is the engine's socket inside its VM, `/var/run/docker.sock`, which the -worker and Init's container mount; it replaces every call's `deploy.docker`, so the -Mac's own socket (`LAYERFS_DOCKER`) never reaches Linux. The worker -checks every file of the bundle's `manifest.json` before it listens. +The runtime starts a worker only on macOS and Windows, and in a +`vm:` home. The worker route is open to any integration (owner, +2026-10-24): on macOS and Windows the runtime hands its endpoint to +`@layerfs/agent` (`useWorker(endpoint)`, without changing the environment), +and on `vm:` it hands over a resolver (`useWorker((project) => endpoint)`), +so every client made after `open` in its process uses the worker; a client +made before keeps its in-process binding. One worker route per process: a +second runtime of another home is refused. Another process, such as an +agent's command, uses the worker when its `LAYERFS_WORKER_ENDPOINT` names +the endpoint file; the integration that runs it sets that (on a `vm:` home +`deploymentEnv` sets it to that project's VM). Such a process also needs +the project's settings: make the project with `project.start_init`, which +creates and records its volume, then set `LAYERFS_VOLUME` to the volume +`project.list` gives, +`LAYERFS_STORE_PATH` to `/var/lib/docker/volumes//_data/global/store.sqlite` +(or on `vm:` `/layerfs/store/global/store.sqlite`), and `LAYERFS_IMAGE` and +`LAYERFS_DAEMON` for its Sandboxes (on macOS, the bundle's `layerfs-daemon`). +A volume made outside the runtime works for Store calls, but `project.list` +shows the project with `volume: null` and `volume.list` marks the volume +`unknown`. On macOS and Windows the endpoint is +`/worker/endpoint.json`, on `vm:` `/worker/.json`, `{ home, +port, secret, build, daemonHost?, docker?, init?, vm? }`, readable by its user +only. On macOS `docker` is the engine's socket inside its VM, +`/var/run/docker.sock`, which the worker and Init's container mount; it +replaces every call's `deploy.docker`, so the Mac's own socket +(`LAYERFS_DOCKER`) never reaches Linux. The worker checks every file of the +bundle's `manifest.json` before it listens. + +## Placement: one Linux VM per Store + +`placement` (also read from `LAYERFS_PLACEMENT` when the setting is absent) is +`local` (the default: this machine's Docker, macOS worker container or WSL2 +worker, unchanged) or `vm:`: **each of this home's projects — its +Stores — lives in one Linux VM of its own** on this machine (#515 §6, V9, V1). +A project's VM holds that project's Store on its own ext4 disk, its Sandboxes +are Linux namespaces (`LAYERFS_SANDBOX_BACKEND=namespaces`, H36's backend) and +its worker runs the real SDK as root; the host reaches only each VM's worker, +with one attempt per call and no replay. Providers, both verified against the +real CLI by `test/verify/vm_real.mjs` and by H40's own run: `microsandbox` +(`msb` 0.7.8), `apple-container` (`container` 1.6.1) and `ssh` (any Linux host +the user provides, reached with the system `ssh` client; see +[the SSH provider](#the-ssh-provider-any-linux-host) below). + +```sh +LAYERFS_PLACEMENT=vm:microsandbox layerfs-home # any integration that sets it +``` + +Every per-VM thing is that project's alone: the VM `-vm`, its +Store disk `-vm-store`, its host port, its bundle copy and its +worker secret. Nothing is shared between two projects' VMs except the read-only +host home bind mount (§ Guest layout). + +Settings: `vm: { image, cpus, memory, storeSize }` (`LAYERFS_VM_IMAGE`, +`LAYERFS_VM_CPUS`, `LAYERFS_VM_MEMORY`, `LAYERFS_VM_STORE_SIZE`; defaults +`node:22-trixie-slim`, 2, `2G`, `20G`). The image is the VM image: it must hold +`node`, and it is not the Sandbox image. A VM home needs `bundle` and no +`image`; its `daemon` setting is the guest's, `/layerfs-daemon`, with +`layerfs-sandbox-helper` beside it. `storeSize` is the disk's size where the +provider has one: Microsandbox takes it when it makes the volume, Apple +Container at `volume create -s` (the CLI's own default is a sparse 504 GiB +image), and an existing volume keeps the size it was made with. + +Guest layout (`settings.js`'s `GUEST`), the same paths in every project's VM: +the home is a bind mount of the host's ``, so project entries and traces +are the same files on both sides; **the Store disk is mounted at the Store +directory**, so the Store is `/layerfs/store/global/store.sqlite`; the sealed +bundle is copied in at `/layerfs/bundle`; an Init source folder is copied to +`/layerfs/import/`; the worker's settings, pid and log live in +`/layerfs/state`; **Sandbox directories are on the Store disk, at +`/layerfs/store/local`**, not on the VM's root: a namespace Sandbox's root is +an `overlay` whose writable layer must sit on a local filesystem, and a +provider's own root can be an overlay itself (Microsandbox's is), while the +Store disk is an ext4 disk on both providers. The home mount is the one thing +two VMs share, and it holds records, not Stores: what a VM cannot see is +another project's Store, which is on another disk in another VM. + +A bundle may also carry the Sandbox image as `/images/<64 hex>.tar` +(#515 §4.7 V7, the namespaces backend's convention): it comes in with the rest +of the bundle — its own copy in each VM — and the worker is started with +`LAYERFS_IMAGE=sha256:<64 hex>` and `LAYERFS_IMAGE_ARCHIVE=/layerfs/bundle/images/<64 hex>.tar`, +while `deploymentEnv` gives clients that digest and that archive path as +`LAYERFS_IMAGE` and `LAYERFS_IMAGE_ARCHIVE`. The namespaces backend requires an +image (`capabilities.requires_image`), so a bundle without one starts no +Sandbox: the request is refused, naming the missing setting. + +The adapter's five steps, each attempted once, all for one named project: + +1. the project's VM exists and runs, from `settings.vm`, with its named disk + volume and the home bind mount — `create` once, else `start`; a record of + another home, or one whose mounts or port differ, is refused rather than + reused or repaired; +2. the sealed bundle is copied into that VM once per bundle hash (the hash of + its `manifest.json`), with a marker at `/layerfs/bundle/.sealed`; +3. `user.max_user_namespaces=0` is written and read back (V4; Apple Container's + guest mounts `/proc/sys` read-only, so its adapter remounts it first), then + the worker starts as root with its secret in a 0600 guest file — never in a + command line — and the pid it wrote; +4. the endpoint is written to `/worker/.json` once the worker + answers on `127.0.0.1:`, the VM's `-p 127.0.0.1::7070` + (V5: the Sandboxes share the VM's network, so a daemon's loopback port + stays inside it); +5. `runtime.vm.stop(project)` stops that VM and keeps its disk; `runtime.vm.delete( + project, { confirm: "delete" })` removes the VM record and keeps the Store + disk unless `discardStore`. **A delete is refused while that project + exists**: its Store lies on that disk and nowhere else, and this build has + no remote Store push (#506), so V8's "never delete a Store that was not + pushed" makes the refusal the only safe answer. `project.remove` is the + explicit removal: it stops the project's Sandboxes, closes its Store, then + deletes the VM and its disk together. + +Init runs as one more call to that project's worker: `platform.init` starts its +VM and copies the source folder into `/layerfs/import`, and `LayerFS.init` +reads it there. `deploymentEnv` gives clients and agents the VM's settings: +`LAYERFS_SANDBOX=native`, `LAYERFS_SANDBOX_BACKEND=namespaces` (an explicit +`LAYERFS_SANDBOX_BACKEND` in the caller's environment wins), `LAYERFS_DAEMON`, +`LAYERFS_LOCAL_DIR`, `LAYERFS_HOME=/layerfs/home`, the Store's +`LAYERFS_STORE_PATH`, this project's `LAYERFS_WORKER_ENDPOINT`, and the bundle +image's `LAYERFS_IMAGE` when it has one — no `LAYERFS_DOCKER` or `_VOLUME`. + +A VM home's `status()` reports `placement`, `docker: { level: "not-used" }` +(host Docker is not in its data path), `worker: null` (no single worker serves +the home) and `vm: { provider, placement, vms: [{ project, name, state, image, +home, store, created, worker, disk }] }`, one entry per project of +`project.list()`. The Docker-shaped helpers read host Docker and refuse +`Unsupported` with the reason: `volume.list`, `volume.remove`, `sandbox.list`, +`sandbox.list_leftovers` and `sandbox.clean_leftovers`; `sandbox.sweep` answers +`{ reclaimed: {}, failed: [], host: false }`. The SDK's +`project.list_sandboxes` reads the VM's Sandboxes. + +The real proof is `node api/runtime/test/verify/vm_real.mjs [microsandbox|apple-container]`: two projects, each in its own VM — Init +through each VM's worker, a fork, a write, a Sandbox, an exec as the nonroot +identity with user namespaces blocked, a Commit, a stop/start of one VM and a +worker restart, then a check that neither Store holds the other's Branch, and +that removing one project leaves the other's VM running. The unit tests +(`node api/runtime/test/vm_placement.mjs settings|flow|two|bundle|delete|hosts|apple-container|ssh`) +use a stand-in provider CLI. + +### The SSH provider (any Linux host) + +`vm:ssh` places the home on a Linux host the user already has — an EC2 +instance, a machine on the network, or a VM they made — instead of one this +runtime starts (#515 §6). The host and its disk are the user's: the adapter +makes no disk and formats nothing, it proves that the path given is an ext4 +filesystem on that host's own kernel (`uname -s` is `Linux`, `findmnt` or +`blkid` says `ext4`) and attaches it at `/layerfs/store` (a bind mount for a +directory, `mount -t ext4` for a block device) when it is somewhere else. + +```sh +LAYERFS_PLACEMENT=vm:ssh LAYERFS_SSH_HOST=ec2-…amazonaws.com \ +LAYERFS_SSH_IDENTITY=~/.ssh/deploy LAYERFS_SSH_STORE=/mnt/layerfs \ + layerfs-home +``` + +Settings (each also read from the environment named): + +| Setting | Environment | Default | +| --- | --- | --- | +| `vm.host` | `LAYERFS_SSH_HOST` | required | +| `vm.user` | `LAYERFS_SSH_USER` | `root` | +| `vm.port` | `LAYERFS_SSH_PORT` | `22` | +| `vm.identityFile` | `LAYERFS_SSH_IDENTITY` | none: the agent's keys | +| `vm.storePath` | `LAYERFS_SSH_STORE` | `/layerfs/store` | +| `vm.strictHostKeyChecking` | — | `yes` | +| `vm.knownHostsFile` | `LAYERFS_SSH_KNOWN_HOSTS` | the agent's | + +Key login only: a password is refused by name, the key is a path that goes to +`ssh -i` (its bytes never enter an argument list or a log), and +`StrictHostKeyChecking` is `yes` unless the setting says otherwise — the +loosest it accepts is `accept-new`. + +What differs from a microVM provider, and why: + +- **One host serves one Store.** The Store path is one directory on the user's +disk, so a second project naming the same host and path is refused `Busy`, +naming the VM that holds it, instead of writing one Store file twice. +- **The private stream is `ssh -L 127.0.0.1::127.0.0.1:7070`**, started +with `-o ExitOnForwardFailure=yes`: a busy local port fails at once, before +anything is uploaded or started. Its stderr goes to +`/-forward.log`, not a pipe, so the host process (CLI, test, an +integration) exits while the user's VM keeps running. `stop` ends the forward +and the remote worker (SIGTERM, then SIGKILL, bounded). +- **Init sources are copied, not shared.** A microVM bind-mounts the host home; +an SSH host shares no path with this machine, so `platform.init` copies the +source to `/layerfs/import/`. The host's `/layerfs/home` is therefore its +own directory, not this machine's home. +- **A delete never touches the disk.** `runtime.vm.delete` removes the session +record, the forward, `/layerfs/bundle` and `/layerfs/state`, and keeps the +Store; the refusal while the project exists is the same V8 rule as above. The +host's `/layerfs/home` and `/layerfs/import` also stay (they are LayerFS's own +copies, but removing the home entry would let a second Init duplicate a project +that the live Store already holds). + +Its session records live in `LAYERFS_SSH_STATE_DIR` (default +`~/.layerfs/ssh`), one JSON file per VM, and a forward that no longer runs reads +as a stopped VM, so the next `worker.start` re-opens it. The failure matrix is +`node api/runtime/test/verify/ssh_faults.mjs [port|store|ext4|host|drop|disconnect|remote|delete|shared]`: +a busy local port, a missing or non-ext4 Store path, a non-Linux host, an +unreachable network and a reconnected one, a dropped forward, a rebooted host, +a refused and a permitted delete, and a second project on the same host. ## Docker diff --git a/api/runtime/src/clients.js b/api/runtime/src/clients.js index dd52c0e64a..6cbb348fe2 100644 --- a/api/runtime/src/clients.js +++ b/api/runtime/src/clients.js @@ -8,8 +8,12 @@ import { callHome } from "./settings.js"; const IDLE_MS = 60_000; -/** The environment one project's client (and its agents' bash calls) use. */ +/** The environment one project's client (and its agents' bash calls) use. A + * platform may replace it whole (`deploymentEnv`), as a `vm:` home + * does: its deployment is native Sandboxes in the VM's namespace backend, not + * host Docker. */ export function deploymentEnv(settings, platform, project) { + if (platform.deploymentEnv) return platform.deploymentEnv(settings, project); return { HOME: callHome(settings), DOCKER: settings.docker, diff --git a/api/runtime/src/index.js b/api/runtime/src/index.js index f33f05084b..5a390526be 100644 --- a/api/runtime/src/index.js +++ b/api/runtime/src/index.js @@ -20,6 +20,7 @@ import { resolveSettings } from "./settings.js"; import { status } from "./status.js"; import linux from "./platform/linux/index.js"; import macos from "./platform/macos/index.js"; +import { vmPlatform } from "./platform/vm/index.js"; import windows from "./platform/windows/index.js"; import { list } from "./project/list.js"; import { InitJobs } from "./project/jobs.js"; @@ -34,6 +35,8 @@ import { list as listVolumes, remove as removeVolumes } from "./volume.js"; import { subscribe } from "./events/subscribe.js"; const PLATFORMS = { linux, macos, windows }; +/** The platform object `settings` selects: this host's, or one VM's (§6). */ +const platformFor = (settings) => (settings.placement.kind === "vm" ? vmPlatform(settings.placement.provider) : PLATFORMS[settings.platform]); /** Held only by `open`: the constructor refuses without it. */ const FACTORY = Symbol("LayerFSRuntime.open"); @@ -50,10 +53,15 @@ export class LayerFSRuntime { constructor(settings, token = undefined) { if (token !== FACTORY) throw new LayerFSError("InvalidInput", "use `await LayerFSRuntime.open(settings)`; there is no public constructor", null); this.settings = settings; - this.platform = PLATFORMS[settings.platform]; + this.platform = platformFor(settings); this.closed = false; - const endpoint = this.platform.endpoint(settings); - if (endpoint) useWorker(endpoint); + // one worker for the home, or (a `vm:` placement) one endpoint per project + const endpoints = this.platform.endpoints?.(settings); + if (endpoints) useWorker(endpoints); + else { + const endpoint = this.platform.endpoint?.(settings); + if (endpoint) useWorker(endpoint); + } this.clients = new Clients(this); const jobs = new InitJobs((prepared, report) => runInit(this, prepared, report)); const live = (method) => async (...args) => { @@ -69,10 +77,13 @@ export class LayerFSRuntime { }; this.status = live(async () => ({ ...(await status(this)), sweep: this.swept })); this.worker = { - start: live(() => this.platform.start(settings)), - restart: live((options) => { + start: live((project) => this.platform.start(settings, project)), + restart: live((first, second) => { + // `restart(project, { confirm })` on a `vm:` placement, `restart({ confirm })` elsewhere + const project = typeof first === "string" ? first : undefined; + const options = project ? second : first; if (options?.confirm !== "restart") throw new LayerFSError("InvalidInput", "a restart cuts off every project's Store calls; confirm it with { confirm: \"restart\" }", null); - return this.platform.restart(settings, options); + return this.platform.restart(settings, project, options); }), }; this.project = { @@ -93,6 +104,18 @@ export class LayerFSRuntime { list: live(() => listVolumes(this)), remove: live((names) => removeVolumes(this, names)), }; + /** `vm:` only: one project's VM (`status(project)`), else every project's; absent on a local placement. */ + if (this.platform.vm) { + this.vm = { + status: live(async (project) => { + const names = project ? [project] : (await this.project.list()).map((one) => one.name); + const reported = await this.platform.vm.status(settings, names); + return project ? reported.vm.vms[0] : reported.vm; + }), + stop: live((project) => this.platform.vm.stop(settings, project)), + delete: live((project, options) => this.platform.vm.delete(settings, project, options)), + }; + } /** The open-time sweep: null when off, else { state: running | done | failed, … }. */ this.swept = null; this.sweeping = null; diff --git a/api/runtime/src/platform/linux/index.js b/api/runtime/src/platform/linux/index.js index 58adc9577c..3770bd969a 100644 --- a/api/runtime/src/platform/linux/index.js +++ b/api/runtime/src/platform/linux/index.js @@ -4,7 +4,7 @@ // keeps root-only: `init` checks this process can write there first. import { accessSync, constants, existsSync, rmSync, statSync, statfsSync } from "node:fs"; import { join } from "node:path"; -import { docker, homeNames, refuse } from "../shared.js"; +import { docker, homeNames, makeStore, refuse, releaseStoreHolders, removeStore, storeHolders } from "../shared.js"; const NO_WORKER = "Linux runs Store calls in-process; there is no worker"; @@ -27,6 +27,10 @@ export default { }, path: (settings, hostPath) => hostPath, docker, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `${settings.volumes}/${volume}/_data/global/store.sqlite`, homeNames: (settings) => homeNames(settings.home), async removeHome(settings, name) { rmSync(join(settings.home, name), { recursive: true, force: true }); }, diff --git a/api/runtime/src/platform/macos/index.js b/api/runtime/src/platform/macos/index.js index 3a815089ca..5702bd8dcc 100644 --- a/api/runtime/src/platform/macos/index.js +++ b/api/runtime/src/platform/macos/index.js @@ -14,7 +14,7 @@ import { randomBytes } from "node:crypto"; import { mkdirSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { docker as cli, endpointFile, homeNames, ping, ready, refuse, writeEndpoint } from "../shared.js"; +import { docker as cli, endpointFile, homeNames, makeStore, ping, ready, refuse, releaseStoreHolders, removeStore, storeHolders, writeEndpoint } from "../shared.js"; const VOLUMES = "/var/lib/docker/volumes"; const WORKER_PORT = 7070; @@ -126,6 +126,10 @@ const platform = { }, path: (settings, hostPath) => hostPath, docker: cli, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `${VOLUMES}/${volume}/_data/global/store.sqlite`, homeNames: (settings) => homeNames(settings.home), async removeHome(settings, project) { rmSync(join(settings.home, project), { recursive: true, force: true }); }, diff --git a/api/runtime/src/platform/shared.js b/api/runtime/src/platform/shared.js index cb0b1dd7f1..8a1d826384 100644 --- a/api/runtime/src/platform/shared.js +++ b/api/runtime/src/platform/shared.js @@ -11,12 +11,24 @@ import { frame, readFrames } from "@layerfs/agent/internal"; export const run = promisify(execFile); export const refuse = (code, message, detail = null) => new LayerFSError(code, message, detail); +/** + * The refusal a Docker-shaped helper gives on a `vm:` home: host + * Docker holds none of that home's Stores or Sandboxes, so answering from it + * would describe another machine. + */ +export const noHostDocker = (runtime, what, instead) => + refuse("Unsupported", `${what} reads host Docker, and this home is placed in ${runtime.platform.name}: ${instead}`, null); + /** One Docker CLI call on the configured engine: its stdout, trimmed. */ export const docker = (settings, args, options = {}) => run("docker", ["-H", `unix://${settings.docker}`, ...args], { timeout: 5000, ...options }).then((r) => r.stdout.trim()); -/** Where a home's worker endpoint lives: `/worker/endpoint.json`. */ -export const endpointFile = (home) => join(home, "worker", "endpoint.json"); +/** + * Where a worker's endpoint lives: `/worker/endpoint.json` for a home + * whose one worker serves it (macOS, Windows), or `/worker/.json` + * when each project's Store is in its own VM (`vm:`). + */ +export const endpointFile = (home, project = undefined) => join(home, "worker", project ? `${project}.json` : "endpoint.json"); /** Writes an endpoint atomically, readable by this user only. */ export function writeEndpoint(file, info) { @@ -54,3 +66,39 @@ export function homeNames(home) { if (!existsSync(home)) return []; return readdirSync(home, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(home, e.name, "project.record"))).map((e) => e.name).sort(); } + +// The four Store hooks Init and project removal use, as Docker has them (the +// `local` placement): a project's Store is a labelled volume, its holders are +// containers, and removing the project removes the volume. A `vm:` placement +// implements the same four over the project's VM instead (vm/index.js). Each +// takes the runtime and the project `{ name, volume }`, so the hook uses that +// platform's own Docker CLI (Windows goes through WSL) or that project's VM. + +/** Container states that hold a volume but run nothing (Docker's; a namespace Sandbox's record has none). */ +export const STOPPED = new Set(["created", "exited", "dead"]); + +/** Init makes the project's Store volume before installing into it. */ +export async function makeStore(runtime, project) { + const { settings, platform } = runtime; + await platform.docker(settings, ["volume", "create", "--label", "layerfs.role=store", "--label", `layerfs.home=${settings.home}`, "--label", `layerfs.project=${project.name}`, project.volume], { timeout: 30_000 }); +} + +/** The containers holding the volume now: [{ id, state }], running or not. */ +export async function storeHolders(runtime, project) { + const { settings, platform } = runtime; + const rows = await platform.docker(settings, ["ps", "-a", "--filter", `volume=${project.volume}`, "--format", "{{.ID}} {{.State}}"]); + return rows.split("\n").filter(Boolean).map((line) => { const [id, state] = line.split(" "); return { id, state }; }); +} + +/** Removes the volume's holders: the running ones only when `force`. */ +export async function releaseStoreHolders(runtime, project, holders, { force = false } = {}) { + const { settings, platform } = runtime; + if (!holders.length) return; + await platform.docker(settings, ["rm", ...(force ? ["-f"] : []), ...holders.map((holder) => holder.id)], { timeout: 30_000 }); +} + +/** The volume itself, once nothing holds it. */ +export async function removeStore(runtime, project) { + const { settings, platform } = runtime; + await platform.docker(settings, ["volume", "rm", project.volume], { timeout: 30_000 }); +} \ No newline at end of file diff --git a/api/runtime/src/platform/vm/apple-container.js b/api/runtime/src/platform/vm/apple-container.js new file mode 100644 index 0000000000..dca57ac1e6 --- /dev/null +++ b/api/runtime/src/platform/vm/apple-container.js @@ -0,0 +1,151 @@ +// Apple Container (`container`, #515 §6, V3's third adapter). The VM is one +// container of the `container` CLI; the Store is a named volume the CLI keeps +// across stop and start, mounted at the Store directory; the private stream is +// `-p 127.0.0.1:H:G`. +// +// Verified against `container` 1.6.1 (build aa8d76b) on macOS 27.2 / M2, +// 2026-10-31, one VM per Store. What the CLI actually does, and how this file +// answers it: +// - `inspect NAME` prints JSON by default (it has no `--format`): an +// **array** of `[{ id, status: { state, +// startedDate, networks }, configuration: { image, labels, mounts, +// publishedPorts, creationDate, … } }]`. `status` is an object, not a +// string; mounts carry `destination`/`source`/`type.volume.name`; ports are +// `publishedPorts[].{containerPort,hostPort,hostAddress}`; the image +// reference and digest are `image.reference`/`image.descriptor.digest`. +// - a volume keeps its size: `volume create -s ` (the CLI's default is +// a sparse 504 GiB image), so `ensureStore` sizes it from `storeSize`; +// - `exec` needs `-i` for a caller's stdin (`cat > file` gets EOF at once +// without it), which `write` and `copy` rely on; +// - the guest's `/proc/sys` is mounted **read-only** and user namespaces are +// open by default: V4's block needs `mount -o remount,rw /proc/sys` first +// (CAP_SYS_ADMIN, which this adapter adds anyway), then the write; +// - `delete --force` ends a running container; `start`/`stop` keep the +// published port; `volume delete` removes the Store disk. +// It reads its facts strictly: an `inspect` answer that is not this shape is +// refused by name rather than guessed at, so a changed CLI shows up as an error +// and never as a VM this adapter believes is running. +import { answer, capture, readFile, writeFile } from "./cli.js"; + +const CLI = "container"; +const CREATE_MS = 120_000, CALL_MS = 30_000, EXEC_MS = 60_000; +/** The guest kernel shares FUSE, cgroup and loopback with these capabilities (#512: they were required). */ +const CAPABILITIES = ["--cap-add", "SYS_ADMIN", "--cap-add", "SYS_RESOURCE", "--cap-add", "NET_ADMIN"]; + +const call = (args, options = {}) => answer(CLI, args, { timeout: CALL_MS, ...options }); + +/** One `container exec`, plain streams, stdin kept open when the caller sends some. */ +const exec = (name, argv, { input = undefined, timeout = EXEC_MS } = {}) => + capture(CLI, ["exec", ...(input === undefined ? [] : ["-i"]), name, ...argv], { input, timeout }); + +const STRANGE = (what) => new Error(`the Apple Container CLI's ${what} is not the shape this adapter knows; check the CLI's output before using it`); + +/** The VM's record, or null when it does not exist. See the header: strict, never guessed. */ +export async function inspect(name) { + let text; + try { + text = await call(["inspect", name]); + } catch (error) { + const said = `${error.stderr ?? ""}${error.message ?? ""}`; + if (/not found|no such (container|sandbox)/i.test(said)) return null; + throw error; + } + const parsed = JSON.parse(text); + const record = Array.isArray(parsed) ? parsed[0] : parsed; + if (!record || typeof record !== "object") throw STRANGE("inspect"); + const configuration = record.configuration; + const status = record.status; + if (!configuration || typeof status !== "object" || typeof status?.state !== "string") throw STRANGE("inspect (status or configuration)"); + const mounts = configuration.mounts ?? []; + if (!Array.isArray(mounts)) throw STRANGE("inspect (mounts)"); + const mounted = (guest) => mounts.find((mount) => mount.destination === guest) ?? null; + const ports = configuration.publishedPorts ?? []; + if (!Array.isArray(ports)) throw STRANGE("inspect (publishedPorts)"); + return { + state: status.state.toLowerCase(), + image: configuration.image?.reference ?? null, + digest: configuration.image?.descriptor?.digest ?? null, + labels: configuration.labels ?? {}, + ports: ports.map((port) => ({ guest: Number(port.containerPort), host: Number(port.hostPort), bind: port.hostAddress ?? null })), + home: mounted("/layerfs/home")?.source ?? null, + // a named Store volume is named in the mount's type, and its source is the + // volume's image file on the host + store: mounted("/layerfs/store")?.type?.volume?.name ?? null, + created: configuration.creationDate ?? null, + }; +} + +/** Newly created named Store volumes in this session; cleaned up if create fails. */ +const newlyCreatedStores = new Set(); + +/** + * The Store disk: a named volume of `size`, made once. An existing volume keeps + * the size it was made with (the CLI has no resize), so only its presence is + * checked here. + */ +export async function ensureStore(name, size) { + const found = await capture(CLI, ["volume", "inspect", name]); + if (found.code === 0) return; + const made = await capture(CLI, ["volume", "create", "-s", size, name], { timeout: CREATE_MS }); + if (made.code !== 0) throw new Error(`could not create the Store volume ${name}: ${(made.stderr || made.stdout).trim()}`); + newlyCreatedStores.add(name); +} + +export async function create(config) { + const wasNew = newlyCreatedStores.has(config.storeName); + const existedBefore = wasNew ? false : (await capture(CLI, ["volume", "inspect", config.storeName])).code === 0; + try { + await call([ + "run", "--detach", "--name", config.name, "--cpus", String(config.cpus), "--memory", config.memory, + ...CAPABILITIES, + "--mount", `type=volume,source=${config.storeName},target=/layerfs/store`, + "--mount", `type=bind,source=${config.home},target=/layerfs/home`, + "--publish", `127.0.0.1:${config.hostPort}:${config.workerPort}`, + "--label", "layerfs.role=vm", "--label", `layerfs.home=${config.home}`, + config.image, "sleep", "infinity", + ], { timeout: CREATE_MS }); + newlyCreatedStores.delete(config.storeName); + return inspect(config.name); + } catch (error) { + if (wasNew || !existedBefore) { + newlyCreatedStores.delete(config.storeName); + await removeStore(config.storeName).catch(() => {}); + } + throw error; + } +} + +export const start = (name) => call(["start", name], { timeout: CREATE_MS }); +export const stop = (name) => call(["stop", name]); +export const restart = async (name) => { await stop(name); await start(name); }; +/** The container record only: the Store volume is a separate object and is never removed here. */ +export const remove = (name) => call(["delete", "--force", name], { timeout: CREATE_MS }); +export const removeStore = (storeName) => call(["volume", "delete", storeName]); + +/** + * V4: the VM-wide user-namespace block. The guest mounts `/proc/sys` + * read-only, so it is remounted read-write first; the caller reads the sysctl + * back, so a block that did not take is reported, not assumed. + */ +export const blockUserNamespaces = (name) => + exec(name, ["sh", "-c", "mount -o remount,rw /proc/sys 2>/dev/null; echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the VM's Store volume, or null when `df` cannot answer. */ +export async function storeFree(name) { + const { code, stdout } = await exec(name, ["df", "-kP", "/layerfs/store"]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +/** Copies a host folder into a guest folder. */ +export const copy = (name, source, destination) => call(["copy", source, `${name}:${destination}`], { timeout: CREATE_MS }); + +export const read = (name, path) => readFile(CLI, name, path, exec); +export const write = (name, path, text) => writeFile(CLI, name, path, text, exec); + +export default { + name: "apple-container", + defaultImage: "node:22-trixie-slim", + inspect, ensureStore, create, start, stop, restart, remove, removeStore, blockUserNamespaces, storeFree, read, write, copy, exec, +}; diff --git a/api/runtime/src/platform/vm/cli.js b/api/runtime/src/platform/vm/cli.js new file mode 100644 index 0000000000..e0675f6780 --- /dev/null +++ b/api/runtime/src/platform/vm/cli.js @@ -0,0 +1,45 @@ +// One VM provider CLI call, for the two providers in this folder: the command +// is spawned with pipes, its output stays apart from its error stream, and its +// exit code comes back. Nothing is retried, nothing is buffered beyond the +// call, and a call that runs past its timeout is killed and reported. +import { spawn } from "node:child_process"; +import { run } from "../shared.js"; + +/** + * `command args` with `input` on stdin, answered as { code, stdout, stderr }. + * `detach` marks a call whose started work outlives it and is not waited for. + */ +export function capture(command, args, { input = undefined, timeout = 60_000 } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] }); + const out = [], err = []; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`${command} ${args.slice(0, 2).join(" ")} did not end within ${timeout / 1000} s`)); + }, timeout); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => err.push(chunk)); + child.on("error", (error) => { clearTimeout(timer); reject(error); }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout: Buffer.concat(out).toString(), stderr: Buffer.concat(err).toString() }); + }); + if (input === undefined) child.stdin.end(); else child.stdin.end(input); + }); +} + +/** One call whose stdout, trimmed, is the command's answer. */ +export const answer = (command, args, options = {}) => + run(command, args, { timeout: 30_000, ...options }).then((result) => result.stdout.trim()); + +/** A guest file's text, or null when it does not exist. */ +export async function readFile(command, name, path, exec) { + const { code, stdout } = await exec(name, ["sh", "-c", "test -f \"$1\" && cat \"$1\"", "sh", path]); + return code === 0 ? stdout : null; +} + +/** Writes a guest file from stdin: the text never appears in a command line. */ +export async function writeFile(command, name, path, text, exec) { + const { code, stderr } = await exec(name, ["sh", "-c", "umask 077 && mkdir -p \"$(dirname \"$1\")\" && cat > \"$1\" && chmod 600 \"$1\"", "sh", path], { input: text }); + if (code !== 0) throw new Error(`could not write ${path} in ${name}: ${stderr.trim()}`); +} diff --git a/api/runtime/src/platform/vm/index.js b/api/runtime/src/platform/vm/index.js new file mode 100644 index 0000000000..6cf05cf905 --- /dev/null +++ b/api/runtime/src/platform/vm/index.js @@ -0,0 +1,453 @@ +// `vm:` placement (#515 §6, V9): **each of this home's projects — +// strictly, each Store — lives in one Linux VM of its own** (V1). That VM holds +// that project's Store on its own ext4 disk, shares that disk with the project's +// Sandboxes' daemons (Linux namespaces inside it, `LAYERFS_SANDBOX_BACKEND= +// namespaces`, H36), and runs its own worker with its own secret and its own +// private loopback channel (`worker/server.js`, one attempt per call, no replay). +// The host reaches only each VM's worker. +// +// This module is the platform object the runtime uses instead of the host's. +// Every step names the project, so the VM, its Store disk, its host port, its +// bundle copy and its worker secret are that project's alone: +// ① ensure the project's VM exists and runs, with its Store disk attached; +// ② copy the sealed bundle in, once per bundle hash and per VM; +// ③ start the worker as root with its secret, and block user namespaces +// (V4: `user.max_user_namespaces=0`); the namespaces backend refuses a +// host where that is not set, so step ③ also proves it; +// ④ open the private stream, `-p 127.0.0.1::`, and write the +// endpoint `useWorker` reads for that project; +// ⑤ stop the VM, or delete it: a delete never destroys a Store that was not +// pushed (V8), and there is no remote Store push yet (#506). +// The provider module behind it (`microsandbox.js`, `apple-container.js`) knows +// the platform; this file knows LayerFS's side and nothing platform-specific. +// Guest paths are in settings.js (GUEST): the home is a bind mount of the +// host's, the Store disk is mounted at the Store directory itself, and the +// bundle, Init sources, worker state and Sandbox directories live on the VM's +// root disk. +import { createHash, randomBytes } from "node:crypto"; +import { createServer } from "node:net"; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { connection } from "@layerfs/agent/internal"; +import { endpointFile, homeNames, ping, ready, refuse, writeEndpoint } from "../shared.js"; +import { callHome, GUEST } from "../../settings.js"; +import { NAME, record as projectRecord } from "../../project/list.js"; +import microsandbox from "./microsandbox.js"; +import appleContainer from "./apple-container.js"; +import ssh from "./ssh.js"; + +const PROVIDERS = { microsandbox, "apple-container": appleContainer, ssh }; +const WORKER_PORT = 7070; +const SERVER = "api/runtime/src/worker/server.js"; +/** Bundle hashing in the worker comes first, so readiness takes a while. */ +const READY_MS = 60_000; + +const providerOf = (settings) => { + const provider = PROVIDERS[settings.vm.provider]; + if (!provider) throw refuse("Unsupported", `no VM provider is implemented for vm:${settings.vm.provider}`, null); + return provider; +}; + +/** The project's VM and its Store disk: one VM per Store means one of each per project. */ +const vmNameOf = (settings, project) => `${settings.containerPrefix}${project}-vm`; +const storeNameOf = (settings, project) => `${settings.containerPrefix}${project}-vm-store`; + +/** A project name a VM can be named after and its driver paths hold. */ +function checked(project) { + if (typeof project !== "string" || !NAME.test(project)) throw refuse("InvalidInput", "a project name is required: each project's Store runs in its own VM", null); + return project; +} + +/** The project's Store file inside its own VM: the disk is the Store's directory. */ +const storeFile = () => `${GUEST.store}/global/store.sqlite`; + +/** A host loopback port this process just freed: a VM's published worker port. */ +const freePort = () => new Promise((resolve, reject) => { + const server = createServer(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); +}); + +/** + * Step ①: the project's VM runs, or it is created once from the home's settings. + * A record under our name that serves another home, or does not match what this + * project needs, is refused: nothing is replaced, restarted or repaired on a + * guess. + */ +async function ensure(settings, project) { + const provider = providerOf(settings), name = vmNameOf(settings, project), storeName = storeNameOf(settings, project); + const current = await provider.inspect(name); + if (current) { + if (current.labels?.["layerfs.home"] && current.labels["layerfs.home"] !== settings.home) { + throw refuse("Busy", `the VM ${name} serves the LayerFS home ${current.labels["layerfs.home"]}; give this home its own containerPrefix`, null); + } + if (current.store && current.store !== storeName) throw refuse("Failed", `the VM ${name} has the Store volume ${current.store}, not ${storeName}; delete it before using this project`, null); + if (current.home && current.home !== settings.home) throw refuse("Failed", `the VM ${name} mounts ${current.home} as its home, not ${settings.home}; delete it before using this project`, null); + const port = current.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (current.state === "running") { + if (!port) throw refuse("Failed", `the VM ${name} publishes no host port for ${WORKER_PORT}; delete it and start again`, null); + return { ...current, name, port }; + } + if (current.state !== "stopped") throw refuse("Failed", `the VM ${name} is ${current.state}; stop or delete it before using this project`, null); + await provider.start(name); + const started = await provider.inspect(name); + if (started?.state !== "running") throw refuse("Failed", `the VM ${name} did not start (${started?.state ?? "missing"})`, null); + const again = started.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (!again) throw refuse("Failed", `the VM ${name} publishes no host port for ${WORKER_PORT}; delete it and start again`, null); + return { ...started, name, port: again }; + } + if (provider.ensureStore) await provider.ensureStore(storeName, settings.vm.storeSize); + const hostPort = await freePort(); + const created = await provider.create({ + ...settings.vm, + name, image: settings.vm.image, cpus: settings.vm.cpus, memory: settings.vm.memory, + storeName, storeSize: settings.vm.storeSize, home: settings.home, hostPort, workerPort: WORKER_PORT, + }); + if (created?.state !== "running") throw refuse("Failed", `the VM ${name} was created but is ${created?.state ?? "missing"}; its record is kept for inspection, not replayed`, null); + const port = created.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (!port) throw refuse("Failed", `the VM ${name} was created without its port forward; delete it before using this project`, null); + return { ...created, name, port }; +} + +/** The sealed bundle's identity: its manifest hash, which the worker checks file by file. */ +function bundleIdentity(bundle) { + if (!bundle || !existsSync(join(bundle, "manifest.json"))) throw refuse("InvalidInput", "a VM placement needs a sealed Linux bundle (settings.bundle)", null); + const text = readFileSync(join(bundle, "manifest.json"), "utf8"); + const manifest = JSON.parse(text); + if (manifest.status !== "sealed" || manifest.platform !== "linux") throw refuse("InvalidInput", "a VM placement needs a sealed Linux bundle", null); + return { build: manifest.build_id, hash: createHash("sha256").update(text).digest("hex"), image: bundleImage(bundle) }; +} + +/** + * The Sandbox image a sealed bundle carries (#515 §4.7 V7, the H38 namespaces + * backend's convention): `/images/<64 hex>.tar`, an uncompressed + * `docker save` archive named by the image's config digest. It is copied in + * with the rest of the bundle — each VM has its own copy, because each VM is + * one Store — and the worker is told about it as a guest path. The namespaces + * backend requires an image, so a bundle of none leaves its Sandboxes to refuse + * the start (`deploy.image`/`deploy.imageArchive` are not configured). + */ +function bundleImage(bundle) { + const folder = join(bundle, "images"); + const found = existsSync(folder) ? readdirSync(folder).filter((name) => /^[0-9a-f]{64}\.tar$/.test(name)).sort() : []; + if (found.length > 1) throw refuse("InvalidInput", `the bundle holds ${found.length} Sandbox images (${found.join(", ")}); a VM placement describes one deployment image`, null); + if (!found.length) return null; + return { digest: `sha256:${found[0].slice(0, -4)}`, archive: `${GUEST.bundle}/images/${found[0]}` }; +} + +/** Step ②: the bundle, copied into this VM once per hash; a changed bundle replaces the copy. */ +async function install(settings, provider, vm) { + const { build, hash } = bundleIdentity(settings.bundle); + const marker = await provider.read(vm.name, `${GUEST.bundle}/.sealed`); + if (marker?.trim() === hash) return { build, copied: false }; + const { code, stderr } = await provider.exec(vm.name, ["rm", "-rf", GUEST.bundle]); + if (code !== 0) throw refuse("Failed", `could not replace ${GUEST.bundle} in ${vm.name}: ${stderr.trim()}`, null); + await provider.copy(vm.name, settings.bundle, GUEST.bundle); + await provider.write(vm.name, `${GUEST.bundle}/.sealed`, `${hash}\n`); + return { build, copied: true }; +} + +/** + * Step ③, the guest's precondition for the namespace backend: the Sandbox + * directory (`GUEST.local`, on the Store disk) and the VM-wide user-namespace + * block (V4). The block is read back — the namespaces backend refuses a host + * without it — so a VM that cannot block them is reported here, by fact. + */ +async function prepareGuest(provider, vm) { + // the namespaces backend wants a root owned by this user (root), not + // group/other writable, and traversable by others (0755): the nonroot + // command identity enters it + const made = await provider.exec(vm.name, ["sh", "-c", "umask 022 && mkdir -p \"$1\" && chmod 755 \"$1\"", "sh", GUEST.local]); + if (made.code !== 0) throw refuse("Failed", `could not make ${GUEST.local} in ${vm.name}: ${made.stderr.trim()}`, null); + const { code, stderr } = await provider.blockUserNamespaces(vm.name); + if (code !== 0) throw refuse("Unsupported", `the VM ${vm.name} cannot block user namespaces (${stderr.trim() || `exit ${code}`}); a Sandbox command could copy the Workspace mount into its own`, null); + const left = (await provider.read(vm.name, "/proc/sys/user/max_user_namespaces"))?.trim(); + if (left !== "0") throw refuse("Unsupported", `the VM ${vm.name} still allows user namespaces (${left ?? "unreadable"})`, null); +} + +/** The worker's own settings, read by `sh` from a 0600 file so the secret is in no command line. */ +const workerEnv = (secret, image) => [ + `LAYERFS_WORKER_SECRET=${secret}`, + `LAYERFS_WORKER_LISTEN=0.0.0.0:${WORKER_PORT}`, + `LAYERFS_HOME=${GUEST.home}`, + `LAYERFS_BUNDLE=${GUEST.bundle}`, + `LAYERFS_NATIVE=${GUEST.bundle}/layerfs.node`, + // a Sandbox reopened from its record has no daemon path to look beside + `LAYERFS_SANDBOX_HELPER=${GUEST.bundle}/layerfs-sandbox-helper`, + // #515 §4.7 V7 (H38's backend): this VM's own copy of the Sandbox image + ...(image ? [`LAYERFS_IMAGE=${image.digest}`, `LAYERFS_IMAGE_ARCHIVE=${image.archive}`] : []), + "", +].join("\n"); + +/** + * Step ③ and ④: the worker as root, its secret in a 0600 file, then the + * project's endpoint once it answers on the private port. A worker left by an + * earlier start that no longer answers is ended by the pid that start wrote (a + * bounded cleanup, not a replay); one launch, one readiness wait. A worker that + * does not answer reports the last lines of its log. + */ +async function launch(settings, provider, vm, project, bundle) { + const secret = randomBytes(32).toString("hex"); + const file = endpointFile(settings.home, project); + await provider.write(vm.name, `${GUEST.state}/worker.env`, workerEnv(secret, bundle.image)); + const script = [ + // one bounded cleanup: a worker an earlier start left is ended, and its + // port is waited for, so this start's worker can take it + `if [ -f ${GUEST.state}/worker.pid ]; then kill "$(cat ${GUEST.state}/worker.pid)" 2>/dev/null;`, + `for i in $(seq 1 50); do kill -0 "$(cat ${GUEST.state}/worker.pid)" 2>/dev/null || break; sleep 0.1; done; fi;`, + `set -a; . ${GUEST.state}/worker.env; set +a;`, + `nohup node ${GUEST.bundle}/${SERVER} >> ${GUEST.state}/worker.log 2>&1 &`, + `echo $! > ${GUEST.state}/worker.pid`, + ].join(" "); + const started = await provider.exec(vm.name, ["sh", "-c", script]); + if (started.code !== 0) throw refuse("Failed", `the worker did not start in ${vm.name}: ${(started.stderr || started.stdout).trim()}`, null); + const info = { home: GUEST.home, port: vm.port, secret, build: bundle.build, vm: { provider: settings.vm.provider, name: vm.name, project } }; + try { + await ready(info, READY_MS, `the worker in ${vm.name}`); + } catch (error) { + const log = (await provider.read(vm.name, `${GUEST.state}/worker.log`).catch(() => null))?.trim().split("\n").slice(-8).join("\n"); + throw refuse("Failed", `${error.message}${log ? `; its log ends: ${log}` : ""}`, null); + } + mkdirSync(join(settings.home, "worker"), { recursive: true, mode: 0o700 }); + writeEndpoint(file, info); + return info; +} + +/** The endpoint the project's worker was last written at, or null. */ +function endpoint(settings, project) { + try { return JSON.parse(readFileSync(endpointFile(settings.home, project), "utf8")); } catch { return null; } +} + +/** Nothing to start when the project's worker already answers with this build. */ +async function answering(settings, project, expected) { + const info = endpoint(settings, project); + if (!info || (expected && info.build !== expected)) return null; + return (await ping(info))?.ready ? info : null; +} + +/** One project's VM record and, when it runs, whether its worker answers. It starts nothing. `disk` + * asks each running VM for its Store disk's free bytes (one guest call), which `status` reports. */ +async function entry(settings, project, disk = false) { + const provider = providerOf(settings), name = vmNameOf(settings, project); + let found; + try { found = await provider.inspect(name); } catch (error) { + return { project, name, state: "unknown", error: error.message, worker: { state: "unknown", error: error.message } }; + } + if (!found) return { project, name, state: "missing", worker: { state: "missing" } }; + const vm = { project, name, state: found.state, image: found.image, home: found.home, store: found.store, created: found.created }; + if (found.state !== "running") return { ...vm, worker: { state: found.state, build: null } }; + const info = endpoint(settings, project); + const reply = info ? await ping(info) : null; + const worker = { state: reply?.ready ? "running" : "unknown", answering: !!reply?.ready, build: reply?.build ?? info?.build ?? null, port: info?.port ?? null }; + if (!disk) return { ...vm, worker }; + return { ...vm, worker, disk: { free: await provider.storeFree(name, GUEST.store).catch(() => null) } }; +} + +/** `{ project, worker, vm }` of one project: what `worker.start` and `vm.stop` answer. */ +async function stateOf(settings, project) { + const vm = await entry(settings, checked(project)); + return { project: vm.project, worker: vm.worker, vm }; +} + +/** The runtime's `worker.start(project)`: steps ①–④ for that project's VM. */ +async function start(settings, project) { + checked(project); + const provider = providerOf(settings), bundle = bundleIdentity(settings.bundle); + const held = await answering(settings, project, bundle.build); + if (held) return stateOf(settings, project); + const vm = await ensure(settings, project); + await install(settings, provider, vm); + await prepareGuest(provider, vm); + await launch(settings, provider, vm, project, bundle); + return stateOf(settings, project); +} + +/** One `provider restart`, then the steps a rebooted VM needs again (③, ④). */ +async function restart(settings, project) { + checked(project); + const provider = providerOf(settings), bundle = bundleIdentity(settings.bundle); + const vm = await ensure(settings, project); + if (provider.restart) await provider.restart(vm.name); + else { await provider.stop(vm.name); await provider.start(vm.name); } + rmSync(endpointFile(settings.home, project), { force: true }); + await install(settings, provider, vm); + await prepareGuest(provider, vm); + await launch(settings, provider, vm, project, bundle); + return stateOf(settings, project); +} + +/** Every project's VM, or all of this home's: `status()` of the runtime and of the `vm` group. */ +async function status(settings, projects = []) { + const provider = providerOf(settings); + const vms = []; + for (const project of projects) vms.push(await entry(settings, checked(project), true)); + return { worker: null, vm: { provider: settings.vm.provider, placement: settings.placement.name, vms } }; +} + +/** Step ⑤: stop one project's VM; its Store disk and its Store stay. */ +async function stop(settings, project) { + const provider = providerOf(settings), name = vmNameOf(settings, checked(project)); + const found = await provider.inspect(name); + if (!found) return { project, name, state: "missing" }; + if (found.state !== "stopped") await provider.stop(name); + const after = await provider.inspect(name); + rmSync(endpointFile(settings.home, project), { force: true }); + return { project, name, state: after?.state ?? "missing" }; +} + +/** + * Step ⑤, delete (V8): one project's VM record and, only with `discardStore`, + * its Store disk. A delete that would leave a project without its only Store + * copy is refused: that project's Store lies on this VM's disk and nowhere + * else, and no remote Store push exists (#506). + */ +async function remove(settings, project, options = {}) { + const provider = providerOf(settings), name = vmNameOf(settings, checked(project)); + if (options?.confirm !== "delete") throw refuse("InvalidInput", `a VM delete ends the project's VM; confirm it with { confirm: "delete" }`, null); + const found = await provider.inspect(name); + if (!found) return { project, name, state: "missing", store: "kept" }; + if (exists(settings, project)) { + throw refuse("Unsupported", `the Store of ${project} is on ${name}'s disk and nowhere else; a VM delete never destroys a Store that was not pushed, and this build has no remote Store push (#506). Remove the project first`, null); + } + await provider.remove(name); + if (options.discardStore) await provider.removeStore(storeNameOf(settings, project)); + rmSync(endpointFile(settings.home, project), { force: true }); + return { project, name, state: "deleted", store: options.discardStore ? "discarded" : "kept" }; +} + +/** Whether this home still holds the project: its record, or its entry in the home. */ +function exists(settings, project) { + if (projectRecord(settings, project)) return true; + try { + return readdirSync(settings.home, { withFileTypes: true }).some((e) => e.isDirectory() && e.name === project && existsSync(join(settings.home, e.name, "project.record"))); + } catch { return false; } +} + +/** The four hooks Init and project removal use instead of host Docker (§4.2 B2, B9). */ +const stores = { + /** + * Init examines the Store's directory before it installs into it (the kernel + * guard checks its filesystem), so the project's Store disk is mounted and + * its directory made here, as a Docker volume is on a local placement. + */ + async makeStore(runtime, project) { + const provider = providerOf(runtime.settings), name = vmNameOf(runtime.settings, project.name); + const { code, stderr } = await provider.exec(name, ["sh", "-c", "umask 022 && mkdir -p \"$1\" && chmod 755 \"$1\"", "sh", GUEST.store]); + if (code !== 0) throw refuse("Failed", `could not make ${GUEST.store} in ${name}: ${stderr.trim()}`, null); + }, + /** The Sandboxes holding the project's Store now: its running Sandboxes, by number. A stopped VM runs none. */ + async storeHolders(runtime, project) { + const { settings } = runtime, provider = providerOf(settings); + const found = await provider.inspect(vmNameOf(settings, project.name)); + if (!found || found.state !== "running") return []; + const held = await runtime.clients.use(project, (client) => client.project.list_sandboxes(1000, 0)); + return held.items.map(({ sandbox }) => ({ id: sandbox, state: "running" })); + }, + /** Stops them the way the SDK does: `sandbox.close`, one per Sandbox. */ + async releaseStoreHolders(runtime, project, holders) { + const { settings } = runtime, provider = providerOf(settings); + const found = await provider.inspect(vmNameOf(settings, project.name)); + if (!found || found.state !== "running") return; + await runtime.clients.use(project, async (client) => { + for (const holder of holders) await client.sandbox.close(holder.id); + }); + }, + /** Removes the project's VM and its Store disk: no Store copy is left anywhere. */ + async removeStore(runtime, project) { + const { settings } = runtime, provider = providerOf(settings), name = vmNameOf(settings, project.name); + const found = await provider.inspect(name); + if (found) await provider.remove(name); + await provider.removeStore(storeNameOf(settings, project.name)); + rmSync(endpointFile(settings.home, project.name), { force: true }); + }, +}; + +/** The guest home, and the deployment inside the project's VM: native Sandboxes, the namespace backend. */ +export function deploymentEnv(settings, project) { + const image = settings.bundle ? bundleImage(settings.bundle) : null; + return { + HOME: callHome(settings), + SANDBOX: "native", + SANDBOX_BACKEND: process.env.LAYERFS_SANDBOX_BACKEND || "namespaces", + DAEMON: GUEST.daemon, + LOCAL_DIR: GUEST.local, + // the Sandbox image this VM's bundle carries, when it carries one (V7): + // the digest and, for the namespaces backend, the archive in this VM's + // own copy of the bundle that holds it (both guest paths) + ...(image ? { IMAGE: image.digest, IMAGE_ARCHIVE: image.archive } : {}), + STORE_PATH: storeFile(), + CONTAINER_PREFIX: settings.containerPrefix, + // a process an integration launches for this project reaches this VM's worker + WORKER_ENDPOINT: endpointFile(settings.home, project.name), + }; +} + +/** + * The platform object of a `vm:` home. `hostDocker`/`hostSandboxes` + * tell the runtime that host Docker is not this home's: its Stores are + * directories on each VM's own disk and its Sandboxes are Linux namespaces + * there, so `volume.list`, `volume.remove`, `sandbox.list`, + * `sandbox.list_leftovers` and `sandbox.clean_leftovers` refuse with that + * reason instead of answering from the host, and `sandbox.sweep` answers + * `{ host: false }`. `endpoints` answers, per project, the endpoint file + * `@layerfs/agent` sends that project's calls to. + */ +export function vmPlatform(settingName) { + if (!PROVIDERS[settingName]) throw refuse("Unsupported", `no VM provider is implemented for vm:${settingName}`, null); + return { + name: `vm:${settingName}`, + hostDocker: false, + hostSandboxes: false, + endpoints: (settings) => (project) => endpointFile(settings.home, project), + status, + start, + restart, + /** Init runs in that project's worker; the source folder is copied into its VM first. */ + async init(settings, project) { + await start(settings, project.name); + const provider = providerOf(settings), name = vmNameOf(settings, project.name); + const target = importPath(project.source); + const { code, stderr } = await provider.exec(name, ["rm", "-rf", target]); + if (code !== 0) throw refuse("Failed", `could not clear ${target} in ${name}: ${stderr.trim()}`, null); + await provider.copy(name, project.source, target); + }, + /** Where the SDK inside the VM reads a host folder: its copy under `/layerfs/import`. */ + path: (settings, hostPath) => (hostPath === settings.home || hostPath.startsWith(`${settings.home}/`) ? hostPath.replace(settings.home, GUEST.home) : importPath(hostPath)), + deploymentEnv, + docker: () => { + throw refuse("Unsupported", `this home is placed in vm:${settingName}: its Stores and Sandboxes are inside each project's VM, so host Docker does not describe it`, null); + }, + storePath: () => storeFile(), + homeNames: (settings) => homeNames(settings.home), + async removeHome(settings, name) { + rmSync(join(settings.home, name), { recursive: true, force: true }); + }, + /** Before a Store is deleted, the project's worker closes the Store it holds open. */ + async releaseStore(settings, project) { + const info = endpoint(settings, project.name); + if (!info) return; + const answer = await connection(info).call({ fn: "releaseStore", args: { home: GUEST.home, storePath: storeFile() } }); + if (answer.refused) throw refuse(answer.refused.code, answer.refused.message, answer.refused.detail); + }, + /** Free bytes of every project VM's Store disk, as that VM sees them. */ + async free(settings) { + const provider = providerOf(settings); + let total = null; + for (const project of homeNames(settings.home)) { + const free = await provider.storeFree(vmNameOf(settings, project), GUEST.store).catch(() => null); + if (free !== null) total = (total ?? 0) + free; + } + return total; + }, + ...stores, + vm: { status, stop, delete: remove }, + }; +} + +/** A host folder's copy inside the VM, named by its path so the mapping needs no state. */ +export function importPath(hostPath) { + return `${GUEST.import}/${createHash("sha256").update(hostPath).digest("hex").slice(0, 16)}`; +} diff --git a/api/runtime/src/platform/vm/microsandbox.js b/api/runtime/src/platform/vm/microsandbox.js new file mode 100644 index 0000000000..38cd030fc5 --- /dev/null +++ b/api/runtime/src/platform/vm/microsandbox.js @@ -0,0 +1,115 @@ +// Microsandbox (`msb` 0.7.8, libkrun microVMs): the VM is one sandbox record +// and the Store is a named `disk` volume that survives stop and start (V8). +// #512 read this provider's facts; the gaps it named are handled here: +// - the name is the only identity: every read goes through `msb inspect` by +// name, and a record that is not ours (another home) is refused, never +// reused; +// - a failed create can leave hidden state under the name, so `create` is +// called once and its failure is reported as it is (the message says what +// `msb` suggests); nothing is replayed or replaced on a guess; +// - `msb exec -u` capability drop is unverified, so the worker, the helper +// and the daemon run as root (the guest's only identity) and the +// namespaces backend drops each command itself (#515 §4.3). +// One CLI call per call, one attempt, no retry. +import { existsSync, rmSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { answer, capture, readFile, writeFile } from "./cli.js"; + +const CLI = "msb"; +const CREATE_MS = 120_000, CALL_MS = 30_000, EXEC_MS = 60_000; + +const call = (args, options = {}) => answer(CLI, args, { timeout: CALL_MS, ...options }); + +/** + * One `msb exec` in the VM. `--no-tty` keeps the streams plain (stdout and + * stderr arrive separately). The worker detaches itself with a shell, so this + * call ends when its shell does. + */ +const exec = (name, argv, { input = undefined, timeout = EXEC_MS } = {}) => + capture(CLI, ["exec", name, "--no-tty", "--", ...argv], { input, timeout }); + +/** The VM's record as `msb inspect --format json` gives it, or null when it does not exist. */ +export async function inspect(name) { + let text; + try { + text = await call(["inspect", "--format", "json", name]); + } catch (error) { + const said = `${error.stderr ?? ""}${error.message ?? ""}`; + if (/sandbox not found|no such sandbox/i.test(said)) return null; + throw error; + } + const record = JSON.parse(text); + const config = record.active_config ?? record.config ?? {}; + const mounted = (guest) => (config.mounts ?? []).find((mount) => mount.guest === guest) ?? null; + return { + state: String(record.status ?? "").toLowerCase(), + image: config.image?.Oci?.reference ?? null, + digest: config.image?.Oci?.manifest_digest ?? null, + labels: config.labels ?? {}, + ports: (config.network?.ports ?? []).map((port) => ({ guest: port.guest_port, host: port.host_port, bind: port.host_bind })), + home: mounted("/layerfs/home")?.host ?? null, + store: mounted("/layerfs/store")?.name ?? null, + created: record.created_at ?? null, + }; +} + +/** + * Step ①: the VM, from `config` when it does not exist yet. Answers the record + * read back after the call, so a create that ended without a usable VM is + * reported by the caller from fact, not from the command's exit code. + */ +export async function create(config) { + const dir = join(homedir(), ".microsandbox", "sandboxes", config.name); + const existedBefore = existsSync(dir); + try { + await call([ + "create", "--name", config.name, "--cpus", String(config.cpus), "--memory", config.memory, + "--mount-named", `${config.storeName}:/layerfs/store:kind=disk,size=${config.storeSize}`, + "-v", `${config.home}:/layerfs/home`, + "-p", `127.0.0.1:${config.hostPort}:${config.workerPort}`, + "--label", "layerfs.role=vm", "--label", `layerfs.home=${config.home}`, + config.image, + ], { timeout: CREATE_MS }); + return inspect(config.name); + } catch (error) { + if (!existedBefore && existsSync(dir)) { + rmSync(dir, { recursive: true, force: true }); + } + throw error; + } +} + +export const start = (name) => call(["start", name], { timeout: CREATE_MS }); +export const stop = (name) => call(["stop", name]); +export const restart = (name) => call(["restart", name], { timeout: CREATE_MS }); +/** The VM record only: its Store volume is a separate object and is never removed here. */ +export const remove = (name) => call(["remove", "--force", name], { timeout: CREATE_MS }); +export const removeStore = (storeName) => call(["volume", "remove", storeName]); + +/** + * V4: the VM-wide user-namespace block. Microsandbox's guest keeps + * `/proc/sys` writable; the caller reads the sysctl back, so a block that did + * not take is reported, not assumed. + */ +export const blockUserNamespaces = (name) => exec(name, ["sh", "-c", "echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the VM's Store disk, or null when `df` cannot answer. */ +export async function storeFree(name) { + const { code, stdout } = await exec(name, ["df", "-kP", "/layerfs/store"]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +/** Copies a host folder's contents into a guest folder. */ +export const copy = (name, source, destination) => call(["copy", source, `${name}:${destination}`], { timeout: CREATE_MS }); + +export const read = (name, path) => readFile(CLI, name, path, exec); +export const write = (name, path, text) => writeFile(CLI, name, path, text, exec); + +export default { + name: "microsandbox", + defaultImage: "node:22-trixie-slim", + inspect, create, start, stop, restart, remove, removeStore, blockUserNamespaces, storeFree, read, write, copy, exec, +}; diff --git a/api/runtime/src/platform/vm/ssh.js b/api/runtime/src/platform/vm/ssh.js new file mode 100644 index 0000000000..1008956867 --- /dev/null +++ b/api/runtime/src/platform/vm/ssh.js @@ -0,0 +1,395 @@ +// `vm:ssh` placement (#515 §6, V3): any Linux host reachable over SSH — an EC2 +// instance or a VM the user made, instead of one this runtime creates. The host +// and its Store disk are the user's: this adapter never makes a disk, it only +// proves that the path given is on ext4, on that host's own kernel. +// +// The five adapter steps (§6) on this provider: +// ① the "VM" is the host; the Store disk is the path `storePath` (a block +// device mounted at /layerfs/store, or a directory bind-mounted there); +// ② the bundle is copied in over SSH (one `tar` stream per bundle hash); +// ③ the worker runs as root with its secret, exactly as on a microVM; +// ④ the private stream is `ssh -L 127.0.0.1::127.0.0.1:7070`; +// ⑤ stop ends the forward and the worker; delete never touches the disk. +// +// Facts this file depends on, and why they are checked rather than assumed: +// - the host must answer `uname -s` with Linux: a macOS/BSD host has no +// namespaces and no /proc/sys/user; +// - the Store path must be an ext4 filesystem on that host: LayerFS's Store +// names its files and relies on that filesystem's ordering; +// - key login only, from a file; a password never reaches argv or a log; +// - host key checking stays on (`StrictHostKeyChecking=yes` unless the +// setting says otherwise, `accept-new` at the loosest); +// - `-o ExitOnForwardFailure=yes` so a busy local port fails at once, before +// anything is uploaded or started, instead of leaving a half-made VM. +// +// The forward runs with its stderr in a log file, not a pipe: a host process +// (the CLI, a test) must be able to exit while the user's VM keeps running, and +// a referenced stdio pipe would hold its event loop open. `unref` on the child +// alone does not release the pipes, so they are files. +import { spawn } from "node:child_process"; +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { LayerFSError } from "@layerfs/agent"; +import { readFile, writeFile } from "./cli.js"; + +const EXEC_MS = 60_000, COPY_MS = 120_000, FORWARD_READY_MS = 600, END_MS = 3_000; +const WORKER_PORT_DEFAULT = 7070; +const STORE_PATH_DEFAULT = "/layerfs/store"; + +const SESSIONS = new Map(); + +const refuse = (code, message) => new LayerFSError(code, message, null); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Where this provider keeps its session records: one JSON file per VM name. */ +export const stateDir = () => process.env.LAYERFS_SSH_STATE_DIR || join(homedir(), ".layerfs", "ssh"); + +function sessionFile(name) { + const dir = stateDir(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return join(dir, `${name}.json`); +} + +function forwardLog(name) { + const dir = stateDir(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return join(dir, `${name}-forward.log`); +} + +function loadSession(name) { + if (SESSIONS.has(name)) return SESSIONS.get(name); + const file = sessionFile(name); + if (!existsSync(file)) return null; + try { + const session = JSON.parse(readFileSync(file, "utf8")); + SESSIONS.set(name, session); + return session; + } catch { + return null; + } +} + +function saveSession(session) { + SESSIONS.set(session.name, session); + writeFileSync(sessionFile(session.name), JSON.stringify(session, null, 2), { mode: 0o600 }); +} + +function deleteSession(name) { + SESSIONS.delete(name); + rmSync(sessionFile(name), { force: true }); + rmSync(forwardLog(name), { force: true }); +} + +/** Whether a pid still runs. Only a plain liveness probe; it proves nothing else. */ +export function isAlive(pid) { + if (!pid || typeof pid !== "number") return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error.code === "EPERM"; + } +} + +/** One argv token for the remote POSIX shell: never left to the shell's word splitting. */ +export function shellEscape(arg) { + const text = String(arg); + if (/^[A-Za-z0-9_\-./=:@%+,]+$/.test(text)) return text; + return `'${text.replace(/'/g, "'\\''")}'`; +} + +/** + * The ssh arguments both `exec` and the forward share. `BatchMode=yes` keeps a + * password prompt from ever being tried (this provider does not support one); + * the identity file is a path here, its content stays in the file. + */ +function sshArgs(session) { + const args = ["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"]; + if (session.port && session.port !== 22) args.push("-p", String(session.port)); + if (session.identityFile) args.push("-i", session.identityFile); + const strict = session.strictHostKeyChecking; + args.push("-o", `StrictHostKeyChecking=${strict === false ? "accept-new" : typeof strict === "string" ? strict : "yes"}`); + if (session.knownHostsFile) args.push("-o", `UserKnownHostsFile=${session.knownHostsFile}`); + if (session.controlMaster && session.controlPath) { + args.push("-o", "ControlMaster=auto", "-o", `ControlPath=${session.controlPath}`, "-o", "ControlPersist=60s"); + } + return args; +} + +const destinationOf = (session) => (session.user ? `${session.user}@${session.host}` : session.host); + +/** One command on the host, its streams kept apart; a call past `timeout` is killed and reported. */ +function remote(session, argv, { input = undefined, timeout = EXEC_MS } = {}) { + const args = [...sshArgs(session), destinationOf(session), "--", ...argv.map(shellEscape)]; + return new Promise((resolve, reject) => { + const child = spawn("ssh", args, { stdio: ["pipe", "pipe", "pipe"] }); + const out = [], err = []; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`ssh ${argv.join(" ")} did not end within ${timeout / 1000} s`)); + }, timeout); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => err.push(chunk)); + child.on("error", (error) => { clearTimeout(timer); reject(error); }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout: Buffer.concat(out).toString(), stderr: Buffer.concat(err).toString() }); + }); + if (input === undefined) child.stdin.end(); else child.stdin.end(input); + }); +} + +/** The last `lines` lines of the forward's log, for an error message. */ +function logTail(name, lines = 3) { + try { + return readFileSync(forwardLog(name), "utf8").trim().split("\n").slice(-lines).join("; "); + } catch { + return ""; + } +} + +/** + * Step ④: the private stream. `-o ExitOnForwardFailure=yes` makes a busy local + * port an immediate failure, so nothing is uploaded or started on a VM that + * cannot be reached. A forward that exits within the readiness window is + * reported with its own reason; one that answers will be ended by `endForward`. + */ +async function startForward(session, hostPort, workerPort) { + const log = forwardLog(session.name); + const fd = openSync(log, "w", 0o600); + const child = spawn("ssh", [ + "-N", + "-o", "ExitOnForwardFailure=yes", + ...sshArgs(session), + "-L", `127.0.0.1:${hostPort}:127.0.0.1:${workerPort}`, + destinationOf(session), + ], { stdio: ["ignore", "ignore", fd] }); + closeSync(fd); + + const ended = new Promise((resolve) => child.once("exit", (code, signal) => resolve({ code, signal }))); + const first = await Promise.race([ended.then((value) => ({ ended: true, ...value })), delay(FORWARD_READY_MS).then(() => ({ ended: false }))]); + child.removeAllListeners("exit"); + if (first.ended) { + child.unref(); + throw new Error(`ssh -L 127.0.0.1:${hostPort} exited (${first.code ?? first.signal}): ${logTail(session.name) || "no reason given"}`); + } + child.on("exit", () => { SESSIONS.delete(session.name); }); + child.unref(); + return child; +} + +/** Ends a forward, bounded: SIGTERM, then SIGKILL; never waits on it indefinitely. */ +async function endForward(session) { + const pid = session.forwardPid; + if (!isAlive(pid)) { session.forwardPid = null; return; } + try { process.kill(pid, "SIGTERM"); } catch { /* already gone */ } + for (let waited = 0; waited < END_MS && isAlive(pid); waited += 50) await delay(50); + if (isAlive(pid)) { + try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } + for (let waited = 0; waited < 1_000 && isAlive(pid); waited += 50) await delay(50); + } + session.forwardPid = null; +} + +/** The record as the runtime reads it: a forward that no longer runs is a stopped VM. */ +export async function inspect(name) { + const session = loadSession(name); + if (!session) return null; + const running = isAlive(session.forwardPid); + return { + state: running ? "running" : "stopped", + image: null, + digest: null, + labels: session.labels ?? {}, + ports: [{ guest: session.workerPort, host: session.hostPort, bind: "127.0.0.1" }], + home: session.home ?? null, + store: session.store ?? null, + created: session.created ?? null, + }; +} + +/** + * One host serves one Store: the Store path is a single directory on a disk the + * user owns, and two projects naming it would write one Store file between + * them. This is the provider's own rule, checked here because the runtime's + * per-project VM names would otherwise hide the collision. + */ +function sharedWith(name, host, storePath) { + for (const file of existsSync(stateDir()) ? readdirSync(stateDir()).filter((entry) => entry.endsWith(".json")) : []) { + if (file === `${name}.json`) continue; + try { + const other = JSON.parse(readFileSync(join(stateDir(), file), "utf8")); + if (other.host === host && other.storePath === storePath) return other.name; + } catch { /* a record that cannot be read is not one of ours to judge */ } + } + return null; +} + +/** + * Step ①–④: the host, its Store path, and the private stream. Everything that + * can refuse is checked before a session is written, so a create that fails + * leaves no session, no forward and nothing uploaded. + */ +export async function create(config) { + const given = config.ssh ?? {}; + const host = String(config.host ?? given.host ?? "").trim(); + if (!host) throw refuse("InvalidInput", "vm:ssh needs the host to reach: settings.vm.host, or LAYERFS_SSH_HOST"); + if (config.password || given.password) throw refuse("InvalidInput", "vm:ssh does not take a password; give a key file as identityFile"); + + const session = { + name: config.name, + host, + user: String(config.user ?? given.user ?? "root").trim(), + port: Number(config.port ?? given.port ?? 22), + identityFile: config.identityFile ?? given.identityFile ?? null, + strictHostKeyChecking: config.strictHostKeyChecking ?? given.strictHostKeyChecking ?? "yes", + knownHostsFile: config.knownHostsFile ?? given.knownHostsFile ?? null, + controlMaster: config.controlMaster ?? given.controlMaster ?? false, + storePath: String(config.storePath ?? given.storePath ?? STORE_PATH_DEFAULT).trim(), + hostPort: config.hostPort, + workerPort: config.workerPort ?? WORKER_PORT_DEFAULT, + home: config.home, + store: config.storeName, + labels: { "layerfs.home": config.home, "layerfs.role": "vm" }, + created: new Date().toISOString(), + }; + if (session.identityFile && !existsSync(session.identityFile)) { + throw refuse("InvalidInput", `the SSH identity file does not exist: ${session.identityFile}`); + } + const shared = sharedWith(session.name, session.host, session.storePath); + if (shared) { + throw refuse("Busy", `${session.host}:${session.storePath} already holds the Store of ${shared}; one host serves one Store, so give this project its own host or disk`, null); + } + + // the host, by fact: a Linux kernel, and the Store path on ext4 + const kernel = await remote(session, ["uname", "-s"]).catch((error) => { throw refuse("Failed", `could not reach ${session.host} over SSH: ${error.message}`); }); + if (kernel.code !== 0) throw refuse("Failed", `SSH to ${session.host} failed (exit ${kernel.code}): ${kernel.stderr.trim()}`); + if (kernel.stdout.trim() !== "Linux") throw refuse("Unsupported", `${session.host} answers ${kernel.stdout.trim() || "nothing"}, not Linux; vm:ssh needs a Linux host`); + + const exists = await remote(session, ["sh", "-c", "test -e \"$1\"", "sh", session.storePath]); + if (exists.code !== 0) throw refuse("InvalidInput", `the Store path ${session.storePath} does not exist on ${session.host}; vm:ssh uses a disk the user provides`); + const block = (await remote(session, ["sh", "-c", "test -b \"$1\"", "sh", session.storePath])).code === 0; + const type = block + ? (await remote(session, ["sh", "-c", "blkid -s TYPE -o value \"$1\" 2>/dev/null || lsblk -no FSTYPE \"$1\"", "sh", session.storePath])).stdout.trim() + : (await remote(session, ["findmnt", "-T", session.storePath, "-o", "FSTYPE", "-n"])).stdout.trim(); + if (type !== "ext4") { + throw refuse("Unsupported", `the Store path ${session.storePath} on ${session.host} is ${type || "an unreadable filesystem"}, not ext4; LayerFS requires an ext4 Store disk`); + } + + // the guest's own view: /layerfs/store is that path (mounted, or bind-mounted) + const attached = await remote(session, ["sh", "-c", + "if [ \"$1\" = /layerfs/store ]; then exit 0; fi; mkdir -p /layerfs/store && mountpoint -q /layerfs/store || { if [ -b \"$1\" ]; then mount -t ext4 \"$1\" /layerfs/store; else mount --bind \"$1\" /layerfs/store; fi; }", + "sh", session.storePath]); + if (attached.code !== 0) throw refuse("Failed", `could not attach ${session.storePath} at /layerfs/store on ${session.host}: ${attached.stderr.trim()}`); + + try { + const forward = await startForward(session, session.hostPort, session.workerPort); + session.forwardPid = forward.pid; + } catch (error) { + throw refuse("Failed", `could not open the private stream to ${session.host}: ${error.message}`); + } + saveSession(session); + return inspect(session.name); +} + +/** A stopped VM is started again by re-opening its stream on the port it kept. */ +export async function start(name) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + if (isAlive(session.forwardPid)) return inspect(name); + const forward = await startForward(session, session.hostPort, session.workerPort); + session.forwardPid = forward.pid; + saveSession(session); + return inspect(name); +} + +/** Step ⑤, stop: the stream and the worker end; the Store disk and the session record stay. */ +export async function stop(name) { + const session = loadSession(name); + if (!session) return; + await endForward(session); + await remote(session, ["sh", "-c", "if [ -f /layerfs/state/worker.pid ]; then kill \"$(cat /layerfs/state/worker.pid)\" 2>/dev/null; fi"]).catch(() => {}); + saveSession(session); +} + +export const restart = async (name) => { + await stop(name); + return start(name); +}; + +/** + * Step ⑤, delete: the record, the forward and what this adapter put on the host. + * The Store disk is the user's and is never unmounted, formatted or removed; a + * delete while the project still holds its Store is refused by the runtime. + */ +export async function remove(name) { + const session = loadSession(name); + if (!session) return; + await endForward(session); + await remote(session, ["sh", "-c", + "if [ -f /layerfs/state/worker.pid ]; then kill \"$(cat /layerfs/state/worker.pid)\" 2>/dev/null; fi; rm -rf /layerfs/state /layerfs/bundle"]).catch(() => {}); + deleteSession(name); +} + +/** The Store disk is the user's: this adapter has nothing of its own to delete. */ +export const removeStore = async () => {}; + +/** V4: the host-wide user-namespace block; the caller reads it back. */ +export const blockUserNamespaces = (name) => + exec(name, ["sh", "-c", "echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the Store disk, as the host sees them. */ +export async function storeFree(name, guestStorePath = STORE_PATH_DEFAULT) { + const { code, stdout } = await exec(name, ["df", "-kP", guestStorePath]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +export function exec(name, argv, options = {}) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + return remote(session, argv, options); +} + +/** + * Step ②: the bundle (or any folder) as one `tar` stream, unpacked on the host. + * Both ends are killed if the transfer passes its budget, so no copy outlives + * its call. + */ +export function copy(name, source, destination) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + const script = `mkdir -p ${shellEscape(destination)} && tar -C ${shellEscape(destination)} -xf -`; + return new Promise((resolve, reject) => { + const pack = spawn("tar", ["-C", source, "-cf", "-", "."], { stdio: ["ignore", "pipe", "pipe"] }); + const unpack = spawn("ssh", [...sshArgs(session), destinationOf(session), "--", script], { stdio: ["pipe", "pipe", "pipe"] }); + let err = "", tarErr = ""; + const timer = setTimeout(() => { + pack.kill("SIGKILL"); + unpack.kill("SIGKILL"); + reject(new Error(`copy into ${name}:${destination} did not end within ${COPY_MS / 1000} s`)); + }, COPY_MS); + unpack.stderr.on("data", (chunk) => { err += chunk.toString(); }); + pack.stderr.on("data", (chunk) => { tarErr += chunk.toString(); }); + pack.on("error", (error) => { clearTimeout(timer); reject(error); }); + unpack.on("error", (error) => { clearTimeout(timer); reject(error); }); + unpack.on("close", (code) => { + clearTimeout(timer); + if (code === 0) resolve(); + else reject(new Error(`copy into ${name}:${destination} failed (exit ${code}): ${(err || tarErr).trim()}`)); + }); + pack.stdout.pipe(unpack.stdin); + }); +} + +export const read = (name, path) => readFile("ssh", name, path, exec); +export const write = (name, path, text) => writeFile("ssh", name, path, text, exec); + +export default { + name: "ssh", + defaultImage: null, + inspect, create, start, stop, restart, remove, removeStore, + blockUserNamespaces, storeFree, read, write, copy, exec, isAlive, +}; diff --git a/api/runtime/src/platform/windows/index.js b/api/runtime/src/platform/windows/index.js index 6351f9ed2c..8bd4b693ce 100644 --- a/api/runtime/src/platform/windows/index.js +++ b/api/runtime/src/platform/windows/index.js @@ -11,7 +11,7 @@ import { createWriteStream, mkdirSync, readFileSync, rmSync } from "node:fs"; import { join, posix } from "node:path"; import { fileURLToPath } from "node:url"; import { connection, frame, readFrames } from "@layerfs/agent/internal"; -import { endpointFile, ping, ready, refuse, run, writeEndpoint } from "../shared.js"; +import { endpointFile, ping, ready, refuse, run, writeEndpoint, makeStore, releaseStoreHolders, removeStore, storeHolders } from "../shared.js"; import { absoluteLinux, projectHome, wsl, wslArgs } from "./paths.js"; const SERVER = "api/runtime/src/worker/server.js"; @@ -131,6 +131,10 @@ export default { async init(settings) { await start(settings); }, path: (settings, hostPath) => hostPath, docker, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `/var/lib/docker/volumes/${volume}/_data/global/store.sqlite`, async homeNames(settings) { const { stdout } = await wsl(settings, ["find", settings.wsl.home, "-mindepth", "2", "-maxdepth", "2", "-name", "project.record"], { timeout: 5000 }).catch(() => ({ stdout: "" })); @@ -139,7 +143,8 @@ export default { // rm gets one checked child directory as an argv item, never shell text async removeHome(settings, name) { await wsl(settings, ["rm", "-rf", "--", projectHome(settings.wsl, name)], { timeout: 10_000 }); }, /** Before a volume is deleted, the worker closes its cached Store of it. */ - async releaseStore(settings, volume) { + async releaseStore(settings, project) { + const { volume } = project; if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(volume)) throw refuse("InvalidInput", "invalid Store volume name"); const info = JSON.parse(readFileSync(endpointFile(settings.home), "utf8")); const answer = await connection(info).call({ fn: "releaseStore", args: { home: settings.wsl.home, storePath: `/var/lib/docker/volumes/${volume}/_data/global/store.sqlite` } }); diff --git a/api/runtime/src/project/init.js b/api/runtime/src/project/init.js index f11645958c..f9899c0769 100644 --- a/api/runtime/src/project/init.js +++ b/api/runtime/src/project/init.js @@ -80,9 +80,10 @@ export async function runInit(runtime, prepared, report = () => {}) { report({ step: "scan" }); const totals = await scanSource(project.source, (counts) => report({ scanned: counts })); report({ step: "volume", totals }); - await platform.init(settings); - // its owner, for runtime.volume.list: Docker sets a volume's labels only when it creates it - await platform.docker(settings, ["volume", "create", "--label", "layerfs.role=store", "--label", `layerfs.home=${settings.home}`, "--label", `layerfs.project=${project.name}`, project.volume], { timeout: 30_000 }); + await platform.init(settings, project); + // its owner, for runtime.volume.list on a local placement (Docker sets a + // volume's labels only when it creates it); a VM's Store is a directory + await platform.makeStore(runtime, project); report({ step: "import" }); let client; try { @@ -96,7 +97,7 @@ export async function runInit(runtime, prepared, report = () => {}) { throw error; } if (error?.code !== "Exists") await platform.removeHome(settings, project.name).catch(() => {}); - await platform.docker(settings, ["volume", "rm", project.volume]).catch(() => {}); + await platform.removeStore(runtime, project).catch(() => {}); throw error; } await client.close(); diff --git a/api/runtime/src/project/remove.js b/api/runtime/src/project/remove.js index 036c099819..914880234d 100644 --- a/api/runtime/src/project/remove.js +++ b/api/runtime/src/project/remove.js @@ -5,12 +5,13 @@ // containers on the volume, left by failed starts and idle Sandboxes, go // with it. The volume is the runtime's record's, else `volume`. The home // entry is the project, so deleting it frees the name at once. One attempt -// each, no retry; a failed step leaves the rest in place. +// each, no retry; a failed step leaves the rest in place. The four Store hooks +// are the platform's: Docker volumes and containers on a local placement, the +// VM's own directory and namespace Sandboxes under `vm:`. import { LayerFSError } from "@layerfs/agent"; +import { STOPPED } from "../platform/shared.js"; import { NAME, dropRecord, record } from "./list.js"; -/** Container states that hold the volume but run nothing. */ -const STOPPED = new Set(["created", "exited", "dead"]); const refuse = (code, message) => new LayerFSError(code, message, null); export async function remove(runtime, name, options = {}) { @@ -21,13 +22,15 @@ export async function remove(runtime, name, options = {}) { if (typeof volume !== "string" || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(volume)) throw refuse("InvalidInput", `the runtime has no record of ${name}'s volume; name it as { volume }`); if (!record(settings, name) && !(await platform.homeNames(settings)).includes(name)) throw refuse("NoSuchProject", `no LayerFS project named ${name}`); await runtime.clients.forget(name); - const rows = await platform.docker(settings, ["ps", "-a", "--filter", `volume=${volume}`, "--format", "{{.ID}} {{.State}}"]); - const containers = rows.split("\n").filter(Boolean).map((line) => line.split(" ")); - const users = containers.filter(([, state]) => !STOPPED.has(state)).length; + // the hooks name the project as well as its volume: a `vm:` placement's Store is + // the project's own VM and disk, a local placement's is a Docker volume + const project = { name, volume }; + const holders = await platform.storeHolders(runtime, project); + const users = holders.filter((holder) => !STOPPED.has(holder.state)).length; if (users && !options.stopSandboxes) throw refuse("Busy", `${users} running container(s) use ${volume}; a Sandbox closes 2 minutes after its last Workspace is unmounted, or pass stopSandboxes`); - if (containers.length) await platform.docker(settings, ["rm", ...(users ? ["-f"] : []), ...containers.map(([id]) => id)], { timeout: 30_000 }); - await platform.releaseStore(settings, volume); - await platform.docker(settings, ["volume", "rm", volume], { timeout: 30_000 }); + await platform.releaseStoreHolders(runtime, project, holders, { force: users > 0 }); + await platform.releaseStore(settings, project); + await platform.removeStore(runtime, project); await platform.removeHome(settings, name); dropRecord(settings, name); return { kind: "Removed", project: name, volume }; diff --git a/api/runtime/src/sandbox/leftovers.js b/api/runtime/src/sandbox/leftovers.js index b4ca7eec7c..caf5528565 100644 --- a/api/runtime/src/sandbox/leftovers.js +++ b/api/runtime/src/sandbox/leftovers.js @@ -1,10 +1,12 @@ // Sandbox containers of this home that no project owns, or that were left // stopped, and their removal. From part of the dsh Host's runtime.js (#428). import { LayerFSError } from "@layerfs/agent"; +import { noHostDocker } from "../platform/shared.js"; /** This home's Sandbox containers (`-`, label layerfs.role=sandbox), running or not. */ export async function containers(runtime) { const { settings, platform } = runtime; + if (platform.hostSandboxes === false) throw noHostDocker(runtime, "runtime.sandbox", "its Sandboxes are Linux namespaces inside the VM; the SDK's project.list_sandboxes reads them"); const out = await platform.docker(settings, ["ps", "-a", "--no-trunc", "--filter", "label=layerfs.role=sandbox", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Label \"layerfs.project\"}}\t{{.Label \"layerfs.sandbox\"}}\t{{.Image}}"]); return out.split("\n").filter(Boolean).map((row) => { const [name, state, status, project, sandbox, image] = row.split("\t"); diff --git a/api/runtime/src/sandbox/list.js b/api/runtime/src/sandbox/list.js index f1b5c710c8..5316b78f65 100644 --- a/api/runtime/src/sandbox/list.js +++ b/api/runtime/src/sandbox/list.js @@ -5,6 +5,7 @@ // there). It goes only when the container is deleted (sandbox.close, an idle // close followed by a reclaim, or runtime.sandbox.sweep). One `docker ps` // and one `docker inspect --size`; it changes nothing. +import { noHostDocker } from "../platform/shared.js"; /** A container gone between the listing and the inspection is left out: inspect still prints the others. */ async function inspect(runtime, ids) { @@ -21,6 +22,7 @@ async function inspect(runtime, ids) { /** `[{ name, project, sandbox, state, running, startedAt, finishedAt, image, ports, scratch, owned }]`, by name. */ export async function list(runtime) { const { settings, platform } = runtime; + if (platform.hostSandboxes === false) throw noHostDocker(runtime, "runtime.sandbox.list", "its Sandboxes are Linux namespaces inside the VM; the SDK's project.list_sandboxes reads them"); const ids = (await platform.docker(settings, ["ps", "-a", "-q", "--no-trunc", "--filter", "label=layerfs.role=sandbox"])).split("\n").filter(Boolean); if (!ids.length) return []; const known = new Set((await runtime.project.list()).map((p) => p.name)); diff --git a/api/runtime/src/sandbox/sweep.js b/api/runtime/src/sandbox/sweep.js index bf1e0d4161..32b3a469a5 100644 --- a/api/runtime/src/sandbox/sweep.js +++ b/api/runtime/src/sandbox/sweep.js @@ -7,6 +7,8 @@ // project of this home records stay for list_leftovers / clean_leftovers. // The runtime runs it once when it opens (settings.sweep, default true); // status().sweep reports that run. One attempt per project, no retry. +// A `vm:` home keeps no host containers, so its sweep reads none +// and says so: `{ reclaimed: {}, failed: [], host: false }`. import { containers } from "./leftovers.js"; /** States of a container whose daemon has ended and that nothing restarts. */ @@ -15,6 +17,7 @@ const ENDED = new Set(["exited", "dead"]); /** `{ reclaimed: { project: [ids] }, failed: [{ project, code, message }] }` */ export async function sweep(runtime) { const { settings } = runtime; + if (runtime.platform.hostSandboxes === false) return { reclaimed: {}, failed: [], host: false }; const known = new Set((await runtime.project.list()).filter((p) => p.volume).map((p) => p.name)); const ended = (await containers(runtime)).filter((c) => ENDED.has(c.state) && known.has(c.project) && c.name.startsWith(`${settings.containerPrefix}${c.project}-`)); const reclaimed = {}, failed = []; diff --git a/api/runtime/src/settings.js b/api/runtime/src/settings.js index 070816dac8..b1ce588cb9 100644 --- a/api/runtime/src/settings.js +++ b/api/runtime/src/settings.js @@ -9,6 +9,59 @@ import { LayerFSError } from "@layerfs/agent"; const PLATFORMS = { linux: "linux", darwin: "macos", win32: "windows" }; const DEFAULT_DOCKER = "/var/run/docker.sock"; +/** Where a home's Store and Sandboxes live: this machine (`local`, today) or one Linux VM per home (V9). */ +const PROVIDERS = ["microsandbox", "apple-container", "ssh"]; +/** The VM's own settings: the image that holds node and the guest base, and its size. */ +const VM = { image: "node:22-trixie-slim", cpus: 2, memory: "2G", storeSize: "20G" }; +/** Where the VM sees the home, the Store disk, the bundle and its working state (§5, §6). */ +export const GUEST = Object.freeze({ + home: "/layerfs/home", + store: "/layerfs/store", + bundle: "/layerfs/bundle", + import: "/layerfs/import", + state: "/layerfs/state", + // The Sandbox directory is on the Store disk, beside the Store's own + // `global/` folder, not on the VM's root: a namespace Sandbox's root is an + // overlay whose writable layer must be on a local filesystem, and a + // provider's own root may be an overlay itself (Microsandbox's is), while + // the Store disk is an ext4 disk on both providers (H38's + // `require_overlay_upper`). Its `parent` is what the backend reads, so the + // Store's own directory `/layerfs/store/global` stays outside it. + local: "/layerfs/store/local", + daemon: "/layerfs/bundle/layerfs-daemon", +}); + +/** + * `placement`: absent or `local` (this machine's Docker, macOS worker container + * or WSL2 worker; behavior unchanged), or `vm:` (one Linux VM per + * home, §6). A VM provider name that is not implemented is refused here, not + * later, so a typo cannot look like `local`. + */ +function placement(given) { + const text = given ?? process.env.LAYERFS_PLACEMENT ?? "local"; + if (typeof text !== "string" || !text) throw new LayerFSError("InvalidInput", "placement must be `local` or `vm:`", null); + if (text === "local") return { kind: "local", name: "local" }; + const provider = /^vm:(.+)$/.exec(text)?.[1]; + if (!provider) throw new LayerFSError("InvalidInput", `placement is \`local\` or \`vm:\`, not ${text}`, null); + if (!PROVIDERS.includes(provider)) throw new LayerFSError("Unsupported", `no VM provider is implemented for ${text}; it is vm:${PROVIDERS.join(" or vm:")}`, null); + return { kind: "vm", name: text, provider }; +} + +/** The VM's size and image, from `settings.vm` else `LAYERFS_VM_*` else the defaults. */ +function vmSettings(given = {}) { + const NAMES = { image: "LAYERFS_VM_IMAGE", cpus: "LAYERFS_VM_CPUS", memory: "LAYERFS_VM_MEMORY", storeSize: "LAYERFS_VM_STORE_SIZE" }; + const read = (name) => given[name] ?? (process.env[NAMES[name]] || undefined); + const image = read("image") ?? VM.image; + if (typeof image !== "string" || !/^[^\s]+$/.test(image)) throw new LayerFSError("InvalidInput", "the VM image is an image reference, such as node:22-trixie-slim", null); + const cpus = Number(read("cpus") ?? VM.cpus); + if (!Number.isSafeInteger(cpus) || cpus < 1) throw new LayerFSError("InvalidInput", "the VM's cpus is a whole number of at least 1", null); + const size = (name, fallback) => { + const value = read(name) ?? fallback; + if (typeof value !== "string" || !/^[1-9][0-9]{0,5}[MG]$/.test(value)) throw new LayerFSError("InvalidInput", `the VM's ${name} is a size such as 2G or 512M`, null); + return value; + }; + return { image, cpus, memory: size("memory", VM.memory), storeSize: size("storeSize", VM.storeSize) }; +} /** * The Docker socket when none is given: the engine's usual path, or on macOS @@ -35,6 +88,10 @@ function defaultDocker(platform) { * @param {string} [given.containerPrefix] this home's container names (default `layerfs-`) * @param {string} [given.stateDir] the runtime's project records (default /.runtime) * @param {object} [given.wsl] Windows: { distribution, user, node, home } + * @param {string} [given.placement] `local` (default) or `vm:`; also read from + * LAYERFS_PLACEMENT when not given (V9). A VM holds the home's Stores and Sandboxes. + * @param {object} [given.vm] VM placement: { image, cpus, memory, storeSize } (also + * LAYERFS_VM_IMAGE, LAYERFS_VM_CPUS, LAYERFS_VM_MEMORY, LAYERFS_VM_STORE_SIZE) * @param {string} [given.platform] linux | macos | windows (default this process's) * @param {boolean} [given.sweep] sweep this home's stopped Sandboxes once at open (default true) */ @@ -44,12 +101,15 @@ export function resolveSettings(given = {}) { const platform = given.platform ?? PLATFORMS[process.platform]; if (!Object.values(PLATFORMS).includes(platform)) throw new LayerFSError("Unsupported", `LayerFS runs on Linux, macOS and Windows, not ${process.platform}`, null); const bundle = given.bundle || null; + const where = placement(given.placement); const settings = { platform, + placement: where, home: given.home, image: given.image || null, bundle, - daemon: given.daemon || (bundle ? (platform === "windows" ? posix.join(bundle, "layerfs-daemon") : join(bundle, "layerfs-daemon")) : null), + // a VM runs the bundle's daemon beside its helper, at the guest's path + daemon: where.kind === "vm" ? GUEST.daemon : given.daemon || (bundle ? (platform === "windows" ? posix.join(bundle, "layerfs-daemon") : join(bundle, "layerfs-daemon")) : null), docker: given.docker || defaultDocker(platform), volumes: given.volumes || "/var/lib/docker/volumes", traceDir: given.traceDir || join(given.home, "traces"), @@ -58,6 +118,34 @@ export function resolveSettings(given = {}) { pollMs: given.pollMs ?? 400, sweep: given.sweep !== false, }; + if (where.kind === "vm") { + settings.vm = { ...vmSettings(given.vm), ...where, home: GUEST.home, store: GUEST.store }; + if (where.provider === "ssh") { + const sshGiven = given.ssh ?? given.vm ?? {}; + if (sshGiven.password) throw new LayerFSError("InvalidInput", "vm:ssh does not support password authentication; use an identityFile instead", null); + const host = sshGiven.host ?? process.env.LAYERFS_SSH_HOST ?? null; + const port = Number(sshGiven.port ?? process.env.LAYERFS_SSH_PORT ?? 22); + if (!Number.isSafeInteger(port) || port < 1 || port > 65535) throw new LayerFSError("InvalidInput", "the SSH port must be a number between 1 and 65535", null); + const identityFile = sshGiven.identityFile ?? sshGiven.identity ?? process.env.LAYERFS_SSH_IDENTITY ?? process.env.LAYERFS_SSH_KEY ?? null; + if (identityFile && typeof identityFile !== "string") throw new LayerFSError("InvalidInput", "the SSH identityFile must be a file path", null); + const storePath = sshGiven.storePath ?? sshGiven.store ?? process.env.LAYERFS_SSH_STORE ?? GUEST.store; + settings.vm.ssh = { + host: host ? String(host).trim() : null, + user: String(sshGiven.user ?? process.env.LAYERFS_SSH_USER ?? "root").trim(), + port, + identityFile: identityFile ? String(identityFile).trim() : null, + storePath: String(storePath).trim(), + strictHostKeyChecking: sshGiven.strictHostKeyChecking ?? true, + knownHostsFile: sshGiven.knownHostsFile ?? process.env.LAYERFS_SSH_KNOWN_HOSTS ?? null, + controlMaster: sshGiven.controlMaster ?? false, + }; + settings.vm.host = settings.vm.ssh.host; + settings.vm.user = settings.vm.ssh.user; + settings.vm.port = settings.vm.ssh.port; + settings.vm.identityFile = settings.vm.ssh.identityFile; + settings.vm.storePath = settings.vm.ssh.storePath; + } + } if (platform === "windows") { const hash = createHash("sha256").update(given.home).digest("hex").slice(0, 16); settings.wsl = { @@ -74,10 +162,15 @@ export function resolveSettings(given = {}) { return settings; } -/** Settings Init and Workspaces need that are not given. macOS and Windows need a bundle too. */ -export const missing = (settings) => [ - ...["image", "daemon", ...(settings.platform === "linux" ? [] : ["bundle"])].filter((name) => !settings[name]), -]; +/** Settings Init and Workspaces need that are not given. macOS, Windows and a VM need a bundle too. */ +export const missing = (settings) => + [ + ...(settings.placement.kind === "vm" ? [] : ["image"]), + "daemon", + ...(settings.platform === "linux" && settings.placement.kind !== "vm" ? [] : ["bundle"]), + // a host this home cannot reach is a missing setting, listed before a start + ...(settings.placement.provider === "ssh" ? ["host"] : []), + ].filter((name) => (name === "host" ? !settings.vm?.host : !settings[name])); -/** The LayerFS home as LayerFS calls see it (Windows: the WSL one). */ -export const callHome = (settings) => settings.wsl?.home ?? settings.home; +/** The LayerFS home as LayerFS calls see it (Windows: the WSL one; a VM: the guest's). */ +export const callHome = (settings) => settings.vm?.home ?? settings.wsl?.home ?? settings.home; diff --git a/api/runtime/src/status.js b/api/runtime/src/status.js index 45ba3b79d4..ed4ace61e3 100644 --- a/api/runtime/src/status.js +++ b/api/runtime/src/status.js @@ -1,23 +1,34 @@ // runtime.status(): Docker reachability, the worker's state (null on Linux), // free disk and missing settings. From part of the dsh Host's runtime.js -// (#428). It starts nothing. +// (#428). It starts nothing. A `vm:` home has no host Docker in its +// data path (#515 §3): its Stores and Sandboxes are inside the VM, so `docker` +// says so instead of reporting the host engine down, and `vm` carries the VM's +// own state. import { statfsSync } from "node:fs"; import { missing } from "./settings.js"; export async function status(runtime) { const { settings, platform } = runtime; let docker; - try { docker = { level: "ok", version: await platform.docker(settings, ["version", "--format", "{{.Server.Version}}"]) }; } catch (error) { - docker = { level: "down", error: error.stderr?.trim() || error.message }; + if (platform.hostDocker === false) { + docker = { level: "not-used", detail: `this home is placed in ${platform.name}; its Stores and Sandboxes are inside each project's VM` }; + } else { + try { docker = { level: "ok", version: await platform.docker(settings, ["version", "--format", "{{.Server.Version}}"]) }; } catch (error) { + docker = { level: "down", error: error.stderr?.trim() || error.message }; + } } - const { worker } = await platform.status(settings).catch((error) => ({ worker: { state: "unknown", error: error.message } })); + // a `vm:` placement has one VM per project, so it is asked about each project + const projects = await runtime.project.list().then((all) => all.map((project) => project.name), () => []); + const reported = await platform.status(settings, projects).catch((error) => ({ worker: { state: "unknown", error: error.message } })); let home = null; try { const s = statfsSync(settings.home); home = s.bavail * s.bsize; } catch { /* unreadable */ } return { - platform: platform.name, + platform: settings.platform, + placement: settings.placement.name, missing: missing(settings), docker, - worker, - disk: { home, volumes: docker.level === "ok" ? await platform.free(settings) : null }, + worker: reported.worker ?? null, + vm: reported.vm ?? null, + disk: { home, volumes: docker.level === "not-used" || docker.level === "ok" ? await platform.free(settings) : null }, }; } diff --git a/api/runtime/src/volume.js b/api/runtime/src/volume.js index dc45d9b142..370922a79c 100644 --- a/api/runtime/src/volume.js +++ b/api/runtime/src/volume.js @@ -14,6 +14,7 @@ // One `docker system df -v`, plus one `docker volume inspect` of the // labelled volumes; listing changes nothing. import { LayerFSError } from "@layerfs/agent"; +import { noHostDocker } from "./platform/shared.js"; const UNITS = { B: 1, kB: 1e3, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12 }; /** "84.05MB" -> 84050000; anything else -> null. */ @@ -25,6 +26,7 @@ export const bytesOf = (text) => { /** `[{ name, size, containers, owner, project, live, removable }]`, by name. */ export async function list(runtime) { const { settings, platform } = runtime; + if (platform.hostDocker === false) throw noHostDocker(runtime, "runtime.volume.list", "its Stores are directories on the VM's disk, not host Docker volumes"); const df = JSON.parse(await platform.docker(settings, ["system", "df", "-v", "--format", "{{json .}}"], { timeout: 60_000 })); const volumes = df.Volumes ?? []; const labelled = volumes.filter((v) => v.Labels).map((v) => v.Name); diff --git a/api/runtime/test/fixture/fake-container.cjs b/api/runtime/test/fixture/fake-container.cjs new file mode 100644 index 0000000000..d61636842e --- /dev/null +++ b/api/runtime/test/fixture/fake-container.cjs @@ -0,0 +1,176 @@ +// The `container` stand-in for @layerfs/runtime's Apple Container test: no VM. +// It answers the calls `platform/vm/apple-container.js` makes **in the shape the +// real CLI prints**, copied from `container` 1.6.1 (build aa8d76b) on macOS 27.2, +// 2026-10-31: `inspect --format json` is an array of +// `{ id, status: { state, startedDate, networks }, configuration: { image, +// labels, mounts, publishedPorts, creationDate } }`, mounts carry +// `type.volume.name`, and it needs `-i` on `exec` for a caller's stdin. What the +// stand-in cannot prove is the real CLI's behaviour under load; the real run is +// `api/runtime/test/verify/vm_real.mjs apple-container`. +// +// container inspect --format json NAME +// container run --detach --name N … IMAGE sleep infinity +// container start|stop NAME, container delete --force NAME +// container volume inspect NAME | create -s SIZE NAME | delete NAME +// container copy HOST NAME:DEST +// container exec [-i] NAME sh -c SCRIPT sh ARGS… +// Records live in FAKE_CONTAINER_STATE/vms/.json, volumes in +// FAKE_CONTAINER_STATE/volumes/.json, guest paths under +// FAKE_CONTAINER_STATE/guest/. +const { appendFileSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_CONTAINER_STATE; +if (!state) { + console.error("fake container: FAKE_CONTAINER_STATE is not set"); + process.exit(2); +} +const recordFile = (name) => join(state, "vms", `${name}.json`); +const volumeFile = (name) => join(state, "volumes", `${name}.json`); +const guestIn = (name) => (path) => join(state, "guest", name, path.replace(/^\//, "")); +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const fail = (message) => { console.error(`error: ${message}`); process.exit(1); }; +const at = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); +const all = (flag) => argv.flatMap((value, index) => (value === flag ? [argv[index + 1]] : [])); +const read = (name) => (existsSync(recordFile(name)) ? JSON.parse(readFileSync(recordFile(name), "utf8")) : null); +const save = (value) => { mkdirSync(dirname(recordFile(value.name)), { recursive: true }); writeFileSync(recordFile(value.name), JSON.stringify(value, null, 1)); }; + +function endWorker(vm) { + if (!vm?.pid) return; + try { process.kill(vm.pid); } catch { /* already gone */ } + vm.pid = null; + save(vm); +} + +function launchWorker(vm) { + endWorker(vm); // a start ends the worker an earlier start left, as the launch script does + const guest = guestIn(vm.name); + const env = Object.fromEntries(readFileSync(guest("/layerfs/state/worker.env"), "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const child = spawn(process.execPath, [guest("/layerfs/bundle/api/runtime/src/worker/server.js")], { + env: { ...process.env, ...env, LAYERFS_HOME: guest(env.LAYERFS_HOME), LAYERFS_BUNDLE: guest(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${vm.port}`, LAYERFS_NATIVE: process.env.FAKE_CONTAINER_NATIVE }, + detached: true, stdio: ["ignore", "ignore", openSync(guest("/layerfs/state/worker.log"), "a")], + }); + child.unref(); + vm.pid = child.pid; + save(vm); + const pid = guest("/layerfs/state/worker.pid"); + mkdirSync(dirname(pid), { recursive: true }); + writeFileSync(pid, `${child.pid}\n`); +} + +const command = argv[0]; +if (command === "inspect") { + const name = argv.at(-1); + const vm = read(name) ?? fail(`container not found: ${name}`); + process.stdout.write(JSON.stringify([{ + id: vm.name, + status: { + state: vm.state, + startedDate: vm.started ?? vm.created, + networks: [{ network: "default", ipv4Address: "192.168.64.2/24", ipv4Gateway: "192.168.64.1", variant: "reserved" }], + }, + configuration: { + creationDate: vm.created, + image: { reference: vm.image, descriptor: { digest: `sha256:${"1".repeat(64)}` } }, + labels: vm.labels, + mounts: [ + { source: vm.home, destination: "/layerfs/home", type: { virtiofs: {} } }, + { source: join(state, "volumes", `${vm.store}.img`), destination: "/layerfs/store", type: { volume: { name: vm.store, format: "ext4" } } }, + ], + publishedPorts: [{ hostPort: vm.port, containerPort: 7070, hostAddress: "127.0.0.1", count: 1, proto: "tcp" }], + }, + }])); +} else if (command === "run") { + const name = at("--name"); + if (read(name)) fail(`container ${name} already exists`); + const mounts = all("--mount").map((value) => Object.fromEntries(value.split(",").map((pair) => pair.split("=")))); + const store = mounts.find((entry) => entry.target === "/layerfs/store")?.source ?? ""; + const home = mounts.find((entry) => entry.target === "/layerfs/home")?.source ?? ""; + // H39v's finding: `container run` makes a named volume it was given before it + // fails on a busy port, and the container record never appears + if (process.env.FAKE_CONTAINER_FAIL_RUN === "port") { + if (!existsSync(volumeFile(store))) { + mkdirSync(join(state, "volumes"), { recursive: true }); + writeFileSync(volumeFile(store), JSON.stringify({ name: store, size: "20G", format: "ext4", creationDate: new Date().toISOString(), labels: {} })); + } + fail(`failed to publish 127.0.0.1:${String(at("--publish") ?? "").split(":")[1]}:7070: port is already allocated`); + } + if (!existsSync(volumeFile(store))) fail(`volume not found: ${store}`); + const labels = Object.fromEntries(all("--label").map((pair) => pair.split("="))); + save({ + name, state: "running", created: new Date().toISOString(), started: new Date().toISOString(), pid: null, + labels, home, store, port: Number(String(at("--publish") ?? "").split(":")[1]), + image: argv[argv.length - 3], + }); +} else if (command === "start") { + const vm = read(argv[1]) ?? fail(`container not found: ${argv[1]}`); + vm.state = "running"; + vm.started = new Date().toISOString(); + save(vm); +} else if (command === "stop") { + const vm = read(argv[1]) ?? fail(`container not found: ${argv[1]}`); + endWorker(vm); + vm.state = "stopped"; + save(vm); +} else if (command === "delete") { + const vm = read(argv.at(-1)) ?? fail(`container not found: ${argv.at(-1)}`); + endWorker(vm); + rmSync(recordFile(vm.name), { force: true }); + rmSync(join(state, "guest", vm.name), { recursive: true, force: true }); +} else if (command === "volume") { + const [action, ...rest] = argv.slice(1); + const name = action === "create" ? rest.at(-1) : rest.at(-1); + if (action === "inspect") { + if (!existsSync(volumeFile(name))) fail(`volume not found: ${name}`); + process.stdout.write(JSON.stringify([{ id: name, configuration: JSON.parse(readFileSync(volumeFile(name), "utf8")) }])); + } else if (action === "create") { + if (existsSync(volumeFile(name))) fail(`volume ${name} already exists`); + mkdirSync(join(state, "volumes"), { recursive: true }); + const size = at("-s") ?? "549755813888"; + writeFileSync(volumeFile(name), JSON.stringify({ name, sizeInBytes: Number(String(size).replace(/[KMGTP]$/, "")), size, format: "ext4", creationDate: new Date().toISOString(), labels: {} })); + } else if (action === "delete") { + rmSync(volumeFile(name), { force: true }); + } +} else if (command === "copy") { + const target = argv[2]; + const guest = guestIn(target.slice(0, target.indexOf(":"))); + const destination = guest(target.slice(target.indexOf(":") + 1)); + mkdirSync(destination, { recursive: true }); + for (const entry of readdirSync(argv[1], { withFileTypes: true })) cpSync(join(argv[1], entry.name), join(destination, entry.name), { recursive: entry.isDirectory() }); +} else if (command === "exec") { + // `exec [-i] NAME argv…` + const interactive = argv.includes("-i"); + const rest = argv.slice(1).filter((value) => value !== "-i"); + const [name, ...command_] = rest; + const vm = read(name) ?? fail(`container not found: ${name}`); + const guest = guestIn(vm.name); + const put = (path, text) => { mkdirSync(dirname(guest(path)), { recursive: true }); writeFileSync(guest(path), text); }; + const [tool, flag, script, ...args] = command_; + if (tool === "rm") { + rmSync(guest(script), { recursive: true, force: true }); + } else if (tool === "df") { + const volume = JSON.parse(readFileSync(volumeFile(vm.store), "utf8")); + const blocks = Math.floor(volume.sizeInBytes / 1024); + process.stdout.write(`Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/vdc ${blocks} 1048576 ${blocks - 1048576} 1% /layerfs/store\n`); + } else if (tool === "sh" && flag === "-c") { + if (/test -f "\$1" && cat "\$1"/.test(script)) { + if (!existsSync(guest(args[1]))) process.exit(1); + process.stdout.write(readFileSync(guest(args[1]))); + } else if (/cat > "\$1"/.test(script)) { + if (!interactive) fail("stdin was not kept open: the adapter must pass -i"); + put(args[1], readFileSync(0)); + } else if (/umask 022/.test(script)) { + mkdirSync(guest(args[1]), { recursive: true, mode: 0o755 }); + } else if (/max_user_namespaces/.test(script)) { + if (!/remount,rw \/proc\/sys/.test(script)) fail("the guest mounts /proc/sys read-only: remount it first"); + put("/proc/sys/user/max_user_namespaces", "0\n"); + } else if (/nohup node/.test(script)) { + launchWorker(vm); + } + } +} else { + fail(`fake container does not know ${command}`); +} diff --git a/api/runtime/test/fixture/fake-msb.cjs b/api/runtime/test/fixture/fake-msb.cjs new file mode 100644 index 0000000000..aecf4a3d7b --- /dev/null +++ b/api/runtime/test/fixture/fake-msb.cjs @@ -0,0 +1,151 @@ +// The `msb` stand-in for @layerfs/runtime's VM tests (#515 §6): no microVM. +// It answers the calls `platform/vm/microsandbox.js` makes, keeps one record +// per VM (one project's Store = one VM), and maps each guest's absolute paths +// under FAKE_MSB_STATE/guest/, so the adapter's guest file reads and writes +// are real file operations and two VMs never share one. +// +// msb inspect --format json NAME the record, else "sandbox not found" +// msb create --name N … IMAGE record it as Running with the -p host port +// msb start|stop|restart NAME flip the record's state +// msb remove --force NAME forget it (and end its worker) +// msb volume remove NAME log it; nothing else happens +// msb copy HOST NAME:DEST copy HOST's contents into DEST +// msb exec NAME --no-tty -- sh -c SCRIPT sh ARGS… +// the scripts the adapter sends: a guest read, a guest write from stdin, +// the user-namespace block, `rm -rf PATH`, and the worker's launch (which +// starts the real worker/server.js of the bundle, with the stand-in +// binding, listening on the VM's published port). +// Every call is appended to FAKE_MSB_STATE/calls.jsonl as { at, argv }. +const { appendFileSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { homedir } = require("node:os"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_MSB_STATE; +if (!state) { + console.error("fake msb: FAKE_MSB_STATE is not set"); + process.exit(2); +} +const vmRoot = (name) => join(state, "vms", `${name}.json`); +const guestIn = (name) => (path) => join(state, "guest", name, path.replace(/^\//, "")); +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const fail = (message) => { console.error(`error: ${message}`); process.exit(1); }; +const at = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); +const read = (name) => (existsSync(vmRoot(name)) ? JSON.parse(readFileSync(vmRoot(name), "utf8")) : null); +const save = (name, value) => { mkdirSync(dirname(vmRoot(name)), { recursive: true }); writeFileSync(vmRoot(name), JSON.stringify(value, null, 1)); }; + +function endWorker(vm) { + if (!vm?.pid) return; + try { process.kill(vm.pid); } catch { /* already gone */ } + vm.pid = null; + save(vm.name, vm); +} + +/** The worker the adapter launched: the bundle's own server.js, on the VM's published port. */ +function launchWorker(vm) { + endWorker(vm); // a start ends the worker an earlier start left, as the launch script does + const guest = guestIn(vm.name); + const env = Object.fromEntries(readFileSync(guest("/layerfs/state/worker.env"), "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const script = guest("/layerfs/bundle/api/runtime/src/worker/server.js"); + if (!existsSync(script)) fail("the bundle holds no worker/server.js"); + mkdirSync(dirname(guest("/layerfs/state/worker.log")), { recursive: true }); + const child = spawn(process.execPath, [script], { + env: { ...process.env, ...env, LAYERFS_HOME: guest(env.LAYERFS_HOME), LAYERFS_BUNDLE: guest(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${vm.port}`, LAYERFS_NATIVE: process.env.FAKE_MSB_NATIVE }, + detached: true, stdio: ["ignore", "ignore", openSync(guest("/layerfs/state/worker.log"), "a")], + }); + child.unref(); + vm.pid = child.pid; + save(vm.name, vm); + const pid = guest("/layerfs/state/worker.pid"); + mkdirSync(dirname(pid), { recursive: true }); + writeFileSync(pid, `${child.pid}\n`); +} + +const command = argv[0]; +if (command === "inspect") { + const vm = read(argv.at(-1)) ?? fail(`sandbox not found: ${argv.at(-1)}`); + process.stdout.write(JSON.stringify({ + name: vm.name, + status: vm.state === "running" ? "Running" : "Stopped", + created_at: vm.created, + active_config: { + image: { Oci: { reference: vm.image, manifest_digest: `sha256:${"0".repeat(64)}` } }, + labels: vm.labels, + mounts: [ + { guest: "/layerfs/home", host: vm.home, type: "Bind", options: { readonly: false } }, + { guest: "/layerfs/store", name: vm.store, type: "Named", options: { readonly: false } }, + ], + network: { ports: [{ guest_port: 7070, host_port: vm.port, host_bind: "127.0.0.1", protocol: "tcp" }] }, + }, + })); +} else if (command === "create") { + const name = at("--name"); + if (read(name)) fail(`sandbox already exists: sandbox '${name}' already exists; remove it, start the stopped sandbox, or recreate with .replace()`); + // H39v's finding: a create that fails on its mounts leaves the sandbox's own + // directory behind, invisible to `msb list` and uncleaned by `msb remove` + if (process.env.FAKE_MSB_FAIL_CREATE === "mount") { + mkdirSync(join(homedir(), ".microsandbox", "sandboxes", name), { recursive: true }); + fail("the host path for one of the mounts does not exist"); + } + const labels = Object.fromEntries(argv.flatMap((value, index) => (value === "--label" ? [argv[index + 1].split("=")] : []))); + const published = String(at("-p") ?? "").split(":"); + save(name, { + name, state: "running", created: new Date().toISOString(), image: argv.at(-1), pid: null, + labels, home: String(at("-v") ?? "").split(":")[0], store: String(at("--mount-named") ?? "").split(":")[0], + port: Number(published[1]), + }); +} else if (command === "start" || command === "restart") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + vm.state = "running"; + save(vm.name, vm); +} else if (command === "stop") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + endWorker(vm); + vm.state = "stopped"; + save(vm.name, vm); +} else if (command === "remove") { + const vm = read(argv.at(-1)) ?? fail(`sandbox not found: ${argv.at(-1)}`); + endWorker(vm); + rmSync(vmRoot(vm.name), { force: true }); + rmSync(join(state, "guest", vm.name), { recursive: true, force: true }); +} else if (command === "volume") { + // `volume remove NAME`: nothing is stored for volumes here +} else if (command === "copy") { + const [source, target] = [argv[1], argv[2]]; + const name = target.slice(0, target.indexOf(":")); + const guest = guestIn(name); + const destination = guest(target.slice(target.indexOf(":") + 1)); + mkdirSync(destination, { recursive: true }); + for (const entry of readdirSync(source, { withFileTypes: true })) { + const from = join(source, entry.name), to = join(destination, entry.name); + if (entry.isDirectory()) cpSync(from, to, { recursive: true }); else cpSync(from, to); + } +} else if (command === "exec") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + const guest = guestIn(vm.name); + const rest = argv.slice(argv.indexOf("--") + 1); + const [tool, flag, script, ...args] = rest; + const put = (path, text) => { mkdirSync(dirname(guest(path)), { recursive: true }); writeFileSync(guest(path), text); }; + if (tool === "rm") { + rmSync(guest(rest[2]), { recursive: true, force: true }); + } else if (tool === "df") { + process.stdout.write("Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/vdc 20971520 1048576 15728640 6% /layerfs/store\n"); + } else if (tool === "sh" && flag === "-c") { + if (/test -f "\$1" && cat "\$1"/.test(script)) { + if (!existsSync(guest(args[1]))) process.exit(1); + process.stdout.write(readFileSync(guest(args[1]))); + } else if (/cat > "\$1"/.test(script)) { + put(args[1], readFileSync(0)); + } else if (/umask 022/.test(script)) { + mkdirSync(guest(args[1]), { recursive: true, mode: 0o755 }); + } else if (/max_user_namespaces/.test(script)) { + put("/proc/sys/user/max_user_namespaces", "0\n"); + } else if (/nohup node/.test(script)) { + launchWorker(vm); + } + } +} else { + fail(`fake msb does not know ${command}`); +} diff --git a/api/runtime/test/fixture/fake-ssh.cjs b/api/runtime/test/fixture/fake-ssh.cjs new file mode 100644 index 0000000000..f2616f42af --- /dev/null +++ b/api/runtime/test/fixture/fake-ssh.cjs @@ -0,0 +1,184 @@ +// The `ssh` stand-in for @layerfs/runtime's vm:ssh tests (#515 §6): no host. +// It answers exactly the calls `platform/vm/ssh.js` makes, keeps one guest root +// under FAKE_SSH_STATE/guest, and serves the private stream itself: +// +// ssh -N -o ExitOnForwardFailure=yes -L 127.0.0.1:H:127.0.0.1:G … the forward +// ssh … -- uname -s | findmnt | blkid | df -kP | sh -c SCRIPT sh ARGS… +// tar -C -cf - . | ssh … -- mkdir -p DEST && tar -C DEST -xf - +// +// Facts worth knowing before changing it: +// - the forward is lazy: the worker's port is read from state/worker.port when +// a client connects, so the forward may start before the worker does, and +// two test runs never fight over one fixed port; +// - it exits 255 with `Address already in use` when its local port is taken, +// which is what `-o ExitOnForwardFailure=yes` reports; +// - `FAKE_SSH_UNREACHABLE=1` makes every call fail like a dead network, +// `FAKE_SSH_STORE_MISSING=/path` makes `test -e` fail for that path, and +// `FAKE_SSH_STORE_NON_EXT4=xfs` answers that filesystem instead of ext4; +// - every call is appended to FAKE_SSH_STATE/calls.jsonl as { at, argv }. +const { appendFileSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { createServer, connect } = require("node:net"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_SSH_STATE; +if (!state) { + console.error("fake ssh: FAKE_SSH_STATE is not set"); + process.exit(2); +} + +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const guestIn = (path) => join(state, "guest", path.replace(/^\//, "")); +const workerPortFile = join(state, "worker.port"); +const workerPidFile = () => guestIn("/layerfs/state/worker.pid"); +const log = (...parts) => process.stderr.write(`${parts.join(" ")}\n`); + +/** A port nothing listens on now: the stand-in worker's own, chosen per launch. */ +function freePort() { + return new Promise((resolve) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +function endWorker() { + const file = workerPidFile(); + if (!existsSync(file)) return; + try { process.kill(Number(readFileSync(file, "utf8").trim())); } catch { /* already gone */ } +} + +/** The bundle's own worker, on a free port this stand-in records. */ +async function launchWorker() { + endWorker(); + const envFile = guestIn("/layerfs/state/worker.env"); + const env = Object.fromEntries(readFileSync(envFile, "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const script = guestIn("/layerfs/bundle/api/runtime/src/worker/server.js"); + if (!existsSync(script)) { log("the bundle holds no worker/server.js"); process.exit(1); } + const port = await freePort(); + writeFileSync(workerPortFile, `${port}\n`); + const logFile = guestIn("/layerfs/state/worker.log"); + mkdirSync(dirname(logFile), { recursive: true }); + const child = spawn(process.execPath, [script], { + env: { ...process.env, ...env, LAYERFS_HOME: guestIn(env.LAYERFS_HOME), LAYERFS_BUNDLE: guestIn(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${port}`, LAYERFS_NATIVE: process.env.FAKE_SSH_NATIVE }, + detached: true, + stdio: ["ignore", "ignore", openSync(logFile, "a")], + }); + child.unref(); + mkdirSync(dirname(workerPidFile()), { recursive: true }); + writeFileSync(workerPidFile(), `${child.pid}\n`); +} + +/** + * The private stream. Lazy on purpose: a client that arrives before the worker + * exists waits up to a second for state/worker.port rather than failing. + */ +function forward(hostPort) { + const server = createServer((client) => { + client.on("error", () => {}); + let tries = 0; + const dial = () => { + const port = existsSync(workerPortFile) ? Number(readFileSync(workerPortFile, "utf8").trim()) : NaN; + if (!Number.isFinite(port)) { + if (tries++ < 20) return setTimeout(dial, 50); + return client.destroy(); + } + const upstream = connect(port, "127.0.0.1"); + upstream.on("error", () => client.destroy()); + client.pipe(upstream); + upstream.pipe(client); + }; + dial(); + }); + server.on("error", (error) => { + if (error.code === "EADDRINUSE") log(`bind [127.0.0.1]:${hostPort}: Address already in use`); + else log(`forward error: ${error.message}`); + process.exit(255); + }); + server.listen(hostPort, "127.0.0.1"); +} + +/** `sh -c SCRIPT sh ARG…`: the handful of scripts the adapter sends. `argv[0]` is + * the `sh` that names the script's `$0`, so the script's `$1` is `args[1]`. */ +function shellScript(script, args) { + const path = args[1]; + if (script.includes("max_user_namespaces")) { + const file = guestIn("/proc/sys/user/max_user_namespaces"); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, "0\n"); + return 0; + } + if (script.includes("test -f \"$1\" && cat \"$1\"")) { + const file = guestIn(path); + if (!existsSync(file)) return 1; + process.stdout.write(readFileSync(file)); + return 0; + } + if (script.includes("cat > \"$1\"")) { + const file = guestIn(path); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + writeFileSync(file, readFileSync(0), { mode: 0o600 }); + return 0; + } + if (script.includes("test -e \"$1\"")) return process.env.FAKE_SSH_STORE_MISSING === path ? 1 : 0; + if (script.includes("blkid")) { process.stdout.write(`${process.env.FAKE_SSH_STORE_NON_EXT4 || "ext4"}\n`); return 0; } + if (script.includes("test -b \"$1\"")) return 1; + if (script.includes("mount") && script.includes("/layerfs/store")) return 0; + if (script.includes("nohup node")) return launchWorker().then(() => 0); + if (script.includes("worker.pid") && script.includes("rm -rf")) { + endWorker(); + rmSync(guestIn("/layerfs/state"), { recursive: true, force: true }); + rmSync(guestIn("/layerfs/bundle"), { recursive: true, force: true }); + return 0; + } + if (script.includes("worker.pid")) { endWorker(); return 0; } + return 0; +} + +/** The unpack end of a bundle copy: `mkdir -p DEST && tar -C DEST -xf -`. */ +function unpack(script) { + const match = /tar -C ('([^']+)'|\S+) -xf -/.exec(script); + const destination = guestIn(match ? match[2] ?? match[1] : "/layerfs/bundle"); + mkdirSync(destination, { recursive: true }); + const tar = spawn("tar", ["-C", destination, "-xf", "-"], { stdio: ["pipe", "ignore", "inherit"] }); + process.stdin.pipe(tar.stdin); + tar.on("close", (code) => process.exit(code ?? 0)); +} + +async function main() { + if (process.env.FAKE_SSH_UNREACHABLE) { + log("ssh: connect to host: Network is unreachable"); + process.exit(255); + } + const dash = argv.indexOf("--"); + const remoteArgv = dash === -1 ? [] : argv.slice(dash + 1); + const joined = remoteArgv.join(" "); + + if (argv.includes("-N")) { + const index = argv.indexOf("-L"); + const port = index === -1 ? NaN : Number(/^127\.0\.0\.1:(\d+):/.exec(argv[index + 1])?.[1]); + if (!Number.isFinite(port)) { log("ssh: no local port to forward"); process.exit(255); } + return forward(port); + } + if (!remoteArgv.length) { log("ssh: no command"); process.exit(255); } + if (remoteArgv[0] === "uname") { process.stdout.write(`${process.env.FAKE_SSH_OS || "Linux"}\n`); return; } + if (remoteArgv[0] === "findmnt") { process.stdout.write(`${process.env.FAKE_SSH_STORE_NON_EXT4 || "ext4"}\n`); return; } + if (remoteArgv[0] === "df") { + process.stdout.write("Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 100000000 5000000 95000000 5% /layerfs/store\n"); + return; + } + if (remoteArgv[0] === "tar" || /tar -C/.test(joined)) return unpack(joined); + if (remoteArgv[0] === "rm") { rmSync(guestIn(remoteArgv.at(-1)), { recursive: true, force: true }); return; } + if (remoteArgv[0] === "sh") { + const code = await shellScript(remoteArgv[2] ?? "", remoteArgv.slice(3)); + process.exit(code); + } + log(`fake ssh does not know ${joined}`); + process.exit(1); +} + +main(); diff --git a/api/runtime/test/verify/ssh_faults.mjs b/api/runtime/test/verify/ssh_faults.mjs new file mode 100644 index 0000000000..fdc300837a --- /dev/null +++ b/api/runtime/test/verify/ssh_faults.mjs @@ -0,0 +1,342 @@ +// The `vm:ssh` failure matrix (#515 §6, V3; task H40 §5), with the stand-in +// transport of `vm_placement.mjs` and the real provider code: no host, no VM. +// Each scenario states what the user sees and what is left behind; every await +// has a deadline and the whole run has a watchdog, so a stuck provider call +// fails this file instead of hanging a shell. +// node api/runtime/test/verify/ssh_faults.mjs [port|store|ext4|host|drop|disconnect|remote|delete] +// (no selector: every scenario, one fresh state folder each) +import assert from "node:assert/strict"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, connect } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SELF = fileURLToPath(import.meta.url); +const ROOT = fileURLToPath(new URL("../../..", import.meta.url)); +const FAKE = fileURLToPath(new URL("../../../agent/test/fixture/fake-native.cjs", import.meta.url)); +const SOURCE = fileURLToPath(new URL("../../src", import.meta.url)); +const SSH = fileURLToPath(new URL("../fixture/fake-ssh.cjs", import.meta.url)); +process.env.LAYERFS_NATIVE = FAKE; +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const provider = (await import("../../src/platform/vm/ssh.js")).default; +const { endpointFile, ping } = await import("../../src/platform/shared.js"); +const { deploymentEnv } = await import("../../src/clients.js"); + +const SCENARIOS = ["port", "store", "ext4", "host", "drop", "disconnect", "remote", "delete", "shared"]; + +/** One await's ceiling: a stuck call fails with its own name. */ +const deadline = (ms, what, promise) => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not finish within ${ms / 1000} s`)), ms).unref()), + ]); + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const sleep = delay; +const fails = (promise, matcher) => + promise.then((value) => assert.fail(`expected a refusal, got ${JSON.stringify(value)}`), (error) => { + assert.match(`${error.code} ${error.message}`, matcher, `${error.code}: ${error.message}`); + return error; + }); + +/** A sealed Linux bundle with this adapter's worker and the real SDK packages. */ +function bundleAt(folder) { + rmSync(folder, { recursive: true, force: true }); + cpSync(join(SOURCE, "worker"), join(folder, "api/runtime/src/worker"), { recursive: true }); + cpSync(join(ROOT, "node_modules/@layerfs"), join(folder, "api/node_modules/@layerfs"), { recursive: true, dereference: true }); + mkdirSync(join(folder, "images"), { recursive: true }); + writeFileSync(join(folder, "images", `${"a".repeat(64)}.tar`), "an uncompressed docker save archive\n"); + writeFileSync(join(folder, "manifest.json"), JSON.stringify({ version: 1, status: "sealed", platform: "linux", build_id: "faults", files: {} })); + return folder; +} + +/** One scenario's own world: a bundle, a home, a stand-in SSH on PATH, a state folder. */ +function world(tag) { + const work = mkdtempSync(join(tmpdir(), `layerfs-ssh-${tag}-`)); + const state = join(work, "state"); + mkdirSync(state, { recursive: true }); + const bin = join(work, "bin"); + mkdirSync(bin, { recursive: true }); + const ssh = join(bin, "ssh"); + writeFileSync(ssh, `#!/bin/sh\nexec "${process.execPath}" "${SSH}" "$@"\n`); + chmodSync(ssh, 0o755); + const saved = { path: process.env.PATH, state: process.env.FAKE_SSH_STATE, native: process.env.FAKE_SSH_NATIVE, dir: process.env.LAYERFS_SSH_STATE_DIR }; + process.env.PATH = `${bin}:${saved.path}`; + process.env.FAKE_SSH_STATE = state; + process.env.FAKE_SSH_NATIVE = FAKE; + process.env.LAYERFS_SSH_STATE_DIR = join(state, "sessions"); + return { + work, + state, + home: join(work, "home"), + bundle: bundleAt(join(work, "bundle")), + sessions: () => (existsSync(join(state, "sessions")) ? readdirSync(join(state, "sessions")).filter((file) => file.endsWith(".json")) : []), + calls: () => (existsSync(join(state, "calls.jsonl")) ? readFileSync(join(state, "calls.jsonl"), "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line).argv) : []), + guest: (...parts) => join(state, "guest", ...parts), + done: () => { + for (const [key, value] of [["PATH", saved.path], ["FAKE_SSH_STATE", saved.state], ["FAKE_SSH_NATIVE", saved.native], ["LAYERFS_SSH_STATE_DIR", saved.dir]]) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + killStandIns(state); + rmSync(work, { recursive: true, force: true }); + }, + }; +} + +/** Ends every process this world started: forwards from their records, workers from their pid files. + * A process that refuses to end is reported: a leaked stand-in is a test failure, not a silent one. */ +function killStandIns(state) { + const pids = []; + const sessions = join(state, "sessions"); + if (existsSync(sessions)) { + for (const file of readdirSync(sessions).filter((name) => name.endsWith(".json"))) { + try { + const { forwardPid } = JSON.parse(readFileSync(join(sessions, file), "utf8")); + if (forwardPid) pids.push(forwardPid); + } catch { /* already gone */ } + } + } + const pid = join(state, "guest/layerfs/state/worker.pid"); + if (existsSync(pid)) pids.push(Number(readFileSync(pid, "utf8").trim())); + for (const one of pids) { + try { process.kill(one, "SIGKILL"); } catch { /* already gone */ } + } + const left = pids.filter((one) => provider.isAlive(one)); + if (left.length) process.stderr.write(`ssh_faults: residue, these stand-ins did not end: ${left.join(", ")}\n`); +} + +/** A project of the home, as Init leaves it: an entry and the runtime's record. */ +function project(home, name) { + mkdirSync(join(home, name), { recursive: true }); + writeFileSync(join(home, name, "project.record"), "x"); + mkdirSync(join(home, ".runtime", "projects"), { recursive: true }); + writeFileSync(join(home, ".runtime", "projects", `${name}.json`), JSON.stringify({ name, volume: `lfs-${name}-store-1`, created: 1 })); +} + +/** A local port nothing listens on now, for a forward that must succeed. */ +const freePort = () => new Promise((resolve) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { const { port } = server.address(); server.close(() => resolve(port)); }); +}); + +/** Whether something accepts a connection on a loopback port now. */ +const answers = (port) => new Promise((resolve) => { + const socket = connect({ host: "127.0.0.1", port }); + const done = (value) => { socket.destroy(); resolve(value); }; + socket.setTimeout(500); + socket.on("connect", () => done(true)); + socket.on("timeout", () => done(false)); + socket.on("error", () => done(false)); +}); + +const config = (world, overrides = {}) => ({ + name: "vm-a", host: "127.0.0.1", user: "root", port: 22, storePath: "/layerfs/store", + home: world.home, storeName: "store-a", workerPort: 7070, hostPort: 0, ...overrides, +}); + +const scenarios = { + // ① a local forward port someone else holds: create fails at once, and nothing + // was uploaded or started on the host + async port(world) { + const taken = await freePort(); + const server = createServer(() => {}); + await new Promise((resolve) => server.listen(taken, "127.0.0.1", resolve)); + try { + const error = await deadline(40_000, "create with a busy port", fails(provider.create(config(world, { hostPort: taken })), /could not open the private stream.*Address already in use|address already in use/i)); + assert.match(error.message, new RegExp(`127\\.0\\.0\\.1:${taken}`), error.message); + assert.deepEqual(world.sessions(), [], "no session record is written"); + const calls = world.calls(); + assert.ok(!calls.some((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))), "nothing is uploaded"); + assert.ok(!calls.some((argv) => argv.some((value) => typeof value === "string" && value.includes("nohup node"))), "no worker is started"); + // the same VM on a free port is made: the refusal was about the port, not the host + const made = await deadline(60_000, "create on a free port", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(made.state, "running"); + await deadline(30_000, "remove", provider.remove("vm-a")); + assert.deepEqual(world.sessions(), [], "a delete clears the record"); + } finally { + server.close(); + } + }, + + // ② the Store path does not exist: refused before the forward, with nothing left + async store(world) { + process.env.FAKE_SSH_STORE_MISSING = "/mnt/nope"; + try { + await deadline(40_000, "create with a missing Store path", fails(provider.create(config(world, { storePath: "/mnt/nope" })), /InvalidInput.*does not exist/)); + } finally { + delete process.env.FAKE_SSH_STORE_MISSING; + } + assert.deepEqual(world.sessions(), [], "no session record"); + assert.deepEqual(world.calls().filter((argv) => argv.includes("-N")), [], "no forward is opened"); + }, + + // ③ the Store path is not ext4: refused by name, with the filesystem it is + async ext4(world) { + process.env.FAKE_SSH_STORE_NON_EXT4 = "xfs"; + try { + const error = await deadline(40_000, "create on a non-ext4 path", fails(provider.create(config(world, { storePath: "/mnt/store" })), /Unsupported.*not ext4/)); + assert.match(error.message, /xfs/); + } finally { + delete process.env.FAKE_SSH_STORE_NON_EXT4; + } + assert.deepEqual(world.sessions(), []); + }, + + // ④ a host that is not Linux: refused by what it answers, not by a guess + async host(world) { + process.env.FAKE_SSH_OS = "Darwin"; + try { + await deadline(40_000, "create against a non-Linux host", fails(provider.create(config(world)), /Unsupported.*not Linux/)); + } finally { + delete process.env.FAKE_SSH_OS; + } + assert.deepEqual(world.sessions(), []); + }, + + // ⑤ the network is down, then it is back: the first call fails clearly, the + // second one makes the VM + async drop(world) { + process.env.FAKE_SSH_UNREACHABLE = "1"; + try { + const error = await deadline(40_000, "create while unreachable", fails(provider.create(config(world)), /Failed.*(SSH to 127\.0\.0\.1 failed|could not reach)/)); + assert.match(error.message, /Network is unreachable/, "the refusal carries the transport's own reason"); + assert.match(error.message, /127\.0\.0\.1/, "and names the host"); + } finally { + delete process.env.FAKE_SSH_UNREACHABLE; + } + assert.deepEqual(world.sessions(), [], "an unreachable host leaves no record"); + const made = await deadline(60_000, "create after reconnect", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(made.state, "running"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, + + // ⑥ the SSH forward dies under a running VM (a dropped connection): the VM + // reads stopped, its endpoint refuses, and start brings it back + async disconnect(world) { + const hostPort = await freePort(); + const made = await deadline(60_000, "create", provider.create(config(world, { hostPort }))); + assert.equal(made.state, "running"); + assert.equal(await answers(hostPort), true, "the forward answers while it runs"); + const pid = made.ports[0].host; + const record = JSON.parse(readFileSync(join(world.state, "sessions", "vm-a.json"), "utf8")); + process.kill(record.forwardPid); + for (let waited = 0; waited < 2_000 && (await answers(pid)); waited += 50) await sleep(50); + assert.equal(await answers(pid), false, "the dropped forward refuses, so a call fails with a clear connection error"); + assert.equal((await deadline(20_000, "inspect after the drop", provider.inspect("vm-a"))).state, "stopped"); + const started = await deadline(60_000, "start after the drop", provider.start("vm-a")); + assert.equal(started.state, "running"); + assert.equal(await answers(pid), true, "the same port answers again"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, + + // ⑦ the host (or its worker) is gone, then back: the runtime re-opens the + // worker without copying the bundle again + async remote(world) { + project(world.home, "shop"); + const runtime = await LayerFSRuntime.open({ + home: world.home, bundle: world.bundle, placement: "vm:ssh", + containerPrefix: "lfs-fault-", platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + const first = await deadline(90_000, "worker.start", runtime.worker.start("shop")); + assert.deepEqual([first.vm.state, first.worker.state], ["running", "running"]); + const copies = () => world.calls().filter((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))).length; + assert.equal(copies(), 1, "the bundle is uploaded once"); + const endpoint = JSON.parse(readFileSync(endpointFile(world.home, "shop"), "utf8")); + assert.equal((await deadline(5_000, "ping a running VM", ping(endpoint)))?.ready, true); + // the host reboots: its worker and the forward are gone, the disk stays + await deadline(30_000, "stop", runtime.vm.stop("shop")); + const pid = join(world.state, "guest/layerfs/state/worker.pid"); + assert.equal(existsSync(pid), true, "the stand-in recorded the worker it started"); + assert.equal((await deadline(2_000, "ping a stopped VM", ping(endpoint))), null, "a stopped VM's worker does not answer"); + const back = await deadline(90_000, "worker.start after the reboot", runtime.worker.start("shop")); + assert.deepEqual([back.vm.state, back.worker.state], ["running", "running"]); + assert.equal(copies(), 1, "the bundle on the host's disk was reused, not uploaded again"); + assert.equal((await deadline(5_000, "ping after the restart", ping(JSON.parse(readFileSync(endpointFile(world.home, "shop"), "utf8")))))?.ready, true); + // an un-pushed Store is never destroyed by a delete + await deadline(20_000, "delete with a project", fails(runtime.vm.delete("shop", { confirm: "delete" }), /Unsupported.*never destroys a Store/)); + assert.equal(world.sessions().length, 1, "the refused delete left the VM alone"); + } finally { + await runtime.close(); + } + }, + + // ⑧ delete after the project is gone: the record, the forward and this + // adapter's remote copy go; the user's Store path is untouched + async delete(world) { + mkdirSync(world.guest("layerfs/store/global"), { recursive: true }); + writeFileSync(world.guest("layerfs/store/global/store.sqlite"), "the user's Store\n"); + project(world.home, "shop"); + const runtime = await LayerFSRuntime.open({ + home: world.home, bundle: world.bundle, placement: "vm:ssh", + containerPrefix: "lfs-fault-", platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + await deadline(90_000, "worker.start", runtime.worker.start("shop")); + // the project goes: its entry and the runtime's record + rmSync(join(world.home, "shop"), { recursive: true, force: true }); + rmSync(join(world.home, ".runtime", "projects", "shop.json"), { force: true }); + const gone = await deadline(60_000, "vm.delete", runtime.vm.delete("shop", { confirm: "delete" })); + assert.deepEqual([gone.state, gone.store], ["deleted", "kept"], "the Store disk is kept by default"); + assert.deepEqual(world.sessions(), [], "the session record is gone"); + assert.equal(existsSync(world.guest("layerfs/bundle")), false, "the adapter's bundle copy is removed"); + assert.equal(existsSync(world.guest("layerfs/state")), false, "its worker state is removed"); + assert.equal(readFileSync(world.guest("layerfs/store/global/store.sqlite"), "utf8"), "the user's Store\n", "the user's Store path is untouched"); + } finally { + await runtime.close(); + } + }, + + // ⑨ an un-pushed Store is refused by the runtime, and one host never serves two + async shared(world) { + const first = await deadline(60_000, "create the first VM", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(first.state, "running"); + const second = await deadline(30_000, "create a second VM on the same host", fails(provider.create(config(world, { name: "vm-b", storeName: "store-b", hostPort: await freePort() })), /Busy.*one host serves one Store/)); + assert.match(second.message, /vm-a/, "the refusal names the VM that already holds it"); + assert.deepEqual(world.sessions(), ["vm-a.json"], "the refused create wrote no record"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, +}; + +const selected = process.argv[2]; +if (!selected) { + // One child per scenario: a runtime binds this process's worker route once + // per project (`useWorker`), so two runtimes cannot share a process, and a + // scenario that hangs ends its own child instead of the whole file. + const { spawn } = await import("node:child_process"); + const results = []; + for (const name of SCENARIOS) { + const code = await new Promise((resolve) => { + const child = spawn(process.execPath, [SELF, name], { stdio: ["ignore", "inherit", "inherit"], env: process.env }); + const timer = setTimeout(() => { child.kill("SIGKILL"); }, 120_000); + child.on("close", (value) => { clearTimeout(timer); resolve(value); }); + }); + results.push([name, code]); + } + const failed = results.filter(([, code]) => code !== 0); + process.stdout.write(`${results.map(([name, code]) => `${code === 0 ? "PASS" : "FAIL"} ssh_faults/${name}`).join("\n")}\n`); + if (failed.length) { + process.stderr.write(`ssh_faults: ${failed.length} of ${results.length} scenarios failed: ${failed.map(([name]) => name).join(", ")}\n`); + process.exit(1); + } + process.stdout.write("PASS ssh_faults\n", () => process.exit(0)); +} else { + if (!scenarios[selected]) throw new Error(`select ${SCENARIOS.join(", ")}`); + setTimeout(() => { + process.stderr.write(`ssh_faults/${selected}: FAIL, the scenario did not finish in 120 s\n`); + process.exit(1); + }, 120_000).unref(); + + const box = world(selected); + const started = Date.now(); + try { + await deadline(110_000, `scenario ${selected}`, scenarios[selected](box)); + console.log(`PASS ssh_faults/${selected} (${Date.now() - started} ms)`); + } finally { + box.done(); + } + process.stdout.write("PASS ssh_faults\n", () => process.exit(0)); +} diff --git a/api/runtime/test/verify/vm_real.mjs b/api/runtime/test/verify/vm_real.mjs new file mode 100644 index 0000000000..e5079dbea9 --- /dev/null +++ b/api/runtime/test/verify/vm_real.mjs @@ -0,0 +1,363 @@ +// @layerfs/runtime on a `vm:` placement, against real VMs (#515 §6, +// V9, V1): **one Linux VM per Store**. The runtime makes each project's VM, +// copies the sealed bundle into it, blocks user namespaces, starts its worker as +// root and writes that project's endpoint; each project's agent process reaches +// only its own VM, where the Sandboxes are Linux namespaces (H36's backend). +// Two projects run side by side: neither sees the other's Store, removing one +// leaves the other alone, and V8 refuses a delete while its Store exists. +// The Sandbox root is the bundle's image (#515 §4.7 V7, H38): a command sees the +// image's own `/etc/os-release` and marker and never the VM's tree, the image's +// directories are not writable by it while its own overlay layer is, two +// Sandboxes' writes to the image stay each one's, one digest is unpacked once, +// and `LAYERFS_LOCAL_DIR` (their overlay uppers) is on the VM's ext4 disk. +// A Sandbox left behind by a VM restart is a stopped record: it is not reopened +// onto a root that boot did not start, the next `sandbox.create` reclaims it and +// reuses its number, and that Sandbox runs from the same unpacked image. Everything it made (both VMs, their Store disks, the home and +// the sources) is removed afterwards. +// node api/runtime/test/verify/vm_real.mjs [microsandbox|apple-container] +// Needs that provider's CLI and the bundle built for the host's Linux VM +// (aarch64 on an M-series Mac), with a Sandbox image at `images/<64 hex>.tar`. +// Under 2 minutes for small source folders; the first run pulls the image. +// LAYERFS_TEST_DIR (default ~/.cache/layerfs-test) holds the run's home, sources +// and traces. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SELF = fileURLToPath(import.meta.url); +const CLI = { microsandbox: "msb", "apple-container": "container" }; + +/** An agent process: it names its project, and the endpoint of that project's VM comes from the deployment. */ +async function agent(project, deployment, phase = undefined, extra = {}) { + const env = { ...process.env, ...deployment, PROJECT: project, ...(phase ? { AGENT_PHASE: phase } : {}), ...extra }; + return new Promise((resolve) => { + const child = spawn(process.execPath, [SELF, "agent"], { env, stdio: ["ignore", "pipe", "pipe"] }); + const out = []; + const timer = setTimeout(() => child.kill("SIGKILL"), 100_000); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => out.push(chunk)); + child.on("close", (code) => { clearTimeout(timer); resolve({ code, text: Buffer.concat(out).toString() }); }); + }); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "resume") { + // after the project's VM was restarted and its worker restarted: the committed + // lane is still there, and the Sandbox the previous boot left is a stopped + // record, never reopened onto a root this boot did not start + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const left = process.env.AGENT_LEFTOVER; + try { + assert.equal((await shop.fs.read("lane", "/fresh.txt")).text, "two\n"); + if (left) { + // the Sandbox the previous boot left: it is not running, and it is not + // reopened onto a root this boot did not start + assert.ok(!(await shop.project.list_sandboxes(1000, 0)).items.some((entry) => entry.sandbox === left), `the Sandbox ${left} of the previous boot is not running`); + await assert.rejects(shop.workspace.mount(left, "lane"), (error) => { + console.log(`reopening the restarted VM's Sandbox was refused: ${error.code}: ${error.message}`); + return true; + }); + } + // the record whose init the reboot ended is reclaimed by the next create: + // its number comes back, and the same digest is started from the cache the + // VM kept rather than unpacked again + const { sandbox } = await shop.sandbox.create(); + if (left) { + console.log(`reclaimed=${left} reused=${sandbox}`); + assert.equal(sandbox, left, `the reclaimed record's number is free again: ${sandbox}`); + } + const ws = await shop.workspace.mount(sandbox, "lane"); + const ran = await shop.workspace.exec(ws, "cat fresh.txt; cat /etc/layerfs-image"); + assert.equal(ran.stdout, "two\nh39-sandbox-image\n", ran.stderr); + await shop.workspace.unmount(ws); + await shop.sandbox.close(sandbox); + } finally { + await shop.close(); + } + console.log(`${process.env.PROJECT} after the worker restart: ok`); + process.exit(0); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "third") { + // the bundle's Sandbox image (#515 §4.7 V7): a command's root is the image, + // not the VM's tree; the image's directories are read-only to it while its + // own writable layer is not, and two Sandboxes' writes stay each one's + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const open = []; + try { + const { sandbox } = await shop.sandbox.create(); + const ws = await shop.workspace.mount(sandbox, "lane"); + open.push(() => shop.sandbox.close(sandbox), () => shop.workspace.unmount(ws, true)); + const image = await shop.workspace.exec(ws, "cat /etc/os-release; cat /etc/layerfs-image; ls /layerfs"); + assert.equal(image.exitCode, 0, image.stderr); + assert.match(image.stdout, /^PRETTY_NAME="LayerFS Sandbox Image \(busybox static\)"$/m, image.stdout); + assert.match(image.stdout, /^h39-sandbox-image$/m, image.stdout); + assert.equal(image.stdout.split("\n").at(-2), "store", `only the Store's directory is bound into the image, not the VM's /layerfs: ${image.stdout}`); + // the image is not the VM's root: the VM's node and bundle are absent + const shape = await shop.workspace.exec(ws, "command -v node; echo node=$?; ls /bin/busybox; id -u"); + assert.match(shape.stdout, /^node=[1-9]/m, shape.stdout); + assert.match(shape.stdout, /^\/bin\/busybox$/m, shape.stdout); + assert.match(shape.stdout, /^501$/m, shape.stdout); + // the image's own directories are root's: a command cannot write them, and + // what it writes goes to its own writable layer instead + const writes = await shop.workspace.exec(ws, "printf x > /etc/layerfs-probe; echo etc-write=$?; test -w /etc; echo etc-w=$?; printf mine > /var/tmp/mine; echo tmp-write=$?"); + assert.match(writes.stdout, /^etc-write=[1-9]/m, writes.stdout); + assert.match(writes.stdout, /^etc-w=[1-9]/m, writes.stdout); + assert.match(writes.stdout, /^tmp-write=0$/m, writes.stdout); + // a second Sandbox on the same image reads the image, not the first one's layer + const { sandbox: other } = await shop.sandbox.create(); + const ws2 = await shop.workspace.mount(other, "lane"); + open.push(() => shop.sandbox.close(other), () => shop.workspace.unmount(ws2, true)); + const apart = await shop.workspace.exec(ws2, "cat /var/tmp/mine 2>&1; echo mine=$?; printf other > /var/tmp/mine; cat /var/tmp/mine"); + assert.match(apart.stdout, /No such file|not found/, apart.stdout); + assert.match(apart.stdout, /^mine=[1-9]$/m, apart.stdout); + assert.match(apart.stdout, /^other$/m, apart.stdout); + } finally { + for (const step of open.reverse()) await step().catch((error) => console.error(`teardown: ${error.message}`)); + await shop.close(); + } + console.log("the Sandbox root is the bundle's image: ok"); + process.exit(0); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "blog") { + // the other project's agent: its own VM, its own Store + const { LayerFS } = await import("@layerfs/agent"); + const blog = await LayerFS.open(process.env.PROJECT); + try { + assert.equal((await blog.fs.read("main", "/README.md")).text, "# Blog\n"); + // the first project's Branch and files are not in this Store at all + assert.ok(!(await blog.project.list_branches(1000, 0)).items.some((branch) => branch.name === "lane"), "the other project's Branch is not here"); + await assert.rejects(blog.fs.read("lane", "/note.txt"), { code: "NoSuchBranch" }); + await blog.branch.fork("blog-work"); + const { sandbox } = await blog.sandbox.create(); + const ws = await blog.workspace.mount(sandbox, "blog-work"); + try { + const ran = await blog.workspace.exec(ws, "id -u; cat README.md"); + assert.match(ran.stdout, /^501\n# Blog\n$/m, ran.stdout); + } finally { + await blog.workspace.unmount(ws, true); + await blog.sandbox.close(sandbox); + } + } finally { + await blog.close(); + } + console.log("blog over its own VM worker: ok"); + process.exit(0); +} + +if (process.argv[2] === "agent") { + // the first project's agent: only the deployment's endpoint is set, never LAYERFS_WORKER_ENDPOINT by hand + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const undone = []; + try { + await shop.branch.fork("lane"); + await shop.fs.write("lane", "/note.txt", "one\n", "Add note"); + const { sandbox } = await shop.sandbox.create(); + const ws = await shop.workspace.mount(sandbox, "lane"); + undone.push(() => shop.sandbox.close(sandbox), () => shop.workspace.unmount(ws, true)); + // the command runs in a namespace Sandbox inside the project's VM: nonroot, + // no capabilities, and the VM's home and Store around it + const ran = await shop.workspace.exec(ws, "id -u; cat README.md note.txt && echo two > fresh.txt"); + assert.equal(ran.exitCode, 0, ran.stderr); + assert.match(ran.stdout, /^501\n# Shop\none\n$/m); + // V4: the VM blocks user namespaces, so a command cannot copy the Workspace + // mount into one of its own; the command's own identity is nonroot + const blocked = await shop.workspace.exec(ws, "cat /proc/sys/user/max_user_namespaces; unshare -U /bin/true 2>&1; echo rc=$?"); + assert.match(blocked.stdout, /^0\n/m, blocked.stdout); + assert.match(blocked.stdout, /rc=[1-9]/, blocked.stdout); + assert.equal((await shop.workspace.commit(ws, "Add fresh")).kind, "Committed"); + await shop.workspace.exec(ws, "echo hidden > secret.txt"); + await shop.workspace.unmount(ws, true); + await shop.sandbox.close(sandbox); + undone.length = 0; + assert.equal((await shop.fs.read("lane", "/fresh.txt")).text, "two\n"); + // a second Sandbox on the same Store has its own mount + const { sandbox: second } = await shop.sandbox.create(); + const ws2 = await shop.workspace.mount(second, "lane"); + undone.push(() => shop.sandbox.close(second), () => shop.workspace.unmount(ws2, true)); + const isolated = await shop.workspace.exec(ws2, "test -e secret.txt && echo leaked || echo isolated"); + assert.equal(isolated.stdout.trim(), "isolated", isolated.stdout); + const again = await shop.workspace.exec(ws2, "id -u; cat fresh.txt"); + assert.match(again.stdout, /^501\ntwo\n$/m, again.stdout); + // this Sandbox is left open on purpose: the parent stops the VM under it, so + // the reboot leaves a stopped record for the resume phase to close. Its + // Workspace is unmounted first: a stopped Sandbox's daemon is gone, so a + // Workspace still recorded as mounted there could not be unmounted with it. + await shop.workspace.unmount(ws2, true); + undone.length = 0; + console.log(`left=${second}`); + } finally { + for (const step of undone.reverse()) await step().catch((error) => console.error(`teardown: ${error.message}`)); + await shop.close(); + } + console.log("shop over its own VM worker: ok"); + process.exit(0); +} + +const [bundle, provider = "microsandbox"] = process.argv.slice(2); +if (!CLI[provider] || !bundle || process.platform !== "darwin") { + console.error("usage: vm_real.mjs [microsandbox|apple-container] (macOS host)"); + process.exit(2); +} +if (!existsSync(bundle)) { + console.error(`no sealed bundle at ${bundle}`); + process.exit(2); +} +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const { deploymentEnv } = await import("../../src/clients.js"); +const { endpointFile } = await import("../../src/platform/shared.js"); +// the provider module of the VM this run uses: the verify script reads the VM's +// own disk (filesystem, image cache), which no public call answers +const DRIVER = { microsandbox: "../../src/platform/vm/microsandbox.js", "apple-container": "../../src/platform/vm/apple-container.js" }; +const driver = (await import(DRIVER[provider])).default; + +const shared = process.env.LAYERFS_TEST_DIR || join(homedir(), ".cache", "layerfs-test"); +mkdirSync(shared, { recursive: true }); +const work = mkdtempSync(join(shared, `lfs-${provider}-`)); +const home = join(work, "home"), id = Date.now().toString(36), prefix = `lfs-${provider}-${id}-`; +const sources = {}; +for (const [name, text] of [["shop", "# Shop\n"], ["blog", "# Blog\n"]]) { + sources[name] = join(work, name); + mkdirSync(sources[name]); + writeFileSync(join(sources[name], "README.md"), text); +} +const projectEnv = (runtime, name, volume) => Object.fromEntries(Object.entries(deploymentEnv(runtime.settings, runtime.platform, { name, volume })).map(([key, value]) => [`LAYERFS_${key}`, value])); +const initProject = async (runtime, name) => { + let job = await runtime.project.start_init(name, sources[name]); + for (let i = 0; !job.ended && i < 900; i++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + job = await runtime.project.get_init(job); + } + assert.equal(job.step, "done", `${name}'s Init: ${JSON.stringify(job.error)}`); + return (await runtime.project.list()).find((project) => project.name === name); +}; + +/** One `sh -c` line as the named project's VM answers it. */ +const guest = async (project, command) => (await driver.exec(`${prefix}${project}-vm`, ["sh", "-c", command])).stdout.trim(); + +/** + * The project's VM as its disk shows it: the filesystem `LAYERFS_LOCAL_DIR` is on + * (H38's overlay upper needs a local one, not the VM's own overlay root) and the + * one image cache directory — its digest, inode, mtime and mode. An unpack writes + * inside that directory, so an unchanged inode and mtime after another Sandbox + * start is the proof that the same digest was unpacked once. + */ +async function disk(project) { + const digest = await guest(project, "ls -1 /layerfs/store/local/images 2>/dev/null | grep -E '^[0-9a-f]{64}$'"); + return { + digest, + stat: await guest(project, `stat -c '%i %Y' /layerfs/store/local/images/${digest}`), + mode: await guest(project, `stat -c '%a' /layerfs/store/local/images/${digest}`), + partial: await guest(project, "ls -A /layerfs/store/local/images | grep -c '^\\.tmp-' || true"), + local: await guest(project, "findmnt -T /layerfs/store/local -o FSTYPE -n"), + }; +} + +const started = Date.now(); +let runtime; +try { + runtime = await LayerFSRuntime.open({ home, bundle, placement: `vm:${provider}`, containerPrefix: prefix, sweep: false }); + // a project's VM is made on demand, for that project + const first = await runtime.worker.start("shop"); + assert.equal(first.project, "shop"); + assert.equal(first.worker.state, "running", JSON.stringify(first)); + assert.equal(first.vm.state, "running"); + assert.equal(first.vm.name, `${prefix}shop-vm`); + const shopEndpoint = JSON.parse(readFileSync(endpointFile(home, "shop"), "utf8")); + console.log(`${first.vm.name} running; its worker on 127.0.0.1:${shopEndpoint.port}`); + + const shop = await initProject(runtime, "shop"); + const blog = await initProject(runtime, "blog"); + const listed = await runtime.vm.status(); + assert.deepEqual(listed.vms.map((vm) => [vm.project, vm.state, vm.worker.state]).sort(), [["blog", "running", "running"], ["shop", "running", "running"]]); + const blogEndpoint = JSON.parse(readFileSync(endpointFile(home, "blog"), "utf8")); + assert.notEqual(blogEndpoint.port, shopEndpoint.port, "each project's VM owns its host port"); + assert.notEqual(blogEndpoint.secret, shopEndpoint.secret, "each project's VM has its own secret"); + assert.notEqual(blogEndpoint.vm.name, shopEndpoint.vm.name); + console.log(`two projects, two VMs: ok (${shop.volume}, ${blog.volume})`); + + // V8: each Store is on its own VM's disk and nowhere else, so a delete is refused + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + + const shopEnv = projectEnv(runtime, "shop", shop.volume); + const blogEnv = projectEnv(runtime, "blog", blog.volume); + assert.equal(shopEnv.LAYERFS_WORKER_ENDPOINT, endpointFile(home, "shop"), "the deployment names this project's worker"); + assert.equal(process.env.LAYERFS_WORKER_ENDPOINT, undefined, "the runtime hands its routes over in-process, not through its own environment"); + const shopRun = await agent("shop", shopEnv); + assert.equal(shopRun.code, 0, shopRun.text); + console.log(shopRun.text.trim()); + const left = /^left=(\S+)$/m.exec(shopRun.text)?.[1]; + assert.ok(left, `the shop agent left its Sandbox open for the restart: ${shopRun.text}`); + // #515 §4.7 V7: the Sandbox root is the bundle's image, unpacked once on the + // VM's ext4 disk (H38's require_overlay_upper), under its config digest + const unpacked = await disk("shop"); + assert.equal(unpacked.local, "ext4", `LAYERFS_LOCAL_DIR is on the VM's Store disk: ${JSON.stringify(unpacked)}`); + assert.match(unpacked.digest, /^[0-9a-f]{64}$/, `one image, under its digest: ${JSON.stringify(unpacked)}`); + assert.equal(unpacked.mode, "555", `an unpacked image is read-only: ${JSON.stringify(unpacked)}`); + assert.equal(unpacked.partial, "0", `no half-unpacked image is left behind: ${JSON.stringify(unpacked)}`); + console.log(`the image ${unpacked.digest.slice(0, 12)} is unpacked on the VM's ${unpacked.local} disk: ${unpacked.mode}`); + + // the Sandbox's root really is that image, and one more Sandbox reuses the unpack + const third = await agent("shop", shopEnv, "third"); + assert.equal(third.code, 0, third.text); + console.log(third.text.trim()); + const reused = await disk("shop"); + assert.deepEqual([reused.digest, reused.stat], [unpacked.digest, unpacked.stat], `the same digest was unpacked once: ${JSON.stringify([unpacked, reused])}`); + + const blogRun = await agent("blog", blogEnv, "blog"); + assert.equal(blogRun.code, 0, blogRun.text); + console.log(blogRun.text.trim()); + const otherDisk = await disk("blog"); + assert.equal(otherDisk.local, "ext4", `the other project's VM has its own ext4 disk: ${JSON.stringify(otherDisk)}`); + assert.equal(otherDisk.digest, unpacked.digest, "each VM unpacks the bundle's one image"); + + // a stopped VM is started again, and its worker comes back on the same port + assert.equal((await runtime.vm.stop("shop")).state, "stopped"); + assert.equal((await runtime.vm.status("shop")).state, "stopped"); + const again = await runtime.worker.start("shop"); + assert.equal(again.worker.state, "running", JSON.stringify(again)); + assert.equal(again.vm.worker.port, shopEndpoint.port, "the same VM keeps its published port"); + // the reboot ended the Sandbox's init process but not its image: the cache the + // VM kept is the same directory, and the record it left is reclaimed by the + // resume phase, which then starts a Sandbox on that same unpack + assert.deepEqual((await disk("shop")).stat, unpacked.stat, "the image cache survived the VM restart unchanged"); + // the Sandbox the reboot ended is still a record in the home, not a running Sandbox + const record = join(home, "shop", "sandboxes", left, "sandbox.record"); + assert.ok(existsSync(record), `the Sandbox record survived the VM restart: ${record}`); + console.log("stopped and restarted the project's VM: ok"); + + // the restarted worker takes a new secret, and the project's Commit is still readable + const restarted = await runtime.worker.restart("shop", { confirm: "restart" }); + assert.equal(restarted.worker.state, "running"); + const resumed = await agent("shop", shopEnv, "resume", { AGENT_LEFTOVER: left }); + assert.equal(resumed.code, 0, resumed.text); + console.log(resumed.text.trim()); + assert.ok(!existsSync(record), "the record the previous boot left was reclaimed by the resumed agent"); + + // removing one project takes its own VM and Store disk, and leaves the other alone + const removed = await runtime.project.remove("blog", { confirm: "blog", stopSandboxes: true }); + assert.equal(removed.kind, "Removed"); + assert.equal((await runtime.vm.status("blog")).state, "missing"); + const other = await runtime.vm.status("shop"); + assert.deepEqual([other.project, other.state, other.worker.state], ["shop", "running", "running"], "the other project's VM is untouched"); + const shopAgain = await agent("shop", shopEnv, "resume"); + assert.equal(shopAgain.code, 0, shopAgain.text); + console.log("removed one project, the other still runs: ok"); +} finally { + // whatever the assertions decided: every project still here keeps its VM and Store + // disk to itself, so each one's removal takes exactly those + for (const project of (await runtime?.project.list().catch(() => [])) ?? []) { + const volume = project.volume ?? `${project.name}-store`; + await runtime?.project.remove(project.name, { confirm: project.name, volume, stopSandboxes: true }).catch((error) => console.error(`remove ${project.name}: ${error.message}`)); + await runtime?.vm.delete(project.name, { confirm: "delete", discardStore: true }).catch((error) => console.error(`vm delete ${project.name}: ${error.message}`)); + } + await runtime?.close(); + rmSync(work, { recursive: true, force: true }); +} +console.log(`vm real: ok in ${((Date.now() - started) / 1000).toFixed(1)} s`); diff --git a/api/runtime/test/vm_placement.mjs b/api/runtime/test/vm_placement.mjs new file mode 100644 index 0000000000..641ce82332 --- /dev/null +++ b/api/runtime/test/vm_placement.mjs @@ -0,0 +1,501 @@ +// @layerfs/runtime's `vm:` placement (#515 §6, V9) with a stand-in +// provider CLI and the real worker: no microVM, no Store, no daemon. One Linux +// VM per Store (#515 V1): every case names a project, and its VM, Store disk, +// host port, bundle copy and worker secret are that project's alone. The +// fixture keeps one VM record per project, maps each guest's absolute paths +// under a temp folder, and starts the adapter's bundle worker so step ④'s +// readiness is real. +// node api/runtime/test/vm_placement.mjs settings|flow|two|bundle|delete|hosts|apple-container +import assert from "node:assert/strict"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const FAKE = fileURLToPath(new URL("../../agent/test/fixture/fake-native.cjs", import.meta.url)); +const SOURCE = fileURLToPath(new URL("../src", import.meta.url)); +const MSB = fileURLToPath(new URL("fixture/fake-msb.cjs", import.meta.url)); +const CONTAINER = fileURLToPath(new URL("fixture/fake-container.cjs", import.meta.url)); +const SSH = fileURLToPath(new URL("fixture/fake-ssh.cjs", import.meta.url)); +process.env.LAYERFS_NATIVE = FAKE; +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const { resolveSettings, missing, callHome, GUEST } = await import("../src/settings.js"); +const { deploymentEnv } = await import("../src/clients.js"); +const { endpointFile } = await import("../src/platform/shared.js"); + +const selected = process.argv[2]; +const work = mkdtempSync(join(tmpdir(), "layerfs-vm-")); +const home = join(work, "home"), bundle = join(work, "bundle"), prefix = "lfs-vm-test-"; +/** A project's VM and Store disk names: one VM per Store. */ +const vmName = (project) => `${prefix}${project}-vm`; +const storeName = (project) => `${prefix}${project}-vm-store`; + +/** A stand-in provider CLI on PATH: `name` shells out to the fixture with this state folder. */ +function standIn(name, fixture, state) { + const bin = join(work, "bin"); + mkdirSync(bin, { recursive: true }); + const file = join(bin, name); + writeFileSync(file, `#!/bin/sh\nexec "${process.execPath}" "${fixture}" "$@"\n`); + chmodSync(file, 0o755); + const saved = process.env.PATH; + process.env.PATH = `${bin}:${saved}`; + const envKey = name === "msb" ? "FAKE_MSB" : name === "container" ? "FAKE_CONTAINER" : "FAKE_SSH"; + process.env[`${envKey}_STATE`] = state; + process.env[`${envKey}_NATIVE`] = FAKE; + if (name === "ssh") process.env.LAYERFS_SSH_STATE_DIR = join(state, "sessions"); + return () => { + process.env.PATH = saved; + if (name === "ssh") delete process.env.LAYERFS_SSH_STATE_DIR; + }; +} + +/** A sealed Linux bundle with the adapter's worker and the real SDK packages it imports. + * `image` adds a Sandbox image archive at `images/.tar` (H38's convention, V7). */ +function makeBundle(tag = "build-one", image = null) { + rmSync(bundle, { recursive: true, force: true }); + const api = join(bundle, "api"); + cpSync(join(SOURCE, "worker"), join(api, "runtime/src/worker"), { recursive: true }); + cpSync(join(SOURCE, "../../node_modules/@layerfs"), join(api, "node_modules/@layerfs"), { recursive: true, dereference: true }); + if (image) { + mkdirSync(join(bundle, "images"), { recursive: true }); + writeFileSync(join(bundle, "images", `${image}.tar`), "an uncompressed docker save archive\n"); + } + writeFileSync(join(bundle, "manifest.json"), JSON.stringify({ version: 1, status: "sealed", platform: "linux", build_id: tag, files: {} })); + return bundle; +} + +const calls = (state) => (existsSync(join(state, "calls.jsonl")) ? readFileSync(join(state, "calls.jsonl"), "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line).argv) : []); + +/** Every await that could wait on a provider has a ceiling here, so a stuck + * call fails this case with a name instead of hanging the harness. */ +const deadline = (ms, what, promise) => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not finish within ${ms / 1000} s`)), ms).unref()), + ]); + +// The whole case has a ceiling too: an unref'd watchdog that exits non-zero +// rather than letting CI or a shell wait forever. +setTimeout(() => { + process.stderr.write(`vm_placement/${selected}: FAIL, the case did not finish in 300 s\n`); + process.exit(1); +}, 300_000).unref(); +/** A guest path inside one project's VM. */ +const guestPath = (state, vm, path) => join(state, "guest", vm, path.replace(/^\//, "")); +/** The project is gone from this home: its entry and the runtime's record of it. */ +const forget = (name) => { + rmSync(join(home, name), { recursive: true, force: true }); + rmSync(join(home, ".runtime", "projects", `${name}.json`), { force: true }); +}; + +/** A project of the home: its entry, and the runtime's record of its volume. */ +const project = (name) => { + mkdirSync(join(home, name), { recursive: true }); + writeFileSync(join(home, name, "project.record"), "x"); + mkdirSync(join(home, ".runtime", "projects"), { recursive: true }); + writeFileSync(join(home, ".runtime", "projects", `${name}.json`), JSON.stringify({ name, volume: `lfs-${name}-store-1`, created: 1 })); +}; +const endpointAt = (name) => JSON.parse(readFileSync(endpointFile(home, name), "utf8")); + +try { + if (selected === "settings") { + const local = resolveSettings({ home, platform: "linux", image: `sha256:${"a".repeat(64)}` }); + assert.deepEqual([local.placement.kind, local.placement.name, local.vm ?? null], ["local", "local", null]); + assert.deepEqual(missing(local), ["daemon"], "a local Linux home needs its daemon only"); + // LAYERFS_PLACEMENT is the V9 name; the settings object wins when both are set + process.env.LAYERFS_PLACEMENT = "vm:microsandbox"; + const vm = resolveSettings({ home, platform: "linux", bundle: "/b", containerPrefix: prefix }); + assert.deepEqual([vm.placement.kind, vm.placement.provider, vm.vm.provider, vm.vm.image], ["vm", "microsandbox", "microsandbox", "node:22-trixie-slim"]); + assert.equal(vm.daemon, GUEST.daemon, "the daemon is the guest's, beside its helper"); + assert.deepEqual([vm.vm.cpus, vm.vm.memory, vm.vm.storeSize], [2, "2G", "20G"]); + assert.equal(callHome(vm), GUEST.home, "LayerFS calls see the VM's home"); + assert.deepEqual(missing(vm), [], "a VM home needs no runtime image and no daemon setting"); + // sizes and cpus come from settings.vm, else LAYERFS_VM_* + process.env.LAYERFS_VM_CPUS = "3"; + process.env.LAYERFS_VM_STORE_SIZE = "64G"; + const sized = resolveSettings({ home, platform: "linux", bundle: "/b" }); + assert.deepEqual([sized.vm.cpus, sized.vm.storeSize], [3, "64G"]); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "vm:nope" }), { code: "Unsupported", message: /vm:microsandbox or vm:apple-container/ }); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "nope" }), { code: "InvalidInput", message: /local.*vm:/ }); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "vm:microsandbox", vm: { memory: "8" } }), { code: "InvalidInput", message: /memory/ }); + // vm:ssh settings + const sshSettings = resolveSettings({ + home, platform: "linux", bundle: "/b", placement: "vm:ssh", + vm: { host: "10.0.0.1", port: 2222, identityFile: "/path/to/key.pem", storePath: "/mnt/store" } + }); + assert.deepEqual([sshSettings.placement.kind, sshSettings.placement.provider, sshSettings.vm.provider], ["vm", "ssh", "ssh"]); + assert.deepEqual( + [sshSettings.vm.ssh.host, sshSettings.vm.ssh.port, sshSettings.vm.ssh.identityFile, sshSettings.vm.ssh.storePath, sshSettings.vm.ssh.strictHostKeyChecking], + ["10.0.0.1", 2222, "/path/to/key.pem", "/mnt/store", true] + ); + assert.deepEqual(missing(sshSettings), []); + // missing host in vm:ssh + const sshNoHost = resolveSettings({ home, platform: "linux", bundle: "/b", placement: "vm:ssh" }); + assert.deepEqual(missing(sshNoHost), ["host"]); + // password rejected + assert.throws( + () => resolveSettings({ home, platform: "linux", bundle: "/b", placement: "vm:ssh", vm: { host: "10.0.0.1", password: "secret" } }), + { code: "InvalidInput", message: /password/ } + ); + delete process.env.LAYERFS_PLACEMENT; + delete process.env.LAYERFS_VM_CPUS; + delete process.env.LAYERFS_VM_STORE_SIZE; + } else if (selected === "flow") { + const image = "a".repeat(64); + makeBundle("build-one", image); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + assert.equal(existsSync(endpointFile(home, "shop")), false); + // a project's VM is named after the project: there is no home-wide VM + await assert.rejects(runtime.worker.start(), { code: "InvalidInput", message: /project name/ }); + project("shop"); + const started = await runtime.worker.start("shop"); + assert.equal(started.project, "shop"); + assert.equal(started.worker.state, "running"); + assert.deepEqual([started.vm.state, started.vm.store], ["running", storeName("shop")]); + const endpoint = endpointAt("shop"); + assert.equal(endpoint.home, GUEST.home, "the endpoint names the VM's home"); + assert.equal(endpoint.vm.name, vmName("shop")); + assert.equal(endpoint.vm.project, "shop"); + assert.deepEqual(Object.keys(endpoint.vm).sort(), ["name", "project", "provider"]); + assert.equal(process.env.LAYERFS_WORKER_ENDPOINT, undefined, "the endpoint is handed over in-process"); + + const first = calls(state); + const create = first.find((argv) => argv[0] === "create"); + assert.ok(create, "step ① created the project's VM"); + assert.ok(create.includes("--name") && create[create.indexOf("--name") + 1] === vmName("shop")); + assert.ok(create.includes("--mount-named"), "the Store is a named disk volume of its own"); + assert.equal(create[create.indexOf("--mount-named") + 1], `${storeName("shop")}:/layerfs/store:kind=disk,size=20G`); + assert.equal(create[create.indexOf("-v") + 1], `${home}:/layerfs/home`); + assert.equal(create[create.indexOf("-p") + 1], `127.0.0.1:${endpoint.port}:7070`); + assert.equal(create.at(-1), "node:22-trixie-slim"); + assert.ok(first.some((argv) => argv[0] === "copy" && argv[2] === `${vmName("shop")}:/layerfs/bundle`), "step ② copied the bundle into that VM"); + assert.ok(first.some((argv) => argv[0] === "exec" && /max_user_namespaces/.test(argv.join(" "))), "step ③ blocked user namespaces"); + assert.ok(first.some((argv) => argv[0] === "exec" && /nohup node/.test(argv.join(" "))), "step ③ started the worker"); + assert.equal(readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8").includes(endpoint.secret), true, "the secret reached the VM in a file, not in a command line"); + assert.ok(!JSON.stringify(first).includes(endpoint.secret), "no call carried the secret in its arguments"); + + // the Sandbox image the bundle carries (V7, H38's convention): this VM's own copy, + // named for the worker as a guest path, and the deployment's image digest + const workerEnv = readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"); + assert.match(workerEnv, new RegExp(`^LAYERFS_IMAGE=sha256:${image}$`, "m")); + assert.match(workerEnv, new RegExp(`^LAYERFS_IMAGE_ARCHIVE=/layerfs/bundle/images/${image}\\.tar$`, "m")); + assert.equal(existsSync(guestPath(state, vmName("shop"), `/layerfs/bundle/images/${image}.tar`)), true, "the image archive came in with the bundle"); + + // the client side: this project's deployment, and this project's endpoint + const env = deploymentEnv(runtime.settings, runtime.platform, { name: "shop" }); + assert.equal(env.WORKER_ENDPOINT, endpointFile(home, "shop"), "agents an integration launches reach this project's VM"); + assert.equal(env.STORE_PATH, `${GUEST.store}/global/store.sqlite`, "the Store disk is the Store's directory"); + assert.equal(env.SANDBOX_BACKEND, "namespaces"); + assert.equal(env.IMAGE, `sha256:${image}`, "a bundle with an image gives the deployment its digest"); + assert.equal(env.IMAGE_ARCHIVE, `${GUEST.bundle}/images/${image}.tar`, "and the archive in this VM's own copy of the bundle"); + assert.equal(env.LOCAL_DIR, `${GUEST.store}/local`, "Sandbox directories are on the Store disk, never the VM's own root"); + assert.match(readFileSync(join(state, "calls.jsonl"), "utf8"), /\/layerfs\/store\/local/, "the guest's Sandbox directory was made on the Store disk"); + + // the same project again: the VM and its bundle are reused, and only the VM is read + const counted = () => calls(state).filter((argv) => argv[0] === "create" || argv[0] === "copy" || argv[0] === "exec").length; + const before = counted(); + await runtime.worker.start("shop"); + assert.equal(counted(), before, "a running worker with its bundle is reused"); + + // a stopped VM is started, not created + assert.deepEqual(await runtime.vm.stop("shop"), { project: "shop", name: vmName("shop"), state: "stopped" }); + assert.equal(existsSync(endpointFile(home, "shop")), false, "a stopped VM's endpoint is gone"); + const restarted = await runtime.worker.start("shop"); + assert.equal(restarted.worker.state, "running"); + assert.equal(calls(state).filter((argv) => argv[0] === "create").length, 1, "the VM was started, not created again"); + + // status: host Docker is not this home's, and each project's VM is listed + const status = await runtime.status(); + assert.deepEqual([status.placement, status.docker.level], ["vm:microsandbox", "not-used"]); + assert.equal(status.worker, null, "no single worker serves the home"); + assert.deepEqual(status.vm.vms.map((vm) => [vm.project, vm.state, vm.worker.state]), [["shop", "running", "running"]]); + assert.ok(status.disk.volumes > 0, "the project's Store disk has free bytes"); + assert.equal((await runtime.vm.status("shop")).name, vmName("shop")); + assert.deepEqual((await runtime.vm.status()).vms.map((vm) => vm.project), ["shop"]); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "two") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + project("shop"); + project("blog"); + await runtime.worker.start("shop"); + await runtime.worker.start("blog"); + // one VM, one Store disk and one endpoint per project, and they all differ + const shop = endpointAt("shop"), blog = endpointAt("blog"); + assert.notEqual(shop.port, blog.port, "each VM owns its host port"); + assert.notEqual(shop.secret, blog.secret, "each VM has its own worker secret"); + assert.deepEqual([shop.vm.name, blog.vm.name], [vmName("shop"), vmName("blog")]); + const created = calls(state).filter((argv) => argv[0] === "create").map((argv) => argv[argv.indexOf("--mount-named") + 1]); + assert.deepEqual(created.sort(), [`${storeName("blog")}:/layerfs/store:kind=disk,size=20G`, `${storeName("shop")}:/layerfs/store:kind=disk,size=20G`], "each project's Store is on its own disk"); + assert.equal(calls(state).filter((argv) => argv[0] === "copy" && argv[2] === `${vmName("blog")}:/layerfs/bundle`).length, 1, "the bundle is copied into each VM"); + // neither VM's Store disk holds the other's Store: they are different disks + const shopStore = guestPath(state, vmName("shop"), "/layerfs/store/global/store.sqlite"); + mkdirSync(join(shopStore, ".."), { recursive: true }); + writeFileSync(shopStore, "shop"); + assert.equal(existsSync(guestPath(state, vmName("blog"), "/layerfs/store/global/store.sqlite")), false, "the other project's VM has its own Store disk"); + // and one VM's secrets, ports and bundle copy are its own + assert.notDeepEqual(readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"), readFileSync(guestPath(state, vmName("blog"), "/layerfs/state/worker.env"), "utf8")); + const listed = await runtime.vm.status(); + assert.deepEqual(listed.vms.map((vm) => [vm.project, vm.state, vm.worker.port]).sort(), [["blog", "running", blog.port], ["shop", "running", shop.port]]); + // the hook project.remove uses takes that project's VM and disk, and nothing else + // (the whole call, a Sandbox closing included, is vm_real.mjs's, with a real VM) + await runtime.platform.removeStore(runtime, { name: "blog", volume: "lfs-blog-store-1" }); + assert.equal(existsSync(join(state, "vms", `${vmName("blog")}.json`)), false, "the project's VM is gone"); + assert.ok(calls(state).some((argv) => argv[0] === "volume" && argv[1] === "remove" && argv[2] === storeName("blog"))); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), true, "the other project's VM is untouched"); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)) && endpointAt("shop").vm.name, vmName("shop")); + assert.equal((await runtime.worker.start("shop")).worker.state, "running", "and the other project's worker still answers"); + // V8 still holds for the project that is left + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + forget("shop"); + assert.deepEqual(await runtime.vm.delete("shop", { confirm: "delete", discardStore: true }), { project: "shop", name: vmName("shop"), state: "deleted", store: "discarded" }); + assert.ok(calls(state).some((argv) => argv[0] === "volume" && argv[1] === "remove" && argv[2] === storeName("shop"))); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "bundle") { + makeBundle("build-one"); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + await runtime.worker.start("shop"); + const copies = (vm) => calls(state).filter((argv) => argv[0] === "copy" && argv[2] === `${vm}:/layerfs/bundle`).length; + assert.equal(copies(vmName("shop")), 1); + // the same bundle hash: step ② is not repeated + await runtime.worker.start("shop"); + assert.equal(copies(vmName("shop")), 1); + // a second project's VM gets its own copy + await runtime.worker.start("blog"); + assert.equal(copies(vmName("shop")), 1); + assert.equal(copies(vmName("blog")), 1); + // a changed bundle is copied again into the VM that restarts, and the worker reports it + makeBundle("build-two"); + const restarted = await runtime.worker.start("shop"); + assert.equal(copies(vmName("shop")), 2, "a changed bundle hash is copied once more"); + assert.equal(restarted.worker.build, "build-two"); + assert.equal(readFileSync(guestPath(state, vmName("shop"), "/layerfs/bundle/.sealed"), "utf8").trim().length, 64); + // a bundle without a Sandbox image sets neither: the backend needs none there + const plain = readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"); + assert.ok(!/LAYERFS_IMAGE/.test(plain), "no image is named when the bundle carries none"); + assert.equal(copies(vmName("blog")), 1, "the other project's VM keeps its copy until it restarts"); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "delete") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + await runtime.worker.start("shop"); + // V8: a Store that lives only on the project's VM disk is never destroyed by a delete + project("shop"); + await assert.rejects(runtime.vm.delete("shop"), { code: "InvalidInput", message: /confirm/ }); + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), true, "the VM is untouched by the refusal"); + // with no project left, the VM record goes and its Store disk stays unless asked + forget("shop"); + assert.deepEqual(await runtime.vm.delete("shop", { confirm: "delete" }), { project: "shop", name: vmName("shop"), state: "deleted", store: "kept" }); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), false); + assert.equal(calls(state).some((argv) => argv[0] === "volume"), false, "the Store disk is kept by default"); + await runtime.vm.delete("shop", { confirm: "delete", discardStore: true }); + // a store whose VM is gone answers "missing" rather than inventing one + assert.equal((await runtime.vm.delete("shop", { confirm: "delete" })).state, "missing"); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "hosts") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + project("shop"); + await runtime.worker.start("shop"); + // host Docker describes another machine: the Docker-shaped helpers refuse by name + await assert.rejects(runtime.sandbox.list(), { code: "Unsupported", message: /runtime\.sandbox\.list reads host Docker.*project\.list_sandboxes/ }); + await assert.rejects(runtime.sandbox.list_leftovers(), { code: "Unsupported", message: /Linux namespaces inside the VM/ }); + await assert.rejects(runtime.volume.list(), { code: "Unsupported", message: /directories on the VM's disk/ }); + assert.deepEqual(await runtime.sandbox.sweep(), { reclaimed: {}, failed: [], host: false }); + // status lists a project's VM even before it was started + project("late"); + assert.deepEqual((await runtime.vm.status()).vms.map((vm) => [vm.project, vm.state]), [["late", "missing"], ["shop", "running"]]); + // a local home has no vm group at all + const local = await LayerFSRuntime.open({ home: join(work, "local"), image: `sha256:${"a".repeat(64)}`, daemon: "/bin/daemon", platform: "linux", sweep: false }); + assert.equal(local.vm, undefined); + await local.close(); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "apple-container") { + makeBundle(); + const state = join(work, "container"); + mkdirSync(state); + const restore = standIn("container", CONTAINER, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:apple-container", containerPrefix: prefix, platform: "macos", sweep: false }); + try { + const started = await runtime.worker.start("shop"); + assert.equal(started.vm.state, "running"); + const first = calls(state); + // the Store volume is made first, at the configured size, and mounted by name + const volume = first.find((argv) => argv[0] === "volume" && argv[1] === "create"); + assert.ok(volume, "the Store volume is made before the VM"); + assert.equal(volume[volume.indexOf("-s") + 1], "20G"); + assert.equal(volume.at(-1), storeName("shop")); + const run = first.find((argv) => argv[0] === "run"); + assert.ok(run, "the VM was created with `container run`"); + const capabilities = run.flatMap((value, index) => (value === "--cap-add" ? [run[index + 1]] : [])); + assert.deepEqual(capabilities, ["SYS_ADMIN", "SYS_RESOURCE", "NET_ADMIN"], "#512's capabilities for FUSE, cgroups and loopback"); + assert.ok(run.includes("--publish")); + assert.equal(run[run.indexOf("--mount") + 1], `type=volume,source=${storeName("shop")},target=/layerfs/store`); + // the guest's /proc/sys is read-only there: the block remounts it first + const block = first.find((argv) => argv[0] === "exec" && /max_user_namespaces/.test(argv.join(" "))); + assert.match(block.join(" "), /remount,rw \/proc\/sys/); + assert.ok(first.some((argv) => argv[0] === "exec" && /nohup node/.test(argv.join(" ")))); + // the verified 1.6.1 inspect shape: the adapter reads this VM's state, store and home + const entry = await runtime.vm.status("shop"); + assert.deepEqual([entry.state, entry.store, entry.home, entry.worker.state], ["running", storeName("shop"), home, "running"]); + // copying the bundle into an Apple Container VM is `container copy HOST NAME:DEST` + assert.ok(first.some((argv) => argv[0] === "copy" && argv[2] === `${vmName("shop")}:/layerfs/bundle`)); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "ssh") { + makeBundle(); + const state = join(work, "ssh"); + mkdirSync(state); + const restore = standIn("ssh", SSH, state); + const runtime = await LayerFSRuntime.open({ + home, bundle, placement: "vm:ssh", containerPrefix: prefix, platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + const started = await deadline(60_000, "worker.start", runtime.worker.start("shop")); + assert.equal(started.vm.state, "running"); + assert.equal(started.worker.state, "running"); + const first = calls(state); + assert.ok(first.some((argv) => argv.includes("uname")), "the host's kernel is read before anything else"); + assert.ok(first.some((argv) => argv.includes("findmnt")), "the Store path's filesystem is checked"); + assert.ok(first.some((argv) => argv.includes("-N") && argv.includes("-L")), "the private stream is an ssh -L forward"); + assert.ok(first.some((argv) => argv.includes("StrictHostKeyChecking=yes")), "host key checking stays on"); + assert.ok(!first.some((argv) => argv.includes("StrictHostKeyChecking=no")), "never turned off behind the user's back"); + assert.ok(first.some((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))), "the bundle arrives as one tar stream"); + const entry = await deadline(30_000, "vm.status", runtime.vm.status("shop")); + assert.deepEqual([entry.state, entry.worker.state], ["running", "running"]); + // the endpoint the worker was written at is this host's forwarded port + const endpoint = endpointAt("shop"); + assert.equal(endpoint.vm.provider, "ssh"); + assert.ok(endpoint.port > 1024); + // a second start is served by the same forward: no second copy, no second forward + await deadline(30_000, "worker.start again", runtime.worker.start("shop")); + assert.equal(calls(state).filter((argv) => argv.includes("-N")).length, 1, "one forward per VM"); + // stop ends the forward; the record stays and start re-opens it + assert.equal((await deadline(30_000, "vm.stop", runtime.vm.stop("shop"))).state, "stopped"); + assert.equal((await deadline(30_000, "status after stop", runtime.vm.status("shop"))).state, "stopped"); + const again = await deadline(60_000, "worker.start after stop", runtime.worker.start("shop")); + assert.deepEqual([again.vm.state, again.worker.state], ["running", "running"]); + } finally { + await runtime.close(); + restore(); + restoreWorker(work); + } + } else if (selected === "cleanup") { + // H39v's two findings: a provider create that fails must leave nothing + // behind — no Store volume for Apple Container (the CLI makes one before it + // fails on a busy port), and no hidden sandbox directory for Microsandbox. + const containerState = join(work, "container"); + mkdirSync(containerState); + const restoreContainer = standIn("container", CONTAINER, containerState); + try { + const container = (await import("../src/platform/vm/apple-container.js")).default; + await container.ensureStore("lfs-vm-test-clean-store", "20G"); + process.env.FAKE_CONTAINER_FAIL_RUN = "port"; + await assert.rejects(container.create({ + name: "lfs-vm-test-clean", image: "node:22-trixie-slim", cpus: 2, memory: "2G", + storeName: "lfs-vm-test-clean-store", storeSize: "20G", home, hostPort: 18080, workerPort: 7070, + }), /port is already allocated/); + assert.equal(existsSync(join(containerState, "volumes", "lfs-vm-test-clean-store.json")), false, "the Store volume this create made is gone"); + assert.equal(existsSync(join(containerState, "vms", "lfs-vm-test-clean.json")), false, "and no VM record"); + } finally { + delete process.env.FAKE_CONTAINER_FAIL_RUN; + restoreContainer(); + } + const msbState = join(work, "msb"); + mkdirSync(msbState); + const restoreMsb = standIn("msb", MSB, msbState); + const hidden = join(tmpdir(), ".microsandbox-clean-check"); + try { + const msb = (await import("../src/platform/vm/microsandbox.js")).default; + const name = `lfs-vm-test-hidden-${process.pid}`; + const dir = join(process.env.HOME ?? join(tmpdir(), "home"), ".microsandbox", "sandboxes", name); + process.env.FAKE_MSB_FAIL_CREATE = "mount"; + await assert.rejects(msb.create({ + name, image: "node:22-trixie-slim", cpus: 2, memory: "2G", storeName: "lfs-vm-test-hidden-store", + storeSize: "20G", home, hostPort: 18081, workerPort: 7070, + }), /host path for one of the mounts/); + assert.equal(existsSync(dir), false, `the hidden sandbox directory ${dir} the failed create made is gone`); + } finally { + delete process.env.FAKE_MSB_FAIL_CREATE; + restoreMsb(); + rmSync(hidden, { recursive: true, force: true }); + } + } else { + throw new Error("select settings, flow, two, bundle, delete, hosts, apple-container, ssh or cleanup"); + } +} finally { + rmSync(work, { recursive: true, force: true }); +} +console.log(`PASS vm_placement/${selected}`); +// The run is over: exit explicitly so a stand-in child that outlived its case +// cannot hold this process open. `restoreWorker` already ended them. +process.exit(0); + +/** The stand-in's worker outlives its CLI: end it from the pid the fixture wrote. */ +function restoreWorker(folder) { + for (const provider of ["msb", "container", "ssh"]) { + const guests = join(folder, provider, "guest"); + if (existsSync(guests)) { + for (const vm of readdirSync(guests)) { + for (const file of [join(guests, vm, "layerfs/state/worker.pid"), join(guests, "layerfs/state/worker.pid")]) { + if (!existsSync(file)) continue; + try { process.kill(Number(readFileSync(file, "utf8").trim())); } catch { /* already gone */ } + } + } + } + const sessions = join(folder, provider, "sessions"); + if (existsSync(sessions)) { + for (const file of readdirSync(sessions)) { + try { + const session = JSON.parse(readFileSync(join(sessions, file), "utf8")); + if (session.forwardPid) process.kill(session.forwardPid); + } catch {} + } + } + } +} diff --git a/engine/crates/layerfs-api/PROVIDERS.md b/engine/crates/layerfs-api/PROVIDERS.md new file mode 100644 index 0000000000..16bd8b3cf9 --- /dev/null +++ b/engine/crates/layerfs-api/PROVIDERS.md @@ -0,0 +1,13 @@ +# Sandbox runtimes in the SDK + +Status: Current general guide. Native macOS is implemented and host-verified; Linux native is implemented, pending Linux host qualification; the Docker runtime is the default. `layerfs-sandbox` selects between them with the concrete `Runtime` enum; there are no provider traits. + +`layerfs-sandbox::backend` exposes `Runtime` (a deployment's choice: `Docker`, `MacosNative`, `LinuxNative`, `Windows`) and `Sandbox` (the launched owner of one Sandbox). `layerfs_sandbox::{Runtime, Sandbox}` re-export both; `layerfs_sandbox::provider` keeps the inputs and records they share. `ExecRequest` lives in the shared types layer; its Docker re-export remains compatible. `SandboxId` aliases the existing `ContainerId`, preserving encoded identities and records. The runtime layer has no daemon control dependency. + +`Runtime::MacosNative(..)` and `Runtime::LinuxNative(..)` wrap `NativeMacosProvider::new(root)` and `NativeLinuxProvider::new(root)`, the two aliases of the same native implementation whose instances share the existing local runtime owner. Each refuses the wrong platform before root access. macOS uses Darwin process facts, the `mount` table and `umount -f`; Linux uses the retained #402 `/proc` PID/start/boot checks, process groups and lazy `MNT_DETACH`. Linux requires a root daemon and a root directory traversable by command users; ordinary commands keep #402's nonroot identity (Node selects 501:20) and fixed container PATH. macOS keeps the caller identity, inherited PATH and loopback NFS. The provider explicitly selects `LAYERFS_KERNEL=fuse` on Linux and `nfs` on macOS and rejects conflicting launch overrides. Launch preserves the original single-attempt process lifecycle, native identity, inherited PATH, NFS environment and partial owner on failure. `inspect` reports exact process liveness; unknown exit/storage facts stay absent. `Runtime::Windows` refuses every operation with `RuntimeError::Unsupported` before any effect; no Windows host runtime exists. Records remain the Node binding's responsibility; runtimes neither scan directories nor infer project ownership. B8 list aggregation and doctor remain future work. + +`SandboxApi::with_runtime(Arc)` and `create_provider_file(ProviderCreate, store, manifest)` (or `create(ProviderCreate, project)`) launch through the deployment's runtime and return `ProviderSandbox`, whose `runtime` field is the `Sandbox` handle. Both Docker (via `Runtime::Docker(DockerProvider)`) and native runtimes route through this single unified API path. `WorkspaceApi::exec` delegates ordinary execution through `sandbox.runtime.run(...)` with streaming stdout/stderr buffers, and `api.close` drains and removes instances via `sandbox.runtime.stop(1)` and `sandbox.runtime.delete()`. Before launch, API validates authentication identities, selected Store, command identity and local resource ownership: overlay and mount home must belong to the instance, Store must live outside it, and native listen uses dynamic loopback. API encodes daemon configuration, then uses the shared TCP/Noise/Hello and Store-install pipeline in `sandbox/connection.rs`. Failures retain their phase, original cause, runtime, control, observed status, installation acknowledgement and handshake work. No automatic retry or cleanup occurs in API. + +`layerfs-node` builds a `Runtime` from `deploy.sandbox` (`docker` by default, `native` on Linux and macOS), keeps the launched `Sandbox` in its Sandbox state, and reads records, image, scratch directory and background-process tracking through it. Legacy deprecated forwarders (`SandboxApi::new(Docker)`, `SandboxCreate`, `ManagedSandbox`, `from_managed`, `internal::split/exec/control_managed`) remain removed. + +The Store remains explicitly Disposable/WAL/OFF. Native macOS keeps #427's direct Store access without a worker; this refactor does not change persistence authorization or filesystem semantics. diff --git a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs index c2e8f73671..cf2fd9aeed 100644 --- a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs +++ b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs @@ -4,8 +4,8 @@ //! registered with the daemon, read the complete root and mutate it through //! plain syscalls. use layerfs_api::{ - InitRequest, ManagedSandbox, MountedWorkspace, OperationCause, ProjectApi, SandboxApi, - SandboxCreate, WorkspaceApi, + InitRequest, MountedWorkspace, OperationCause, ProjectApi, ProviderCreate, ProviderSandbox, + SandboxApi, WorkspaceApi, }; use layerfs_bridge::{ control::{ControlCode, DaemonPhase, MountPhase, Reply, Request}, @@ -14,7 +14,8 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ @@ -22,6 +23,7 @@ use std::{ fs, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, + sync::Arc, time::{Duration, Instant}, }; @@ -100,59 +102,46 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -type Running = ( - layerfs_sandbox::backend::docker::RuntimeOutput, - layerfs_sandbox::backend::docker::ExecHandle, -); enum Directory<'a> { Mounted(&'a MountedWorkspace), Kernel(&'a str), } -/// Mounted commands use SDK exec; outside-mount commands retain ordinary Sandbox execution. -fn start( - owner: &ManagedSandbox, +fn run_dir( + runtime: &layerfs_sandbox::Sandbox, + identity: layerfs_sandbox::CommandIdentity, script: &str, - directory: Directory<'_>, -) -> Result> { - let arguments = vec!["bash".into(), "-c".into(), script.into()]; - let created = match directory { - Directory::Mounted(mounted) => WorkspaceApi::exec(owner, mounted, arguments, vec![], false), - Directory::Kernel(directory) => layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - arguments, - vec![], - directory.into(), - false, - ), - } - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - Ok((output, handle)) -} -fn finish((output, handle): Running) -> Result, Box> { - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - output - .copy_to(&mut stdout, &mut stderr) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - if inspected.known_root_exit() != Some(0) { + directory: &str, +) -> Result, String> { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = runtime + .run( + &layerfs_sandbox::ExecRequest { + identity, + arguments: vec!["bash".into(), "-c".into(), script.into()], + environment: vec![], + directory: directory.into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("exec {f:?}"))?; + if exit.code != 0 { return Err(format!( - "command {inspected:?}: {} {}", + "command exit {exit:?}: {} {}", String::from_utf8_lossy(&stdout), String::from_utf8_lossy(&stderr) - ) - .into()); + )); } Ok(stdout) } -fn run( - owner: &ManagedSandbox, - script: &str, - directory: Directory<'_>, -) -> Result, Box> { - finish(start(owner, script, directory)?) +fn run(owner: &ProviderSandbox, script: &str, directory: Directory<'_>) -> Result, String> { + let dir = match directory { + Directory::Mounted(mounted) => &mounted.ready.directory, + Directory::Kernel(dir) => dir, + }; + run_dir(owner.runtime.as_ref(), owner.command_identity, script, dir) } fn main() -> Result<(), Box> { let began = Instant::now(); @@ -191,22 +180,30 @@ fn main() -> Result<(), Box> { fs::remove_dir_all(directory.join("source"))?; let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; - let api = SandboxApi::new(Docker::new(&args[1], Duration::from_secs(3))?); + let docker = Docker::new(&args[1], Duration::from_secs(3))?; + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); let mut owner = api .create( - SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "kernel-mount-test".into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup: setup(daemon.private, keys.public), - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }, &project, ) @@ -323,45 +320,54 @@ fn main() -> Result<(), Box> { // Reversible normal unmount: a running command's working directory keeps // the kernel busy. The filesystem neither kills nor waits for the command. - let holder = start( - &owner, - "sleep 1.2", - Directory::Kernel(&format!("{root}/node_modules")), - )?; - run( - &owner, - &format!( - "until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{root}/node_modules'; do sleep 0.02; done" - ), - Directory::Kernel("/"), - )?; - let busy = WorkspaceApi::new(&mut owner) - .unmount(mounted.bound.token, false) - .unwrap_err(); - match &busy.cause { - OperationCause::Remote(refusal) => { - assert_eq!( - (refusal.code, refusal.phase.as_str()), - (ControlCode::Busy, "unmount:kernel") - ); + let node_modules_dir = format!("{root}/node_modules"); + let identity = owner.command_identity; + let (runtime, control) = layerfs_api::internal::provider_parts(&mut owner); + std::thread::scope(|s| { + let holder = s.spawn(|| run_dir(runtime, identity, "sleep 1.2", &node_modules_dir)); + run_dir( + runtime, + identity, + &format!("until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{node_modules_dir}'; do sleep 0.02; done"), + "/", + ) + .unwrap(); + let busy = WorkspaceApi::from_control(control) + .unmount(mounted.bound.token, false) + .unwrap_err(); + match &busy.cause { + OperationCause::Remote(refusal) => { + assert_eq!( + (refusal.code, refusal.phase.as_str()), + (ControlCode::Busy, "unmount:kernel") + ); + } + other => panic!("{other:?}"), } - other => panic!("{other:?}"), - } - let status = WorkspaceApi::new(&mut owner).status(mounted.bound.token)?; - let kernel = status.kernel.expect("kernel status block"); - assert_eq!(kernel.phase, MountPhase::Ready); - assert_eq!( - kernel.ready.as_ref(), - Some(&mounted.ready), - "same connection" - ); - assert_eq!( - run(&owner, "cat .git/index", Directory::Mounted(&mounted))?, - FILES[1].1 - ); - finish(holder)?; - println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); - mark("busy"); + let status = WorkspaceApi::from_control(control) + .status(mounted.bound.token) + .unwrap(); + let kernel = status.kernel.expect("kernel status block"); + assert_eq!(kernel.phase, MountPhase::Ready); + assert_eq!( + kernel.ready.as_ref(), + Some(&mounted.ready), + "same connection" + ); + assert_eq!( + run_dir( + runtime, + identity, + "cat .git/index", + &mounted.ready.directory + ) + .unwrap(), + FILES[1].1 + ); + holder.join().unwrap().unwrap(); + println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); + mark("busy"); + }); // Terminal unmount: detach, loop joins, drain, revocation and Close. WorkspaceApi::new(&mut owner) diff --git a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs index f54c059ff8..f6958361ab 100644 --- a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs +++ b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs @@ -1,6 +1,6 @@ //! Positive-path host Init -> protected owned Sandbox -> direct shared Store control proof. //! Typed failure custody is covered separately by public adapter tests; this example reports failures in stderr. -use layerfs_api::{InitRequest, ManagedSandbox, ProjectApi, SandboxApi, SandboxCreate}; +use layerfs_api::{InitRequest, ProjectApi, ProviderCreate, ProviderSandbox, SandboxApi}; use layerfs_bridge::{ control::{DaemonPhase, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, @@ -8,13 +8,15 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ fs, io::Write, path::PathBuf, + sync::Arc, time::{Duration, Instant}, }; fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { @@ -39,25 +41,26 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { - let command="set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; - let created = layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - vec!["bash".into(), "-c".into(), command.into()], - vec![], - "/".into(), - false, - ) - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - let progress = output - .copy_to(&mut std::io::stdout(), &mut std::io::stderr()) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - println!("ORDINARY_RESULT {:?} {:?}", progress, inspected); - assert_eq!(inspected.known_root_exit(), Some(0)); +fn ordinary(owner: &ProviderSandbox) -> Result<(), Box> { + let command = "set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; + let mut stdout = std::io::stdout(); + let mut stderr = std::io::stderr(); + let exit = owner + .runtime + .run( + &layerfs_sandbox::ExecRequest { + identity: layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), command.into()], + environment: vec![], + directory: "/".into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("run {f:?}"))?; + println!("ORDINARY_RESULT {:?}", exit); + assert_eq!(exit.code, 0); Ok(()) } fn main() -> Result<(), Box> { @@ -96,23 +99,30 @@ fn main() -> Result<(), Box> { let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; let docker = Docker::new(&args[1], Duration::from_secs(3))?; - let api = SandboxApi::new(docker); - let make = |setup: DaemonSetup| SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); + let make = |name: &str, setup: DaemonSetup| ProviderCreate { + launch: LaunchRequest { + name: name.into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup, - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }; let mut first = api - .create(make(setup(daemon.private, keys.public)), &project) + .create(make("first", setup(daemon.private, keys.public)), &project) .map_err(|f| format!("first {f:?}"))?; println!( "OWNED_FIRST_CONTAINER {} phase={:?}", @@ -131,7 +141,7 @@ fn main() -> Result<(), Box> { let mut second_setup = setup(daemon.private, keys.public); second_setup.existing_store = Some(installed.manifest.clone()); let second = api - .create(make(second_setup), &project) + .create(make("second", second_setup), &project) .map_err(|f| format!("second {f:?}"))?; println!( "OWNED_SECOND_CONTAINER {} phase={:?}", @@ -161,7 +171,7 @@ fn main() -> Result<(), Box> { let mut reopened_setup = setup(daemon.private, keys.public); reopened_setup.existing_store = Some(installed.manifest.clone()); let mut reopened = api - .create(make(reopened_setup), &project) + .create(make("reopened", reopened_setup), &project) .map_err(|f| format!("reopen {f:?}"))?; println!( "OWNED_REOPEN_CONTAINER {} phase={:?}", diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 7bffa705c0..108b2f0251 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -98,59 +98,40 @@ pub fn into_connection(control: Control) -> Connection { /// Original authenticated control owner, without a reconnect or reset. #[cfg(unix)] -pub fn control(sandbox: &mut crate::ManagedSandbox) -> &mut Control { +pub fn control(sandbox: &mut crate::sandbox::ProviderSandbox) -> &mut Control { &mut sandbox.control } -/// The runtime, control owner and command identity of one Sandbox, taken -/// apart for an owner that serves several Workspaces at once: commands run on -/// the runtime while the control owner answers one request at a time. +/// Transfers provider runtime, original channel and admitted command identity. #[cfg(unix)] -pub fn split( - sandbox: crate::ManagedSandbox, +pub fn split_provider( + sandbox: crate::sandbox::ProviderSandbox, ) -> ( - layerfs_sandbox::backend::docker::Sandbox, + Box, Control, layerfs_sandbox::CommandIdentity, ) { (sandbox.runtime, sandbox.control, sandbox.command_identity) } +/// Borrows the same provider instance and original channel for concurrent execution. +#[cfg(unix)] +pub fn provider_parts( + sandbox: &mut crate::sandbox::ProviderSandbox, +) -> (&layerfs_sandbox::Sandbox, &mut Control) { + (sandbox.runtime.as_ref(), &mut sandbox.control) +} /// Original one-attempt startup used by existing-Store deployments and lifecycle proofs. #[cfg(unix)] pub fn start( api: &crate::SandboxApi, - request: crate::SandboxCreate, -) -> Result> { - crate::sandbox::create::create(&api.docker, request) + request: crate::sandbox::ProviderCreate, +) -> Result> { + api.start(request) } /// Existing bounded startup observation for control-channel qualification. #[cfg(unix)] pub fn sandbox_status( - sandbox: &mut crate::ManagedSandbox, + sandbox: &mut crate::sandbox::ProviderSandbox, wait_for_store: bool, ) -> Result> { sandbox.status(wait_for_store) } - -/// Disjoint original runtime/control borrows for concurrent Workspace orchestration. -#[cfg(unix)] -pub fn parts( - sandbox: &mut crate::ManagedSandbox, -) -> (&layerfs_sandbox::backend::docker::Sandbox, &mut Control) { - (&sandbox.runtime, &mut sandbox.control) -} - -/// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. -#[cfg(unix)] -pub fn exec( - sandbox: &layerfs_sandbox::backend::docker::Sandbox, - identity: layerfs_sandbox::CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result< - layerfs_sandbox::backend::docker::ExecCreated, - Box, -> { - crate::sandbox::exec::launch(sandbox, identity, arguments, environment, directory, stdin) -} diff --git a/engine/crates/layerfs-api/src/lib.rs b/engine/crates/layerfs-api/src/lib.rs index e652d32640..1641980cd8 100644 --- a/engine/crates/layerfs-api/src/lib.rs +++ b/engine/crates/layerfs-api/src/lib.rs @@ -27,7 +27,7 @@ pub use layerfs_project::store::{ pub use layerfs_project::{InitCounters, InitCounts, InitStep, ProjectError, SourceRefusal}; #[cfg(unix)] pub use sandbox::{ - connect_address, ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, + connect_address, ProviderCreate, ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, }; pub use sandbox::{BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi}; diff --git a/engine/crates/layerfs-api/src/sandbox/close.rs b/engine/crates/layerfs-api/src/sandbox/close.rs index e0a57ec08a..5f55414a9e 100644 --- a/engine/crates/layerfs-api/src/sandbox/close.rs +++ b/engine/crates/layerfs-api/src/sandbox/close.rs @@ -1,10 +1,10 @@ //! Explicit stop then delete of the exact acknowledged Sandbox runtime owner. -use super::{ManagedSandbox, SandboxApi, SandboxCause, SandboxFailure, SandboxPhase}; +use super::{ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase}; impl SandboxApi { /// Stops once, then deletes once after acknowledged stop. No Workspace drain is inferred. /// A failed attempt retains the original runtime, control and deployment inputs; no replay. - pub fn close(&self, mut sandbox: ManagedSandbox) -> Result<(), Box> { + pub fn close(&self, mut sandbox: ProviderSandbox) -> Result<(), Box> { if let Err(cause) = sandbox.runtime.stop(1) { return Err(failure( sandbox, @@ -23,14 +23,14 @@ impl SandboxApi { } } fn failure( - sandbox: ManagedSandbox, + sandbox: ProviderSandbox, phase: SandboxPhase, cause: SandboxCause, -) -> Box { - Box::new(SandboxFailure { +) -> Box { + Box::new(ProviderFailure { request: sandbox.request, phase, - sandbox: Some(sandbox.runtime), + runtime: Some(sandbox.runtime), control: Some(sandbox.control), handshake: sandbox.handshake, observed: Some(sandbox.startup), diff --git a/engine/crates/layerfs-api/src/sandbox/connection.rs b/engine/crates/layerfs-api/src/sandbox/connection.rs new file mode 100644 index 0000000000..ba3a75df2d --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/connection.rs @@ -0,0 +1,108 @@ +//! One shared TCP/Noise/Hello pipeline, retaining progress through every failure. +use super::create::{hello, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest}, + duplex::{initiate_observed, ChannelWork}, +}; +use std::{ + net::{SocketAddr, TcpStream}, + time::Duration, +}; + +pub(super) struct Startup { + pub phase: SandboxPhase, + pub control: Option, + pub handshake: ChannelWork, + pub observed: Option, +} +impl Startup { + pub fn new() -> Self { + Self { + phase: SandboxPhase::Validate, + control: None, + handshake: ChannelWork::default(), + observed: None, + } + } + pub fn connect( + &mut self, + endpoint: SocketAddr, + host: Option<&str>, + wait: Duration, + secret: &[u8; 32], + peer: [u8; 32], + existing: bool, + ) -> Result { + self.phase = SandboxPhase::Connect; + let address = super::create::connect_address(endpoint, host).map_err(SandboxCause::Io)?; + let stream = TcpStream::connect_timeout(&address, wait).map_err(SandboxCause::Io)?; + stream + .set_read_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + stream + .set_write_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + self.phase = SandboxPhase::Authenticate; + let start = initiate_observed(stream, secret, peer); + self.handshake = start.work; + self.control = Some(Control::new(start.result.map_err(SandboxCause::Channel)?)); + self.phase = SandboxPhase::Policy; + self.control + .as_ref() + .expect("authenticated") + .clear_io_waits() + .map_err(SandboxCause::Channel)?; + self.phase = SandboxPhase::Hello; + let status = hello( + self.control.as_mut().expect("authenticated"), + HelloRequest { + expected_instance: None, + wait_for_store: existing, + }, + ) + .map_err(SandboxCause::Control)?; + self.observed = Some(status.clone()); + let expected = if existing { + DaemonPhase::ControlReady + } else { + DaemonPhase::InstallPending + }; + if status.phase != expected { + return Err(SandboxCause::Setup(ControlError("original startup phase"))); + } + Ok(status) + } +} +/// Installation and its Ready observation are identical for every runtime. +pub(super) fn install( + control: &mut Control, + startup: &mut DaemonStatus, + installed: &mut Option, + phase: &mut SandboxPhase, + store: &layerfs_persistence::SealedStore, + manifest: &layerfs_bridge::provision::StoreManifest, +) -> Result<(), SandboxCause> { + *phase = SandboxPhase::Install; + *installed = Some( + control + .install_file(store, manifest) + .map_err(SandboxCause::Install)?, + ); + *phase = SandboxPhase::Hello; + let status = hello( + control, + HelloRequest { + expected_instance: Some(startup.instance), + wait_for_store: true, + }, + ) + .map_err(SandboxCause::Control)?; + *startup = status; + if startup.phase != DaemonPhase::ControlReady { + return Err(SandboxCause::Setup(ControlError( + "original installed startup phase", + ))); + } + Ok(()) +} diff --git a/engine/crates/layerfs-api/src/sandbox/create.rs b/engine/crates/layerfs-api/src/sandbox/create.rs index 2146783582..61837c6693 100644 --- a/engine/crates/layerfs-api/src/sandbox/create.rs +++ b/engine/crates/layerfs-api/src/sandbox/create.rs @@ -1,48 +1,10 @@ //! Ordinary Sandbox startup, original deployment inputs and failure custody. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::{ - control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest, Reply, Request}, - daemon_setup::DaemonSetup, - duplex::{initiate_observed, public_key, ChannelError, ChannelWork}, -}; -use layerfs_sandbox::backend::docker::{ - Docker, ListenerFailure, Sandbox, SandboxCreateFailure, SandboxRequest, UploadFailure, -}; -use std::{fmt, fs::File, io, net::TcpStream, path::PathBuf, time::Duration}; -/// One actual stopped-container deployment, private archive and authenticated startup. -#[derive(Clone)] -pub struct SandboxCreate { - /// Exact immutable image and borrowed shared named volume. - pub deployment: SandboxRequest, - /// Explicit private daemon resources and Disposable install/open selection. - pub setup: DaemonSetup, - /// Sealed Linux executable supplied by the caller; streamed once, never read as Store data. - pub executable: PathBuf, - /// Controller Noise secret; redacted from Debug and never sent in env/argv. - pub controller_private: [u8; 32], - /// Expected actual daemon identity, checked against private deployment config. - pub daemon_peer: [u8; 32], - /// Explicit listener event wait; does not apply to ordinary commands. - pub startup_wait: Duration, - /// Host to dial for the daemon's loopback-published port, when this - /// process does not share the Docker host's loopback (a client in a - /// container: `host.docker.internal`). `None` dials the observed - /// 127.0.0.1 endpoint. The publish and its check are unchanged. - pub connect_host: Option, -} -impl fmt::Debug for SandboxCreate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxCreate") - .field("deployment", &self.deployment) - .field("setup", &self.setup) - .field("executable", &self.executable) - .field("controller_private", &"[redacted]") - .field("daemon_peer", &self.daemon_peer) - .field("startup_wait", &self.startup_wait) - .field("connect_host", &self.connect_host) - .finish() - } -} +use layerfs_bridge::control::{ControlError, DaemonStatus, HelloRequest, Reply, Request}; +use layerfs_bridge::duplex::ChannelError; +use layerfs_sandbox::Runtime; +use std::{io, sync::Arc}; + /// The address to dial for a daemon `endpoint` observed on the Docker host's /// loopback: the endpoint itself, or the port on `host` (one resolution, its /// first address; nothing is tried twice). @@ -62,6 +24,7 @@ pub fn connect_address( ) }) } + /// Concrete boundary reached by one original lifecycle operation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SandboxPhase { @@ -94,6 +57,7 @@ pub enum SandboxPhase { /// One delete attempt after acknowledged stop. Delete, } + /// Exact original failure, independent of owned resource disposition. #[derive(Debug)] pub enum SandboxCause { @@ -101,14 +65,10 @@ pub enum SandboxCause { Io(io::Error), /// Local config or correlated startup validation. Setup(ControlError), - /// Actual original runtime Create including partial decoded ID. - Create(Box), - /// Actual incremental archive extraction failure and work. - Upload(Box), - /// Original listener wait/progress/fence failure. - Listener(Box), /// Original Engine transfer, no replay. Runtime(Box), + /// Original provider launch cause; partial runtime ownership is retained separately. + Provider(Box), /// Original authentication failure; handshake work remains in the receipt. Channel(ChannelError), /// Original correlated daemon control exchange/refusal. @@ -116,181 +76,13 @@ pub enum SandboxCause { /// Original streaming installation failure and exact publication knowledge. Install(Box), } -/// Caller retains exact acknowledged owner, connection and original deployment inputs. -pub struct SandboxFailure { - /// Original inputs with secrets retained privately. - pub request: SandboxCreate, - /// Original reached boundary. - pub phase: SandboxPhase, - /// Exact acknowledged owned container; no automatically adopted partial ID. - pub sandbox: Option, - /// Original authenticated control owner, if established. - pub control: Option, - /// Actual original handshake work, including failure. - pub handshake: ChannelWork, - /// Original correlated Hello, even if its phase failed selection validation. - pub observed: Option, - /// Installation acknowledgement, retained if the following Ready observation fails. - pub installed: Option, - /// Exact original cause. - pub cause: SandboxCause, -} -impl fmt::Debug for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxFailure") - .field("request", &self.request) - .field("phase", &self.phase) - .field("sandbox", &self.sandbox) - .field("has_control", &self.control.is_some()) - .field("handshake", &self.handshake) - .field("observed", &self.observed) - .field("cause", &self.cause) - .finish() - } -} -impl fmt::Display for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) - } -} -impl std::error::Error for SandboxFailure {} + /// Actual local Engine and authenticated daemon lifecycle organization. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SandboxApi { - pub(crate) docker: Docker, -} -/// Exact acknowledged container and authenticated control owner. -pub struct ManagedSandbox { - /// Owned runtime container; explicit stop/delete only. - pub runtime: Sandbox, - /// Existing sequential control owner; correlation and failed state are never reset. - pub(crate) control: Control, - /// Original acknowledged installation, when this create installed the Project. - pub installed: Option, - pub(crate) request: SandboxCreate, - /// Original correlated startup status; this is not kernel Workspace Ready. - pub startup: DaemonStatus, - /// Actual original duplex handshake work. - pub handshake: ChannelWork, - pub(crate) command_identity: layerfs_sandbox::CommandIdentity, -} -impl SandboxApi { - /// Uses one explicit local Engine endpoint and metadata wait policy. - pub fn new(docker: Docker) -> Self { - Self { docker } - } - /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. - pub fn create( - &self, - request: SandboxCreate, - project: &crate::SealedProject, - ) -> Result> { - self.create_file(request, &project.store, &project.manifest) - } - /// Creates a new daemon using exact persisted sealed-file and manifest facts. - pub fn create_file( - &self, - request: SandboxCreate, - store: &layerfs_persistence::SealedStore, - manifest: &layerfs_bridge::provision::StoreManifest, - ) -> Result> { - let existing = request.setup.existing_store.is_some(); - if let Some(selected) = &request.setup.existing_store { - let mut expected = manifest.clone(); - expected.daemon_sqlite = selected.daemon_sqlite.clone(); - if *selected != expected { - return Err(Box::new(SandboxFailure { - request, - phase: SandboxPhase::Validate, - sandbox: None, - control: None, - handshake: ChannelWork::default(), - observed: None, - installed: None, - cause: SandboxCause::Setup(ControlError("selected existing Project")), - })); - } - } - let mut owner = create(&self.docker, request)?; - if existing { - // The original startup already selected and waited for this Store. - // Installing again would change the existing one-attempt lifecycle. - return Ok(owner); - } - let installed = owner - .control - .install_file(store, manifest) - .map_err(|cause| { - Box::new(SandboxFailure { - request: owner.request.clone(), - phase: SandboxPhase::Install, - sandbox: None, - control: None, - handshake: owner.handshake, - observed: Some(owner.startup.clone()), - installed: None, - cause: SandboxCause::Install(cause), - }) - }); - let installed = match installed { - Ok(installed) => installed, - Err(mut failure) => { - failure.sandbox = Some(owner.runtime); - failure.control = Some(owner.control); - return Err(failure); - } - }; - owner.installed = Some(installed); - match hello( - &mut owner.control, - HelloRequest { - expected_instance: Some(owner.startup.instance), - wait_for_store: true, - }, - ) { - Ok(status) if status.phase == DaemonPhase::ControlReady => { - owner.startup = status; - Ok(owner) - } - Ok(status) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(status), - installed: owner.installed, - cause: SandboxCause::Setup(ControlError("original installed startup phase")), - })), - Err(cause) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(owner.startup), - installed: owner.installed, - cause: SandboxCause::Control(cause), - })), - } - } -} -impl ManagedSandbox { - /// Explicit startup observation/wait on the same incarnation and Control owner. - /// A later observation never resolves an earlier unknown operation. - pub(crate) fn status( - &mut self, - wait_for_store: bool, - ) -> Result> { - hello( - &mut self.control, - HelloRequest { - expected_instance: Some(self.startup.instance), - wait_for_store, - }, - ) - } + pub(crate) runtime: Arc, } + pub(super) fn hello( control: &mut Control, request: HelloRequest, @@ -301,118 +93,3 @@ pub(super) fn hello( reply => Err(OperationFailure::unexpected(request, reply)), } } -pub(crate) fn create( - docker: &Docker, - request: SandboxCreate, -) -> Result> { - let mut phase = SandboxPhase::Validate; - let mut sandbox = None; - let mut control = None; - let mut handshake = ChannelWork::default(); - let mut observed = None; - let result = (|| { - let config = request.setup.encode().map_err(SandboxCause::Setup)?; - if request.setup.listen != format!("0.0.0.0:{}", request.deployment.port) - || request.setup.store != "/layerfs-store/global/store.sqlite" - || request.setup.overlay != "/layerfs-local/overlay/overlay.sqlite" - || request.setup.mounts != "/workspaces" - || request.startup_wait.is_zero() - || public_key(&request.controller_private).map_err(SandboxCause::Channel)? - != request.setup.control_peer - || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? - != request.daemon_peer - { - return Err(SandboxCause::Setup(ControlError( - "concrete deployment topology/identity", - ))); - } - phase = SandboxPhase::Executable; - let mut file = File::open(&request.executable).map_err(SandboxCause::Io)?; - phase = SandboxPhase::Create; - sandbox = Some( - docker - .create_sandbox(request.deployment.clone()) - .map_err(SandboxCause::Create)?, - ); - let owner = sandbox.as_mut().expect("acknowledged container"); - phase = SandboxPhase::Upload; - owner - .upload(&mut file, &config) - .map_err(SandboxCause::Upload)?; - phase = SandboxPhase::Start; - owner.start().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Listener; - owner - .wait_listener(request.startup_wait) - .map_err(SandboxCause::Listener)?; - phase = SandboxPhase::Endpoint; - let endpoint = owner.endpoint().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Connect; - let address = - connect_address(endpoint, request.connect_host.as_deref()).map_err(SandboxCause::Io)?; - let stream = - TcpStream::connect_timeout(&address, request.startup_wait).map_err(SandboxCause::Io)?; - stream - .set_read_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - stream - .set_write_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - phase = SandboxPhase::Authenticate; - let start = initiate_observed(stream, &request.controller_private, request.daemon_peer); - handshake = start.work; - let connection = start.result.map_err(SandboxCause::Channel)?; - // The startup handshake waits are cleared before control/product operations. - // Bridge owns cloned duplex directions; clear policy through that actual owner. - control = Some(Control::new(connection)); - phase = SandboxPhase::Policy; - control - .as_ref() - .expect("authenticated") - .clear_io_waits() - .map_err(SandboxCause::Channel)?; - phase = SandboxPhase::Hello; - let status = hello( - control.as_mut().expect("authenticated"), - HelloRequest { - expected_instance: None, - wait_for_store: request.setup.existing_store.is_some(), - }, - ) - .map_err(SandboxCause::Control)?; - let expected = if request.setup.existing_store.is_some() { - DaemonPhase::ControlReady - } else { - DaemonPhase::InstallPending - }; - observed = Some(status.clone()); - if status.phase != expected { - return Err(SandboxCause::Setup(ControlError("original startup phase"))); - } - Ok(status) - })(); - match result { - Ok(startup) => Ok(ManagedSandbox { - request: request.clone(), - installed: None, - runtime: sandbox.take().expect("owned"), - control: control.take().expect("authenticated"), - startup, - handshake, - command_identity: layerfs_sandbox::CommandIdentity { - uid: request.setup.command_uid, - gid: request.setup.command_gid, - }, - }), - Err(cause) => Err(Box::new(SandboxFailure { - request, - phase, - sandbox, - control, - handshake, - observed, - installed: None, - cause, - })), - } -} diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index c0db8c8e5b..40e0a5e449 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -1,44 +1,30 @@ //! Optional ordinary Sandbox launch at the acknowledged mounted directory. -use crate::{ManagedSandbox, MountedWorkspace, WorkspaceApi}; -use layerfs_sandbox::{ - backend::docker::{ExecCreateFailure, ExecCreated, ExecRequest, Sandbox}, - CommandIdentity, -}; +use super::{MountedWorkspace, ProviderSandbox, WorkspaceApi}; +use layerfs_sandbox::{provider::ExecExit, ExecRequest, WireFailure}; +use std::io::Write; impl WorkspaceApi<'_> { - /// Creates one ordinary command at this mount using the deployment's admitted identity. - /// The returned Sandbox runtime owner owns Start, streams, exit status and cancellation. + /// Runs one ordinary command at this mount using the deployment's admitted identity. + /// The runtime provider owns execution, streams, exit status and cancellation. /// This sends no daemon request, registers no Exec and performs no Commit or unmount. pub fn exec( - sandbox: &ManagedSandbox, + sandbox: &ProviderSandbox, workspace: &MountedWorkspace, arguments: Vec, environment: Vec, - stdin: bool, - ) -> Result> { - launch( - &sandbox.runtime, - sandbox.command_identity, - arguments, - environment, - workspace.ready.directory.clone(), - stdin, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + sandbox.runtime.run( + &ExecRequest { + identity: sandbox.command_identity, + arguments, + environment, + directory: workspace.ready.directory.clone(), + stdin: false, + }, + stdout, + stderr, ) } } -pub(crate) fn launch( - sandbox: &Sandbox, - identity: CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result> { - sandbox.create_exec(ExecRequest { - identity, - arguments, - environment, - directory, - stdin, - }) -} diff --git a/engine/crates/layerfs-api/src/sandbox/mod.rs b/engine/crates/layerfs-api/src/sandbox/mod.rs index 6983a84746..1bf10e271d 100644 --- a/engine/crates/layerfs-api/src/sandbox/mod.rs +++ b/engine/crates/layerfs-api/src/sandbox/mod.rs @@ -1,6 +1,12 @@ //! Actual ordinary Sandbox lifecycle and its Workspaces, with exact logical and kernel readiness. #[cfg(unix)] mod close; +#[cfg(unix)] +mod connection; +#[cfg(unix)] +mod provider; +#[cfg(unix)] +pub use provider::{ProviderCreate, ProviderFailure, ProviderSandbox}; mod commit; #[cfg(unix)] pub(crate) mod create; @@ -12,9 +18,9 @@ mod unmount; pub mod values; pub use commit::CommitReply; #[cfg(unix)] -pub use create::{connect_address, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase}; +pub use create::SandboxApi; #[cfg(unix)] -pub use create::{ManagedSandbox, SandboxApi}; +pub use create::{connect_address, SandboxCause, SandboxPhase}; pub use mount::BoundWorkspace; pub use mount::WorkspaceApi; pub use mount::{MountFailure, MountedWorkspace}; diff --git a/engine/crates/layerfs-api/src/sandbox/mount.rs b/engine/crates/layerfs-api/src/sandbox/mount.rs index 15d2e70aab..4d0414cebc 100644 --- a/engine/crates/layerfs-api/src/sandbox/mount.rs +++ b/engine/crates/layerfs-api/src/sandbox/mount.rs @@ -12,9 +12,14 @@ pub struct WorkspaceApi<'a> { impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. #[cfg(unix)] - pub fn new(sandbox: &'a mut crate::ManagedSandbox) -> Self { + pub fn new(sandbox: &'a mut super::ProviderSandbox) -> Self { + Self::from_control(&mut sandbox.control) + } + /// Borrows an authenticated control channel directly. + #[cfg(unix)] + pub fn from_control(control: &'a mut Control) -> Self { Self { - control: &mut sandbox.control, + control, observation_scope: None, } } diff --git a/engine/crates/layerfs-api/src/sandbox/provider.rs b/engine/crates/layerfs-api/src/sandbox/provider.rs new file mode 100644 index 0000000000..098f66dc8b --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/provider.rs @@ -0,0 +1,373 @@ +//! Runtime startup uses the same authenticated connection and Store installation as Docker. +use super::{connection::Startup, SandboxApi, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonStatus}, + daemon_setup::DaemonSetup, + duplex::{public_key, ChannelWork}, + provision::StoreManifest, +}; +use layerfs_sandbox::{ + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, Sandbox, +}; +use std::{fmt, sync::Arc}; + +/// Runtime launch and private daemon authentication inputs. +#[derive(Clone)] +pub struct ProviderCreate { + /// Runtime launch; configuration is encoded from setup after validation. + pub launch: LaunchRequest, + /// Explicit Disposable daemon topology and Store selection. + pub setup: DaemonSetup, + /// Controller secret; never printed or sent in argv/environment. + pub controller_private: [u8; 32], + /// Expected daemon public identity. + pub daemon_peer: [u8; 32], +} +impl fmt::Debug for ProviderCreate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderCreate") + .field("launch", &self.launch) + .field("setup", &self.setup) + .field("daemon_peer", &self.daemon_peer) + .finish_non_exhaustive() + } +} +/// Exact startup failure and all acknowledged ownership, without automatic cleanup. +pub struct ProviderFailure { + /// Original private request. + pub request: ProviderCreate, + /// Last attempted stage. + pub phase: SandboxPhase, + /// Acknowledged runtime, including a partially launched instance. + pub runtime: Option>, + /// Authenticated channel when established. + pub control: Option, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Last correlated startup observation. + pub observed: Option, + /// Installation acknowledgement even when the following Hello fails. + pub installed: Option, + /// Original deciding cause. + pub cause: SandboxCause, +} +impl fmt::Debug for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderFailure") + .field("phase", &self.phase) + .field("runtime", &self.runtime.as_ref().map(|r| r.identity())) + .field("cause", &self.cause) + .finish_non_exhaustive() + } +} +impl fmt::Display for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) + } +} +impl std::error::Error for ProviderFailure {} +/// Runtime owner and its original authenticated daemon channel. +pub struct ProviderSandbox { + /// Original private launch and daemon configuration inputs. + pub request: ProviderCreate, + /// Explicit runtime lifecycle owner. + pub runtime: Box, + /// Original correlated startup observation. + pub startup: DaemonStatus, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Store installation acknowledgement, if installed by this create. + pub installed: Option, + pub(crate) control: Control, + /// Command identity admitted for this Sandbox deployment. + pub command_identity: CommandIdentity, +} +impl ProviderSandbox { + /// Explicit startup observation/wait on the same incarnation and Control owner. + pub(crate) fn status( + &mut self, + wait_for_store: bool, + ) -> Result> { + super::create::hello( + &mut self.control, + layerfs_bridge::control::HelloRequest { + expected_instance: Some(self.startup.instance), + wait_for_store, + }, + ) + } +} +impl SandboxApi { + /// Uses the deployment's selected runtime. + pub fn with_runtime(runtime: Arc) -> Self { + Self { runtime } + } + /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. + pub fn create( + &self, + request: ProviderCreate, + project: &crate::SealedProject, + ) -> Result> { + self.create_provider_file(request, &project.store, &project.manifest) + } + /// Launches once and authenticates to Hello without Store installation (internal proof only). + pub(crate) fn start( + &self, + request: ProviderCreate, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + ) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed: None, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed: None, + cause, + })), + } + } + /// Launches once, authenticates, selects or installs the Store, then observes Ready. + pub fn create_provider_file( + &self, + request: ProviderCreate, + store: &layerfs_persistence::SealedStore, + manifest: &StoreManifest, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let mut installed = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if request + .setup + .existing_store + .as_ref() + .is_some_and(|m| m.locator != request.setup.store) + || launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + if let Some(selected) = &request.setup.existing_store { + let mut expected = manifest.clone(); + expected.daemon_sqlite = selected.daemon_sqlite.clone(); + if *selected != expected { + return Err(SandboxCause::Setup(ControlError( + "selected existing Project", + ))); + } + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + let mut status = progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + )?; + if request.setup.existing_store.is_none() { + let mut phase = SandboxPhase::Install; + let result = super::connection::install( + progress.control.as_mut().expect("authenticated"), + &mut status, + &mut installed, + &mut phase, + store, + manifest, + ); + progress.phase = phase; + progress.observed = Some(status.clone()); + result?; + } + Ok(status) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed, + cause, + })), + } + } +} diff --git a/engine/crates/layerfs-api/tests/provider_ownership.rs b/engine/crates/layerfs-api/tests/provider_ownership.rs new file mode 100644 index 0000000000..3d57397ab7 --- /dev/null +++ b/engine/crates/layerfs-api/tests/provider_ownership.rs @@ -0,0 +1,143 @@ +//! Invalid native topology must refuse before spawning or touching another instance. +#![cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; +static NEXT: AtomicUsize = AtomicUsize::new(0); +struct ProjectFixture { + directory: PathBuf, + project: layerfs_api::SealedProject, +} +impl ProjectFixture { + fn new() -> Self { + use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId}; + use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; + let directory = std::env::temp_dir().join(format!( + "sdk-existing-project-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&directory).unwrap(); + let source = directory.join("source"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("payload"), b"original complete project").unwrap(); + let request = layerfs_api::InitRequest { + source, + store: PersistenceConfig::sqlite(directory.join("store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + locator: "/layerfs-store/global/store.sqlite".into(), + policy: layerfs_storage::StoragePolicy::frozen_default(), + catalog: HistoryCatalogConfig { + binding_key: b"sdk-existing-project".to_vec(), + incarnation: 1, + cursor_key: [7; 32], + }, + stack: LayerStackId::from_authority([3; 16]), + stack_name: HistoryName::new("project").unwrap(), + branch: BranchId::from_authority([4; 16]), + branch_name: HistoryName::new("main").unwrap(), + scope_seed: [5; 32], + deadline: Instant::now() + Duration::from_secs(10), + }; + let project = layerfs_telemetry::timer::Timing::disabled("sdk.existing.project", |timer| { + layerfs_api::ProjectApi::new().init(request, timer) + }) + .0 + .unwrap(); + Self { directory, project } + } +} +impl Drop for ProjectFixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +#[test] +fn native_resource_paths_are_validated_before_launch() { + use layerfs_api::{sandbox::ProviderCreate, SandboxApi, SandboxPhase}; + use layerfs_bridge::{ + daemon_setup::{DaemonLimits, DaemonSetup}, + duplex, + }; + use layerfs_sandbox::{ + backend::native::NativeMacosProvider, + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, + }; + use std::{ + os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, + }; + let fixture = ProjectFixture::new(); + let root = fixture.directory.join("runtime"); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = NativeMacosProvider::new(&root).unwrap(); + let paths = provider.paths("one").unwrap(); + let metadata = fs::metadata(&root).unwrap(); + let identity = CommandIdentity { + uid: metadata.uid(), + gid: metadata.gid(), + }; + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [32; 32], + control_peer: duplex::public_key(&[31; 32]).unwrap(), + store: fixture.project.store.path.display().to_string(), + overlay: paths.overlay.display().to_string(), + mounts: paths.mounts.display().to_string(), + command_uid: identity.uid, + command_gid: identity.gid, + existing_store: None, + limits: DaemonLimits { + connections: 2, + handshake_ms: 1000, + cache_bytes: 65536, + namespaces: 2, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + }; + let request = ProviderCreate { + setup, + controller_private: [31; 32], + daemon_peer: duplex::public_key(&[32; 32]).unwrap(), + launch: LaunchRequest { + name: "one".into(), + daemon_executable: "/bin/false".into(), + daemon_config: vec![], + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: identity, + environment: vec![], + }, + }, + }; + let api = SandboxApi::with_runtime(Arc::new(Runtime::MacosNative(provider))); + for field in ["overlay", "mounts", "store", "listen"] { + let mut invalid = request.clone(); + match field { + "overlay" => { + invalid.setup.overlay = root.join("other/overlay.sqlite").display().to_string() + } + "mounts" => invalid.setup.mounts = root.join("other/workspaces").display().to_string(), + "store" => { + invalid.setup.store = paths.directory.join("store.sqlite").display().to_string() + } + _ => invalid.setup.listen = "0.0.0.0:30421".into(), + } + let failure = api + .create_provider_file(invalid, &fixture.project.store, &fixture.project.manifest) + .err() + .unwrap(); + assert_eq!(failure.phase, SandboxPhase::Validate); + assert!(failure.runtime.is_none() && failure.control.is_none()); + assert!(!paths.directory.exists()); + } +} diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 64859816e2..507f177de5 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -3,20 +3,26 @@ #[allow(dead_code)] #[path = "../../layerfs-sandbox/tests/support/engine.rs"] mod engine; -use engine::{fixed, frame, Peer}; +use engine::{fixed, frame, upgrade, Peer}; use layerfs_api::{ - ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxPhase, WorkspaceApi, + ProviderCreate, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, WorkspaceApi, }; use layerfs_bridge::{ control::{Answer, Call, DaemonPhase, DaemonStatus, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, duplex, }; +use layerfs_sandbox::backend::docker::DockerProvider; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::{ fs, net::TcpListener, path::PathBuf, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, thread::{self, JoinHandle}, time::{Duration, Instant}, }; @@ -52,7 +58,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { struct Fixture { api: SandboxApi, - owner: Option, + owner: Option, peer: Option, daemon: Option>, binary: PathBuf, @@ -124,7 +130,7 @@ impl Fixture { ]; responses.extend(tail); let (docker, peer) = Peer::new(responses, 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let binary = std::env::temp_dir().join(format!( "sdk-operations-{}-{}", std::process::id(), @@ -137,19 +143,23 @@ impl Fixture { manifest.daemon_sqlite = Some("3.51.0".into()); deployment_setup.existing_store = Some(manifest); } - let request = SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + let request = ProviderCreate { + launch: LaunchRequest { + name: "sdk-operations".into(), + daemon_executable: binary.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: binary.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }; // No Store is constructed or opened. This is the original InstallPending // startup used by component proofs, not a fabricated SDK owner. @@ -201,10 +211,8 @@ fn failed_stop_preserves_original_owner_channel_inputs_and_attempt_without_delet let handshake = owner.handshake; let failure = fixture.api.close(owner).unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Stop); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(!runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -227,10 +235,8 @@ fn failed_delete_retains_acknowledged_stop_and_exact_delete_failure() { .close(fixture.owner.take().unwrap()) .unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Delete); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -299,39 +305,44 @@ fn workspace_exec_returns_original_runtime_owner_at_the_mount_with_no_daemon_reg const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; let mut fixture = Fixture::new(vec![ fixed(201, format!("{{\"Id\":\"{EID}\"}}").as_bytes()), + upgrade(&frame(1, b"")), + fixed( + 200, + format!( + "{{\"ID\":\"{EID}\",\"ContainerID\":\"{CID}\",\"Running\":false,\"ExitCode\":0,\"Pid\":42}}" + ) + .as_bytes(), + ), fixed(204, b""), fixed(204, b""), ]); let before = layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(); - let created = WorkspaceApi::exec( + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec!["SDK_TEST=1".into()], - false, + &mut stdout, + &mut stderr, ) .unwrap(); - assert_eq!(created.identity().0.to_string(), CID); - assert_eq!(created.identity().1.to_string(), EID); - assert_eq!(created.request().directory, "/workspaces/selected-mount"); - assert_eq!( - created.request().identity, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 } - ); + assert_eq!(exit.code, 0); + assert_eq!(exit.pid, Some(42)); assert_eq!( layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(), before ); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); let calls = fixture.calls(); - assert_eq!(calls.len(), 9); + assert_eq!(calls.len(), 11); let request = std::str::from_utf8(&calls[6]).unwrap(); assert!(request.starts_with(&format!("POST /v1.54/containers/{CID}/exec "))); assert!(request.contains(r#""WorkingDir":"/workspaces/selected-mount""#)); assert!(request.contains(r#""User":"501:20""#)); - assert!(!calls - .iter() - .any(|call| call.windows(12).any(|bytes| bytes == b"/exec/start "))); + assert!(calls[7].starts_with(format!("POST /v1.54/exec/{EID}/start ").as_bytes())); + assert!(calls[8].starts_with(format!("GET /v1.54/exec/{EID}/json ").as_bytes())); } #[test] fn workspace_exec_failure_keeps_exact_original_request_and_container() { @@ -340,19 +351,22 @@ fn workspace_exec_failure_keeps_exact_original_request_and_container() { fixed(204, b""), fixed(204, b""), ]); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); let failure = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec![], - true, + &mut stdout, + &mut stderr, ) .unwrap_err(); - assert_eq!(failure.container.to_string(), CID); - assert_eq!(failure.request.directory, "/workspaces/selected-mount"); - assert_eq!(failure.request.arguments, ["/bin/pwd"]); - assert!(failure.request.stdin); - assert!(failure.transfer.attempted); + assert_eq!( + failure.requested_container, + Some(layerfs_sandbox::ContainerId::parse(CID).unwrap()) + ); + assert!(failure.attempted); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); assert_eq!(fixture.calls().len(), 9); } @@ -429,25 +443,29 @@ fn create_matching_existing_project_keeps_one_original_hello_and_does_not_reinst fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_effect() { let project = ProjectFixture::new(); let (docker, peer) = Peer::new(vec![], 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let mut deployment_setup = setup([32; 32], duplex::public_key(&[31; 32]).unwrap()); let mut manifest = project.project.manifest.clone(); manifest.root[0] ^= 1; deployment_setup.existing_store = Some(manifest); let result = api.create( - SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "mismatched-sandbox".into(), + daemon_executable: project.project.store.path.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: project.project.store.path.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }, &project.project, ); @@ -455,6 +473,6 @@ fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_e panic!("mismatched Project must refuse before effects") }; assert_eq!(failure.phase, SandboxPhase::Validate); - assert!(failure.sandbox.is_none() && failure.control.is_none() && failure.installed.is_none()); + assert!(failure.runtime.is_none() && failure.control.is_none() && failure.installed.is_none()); assert!(peer.finish().is_empty()); } diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..95083db61d 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -1,4 +1,7 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +#[cfg(target_os = "linux")] +use crate::runtime::NAMESPACES; +use crate::runtime::{LOCAL, NATIVE}; use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, @@ -9,10 +12,11 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; -use layerfs_sandbox::{ - backend::docker::{Docker, Sandbox, SandboxRequest}, - CommandIdentity, ContainerId, -}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider, SandboxRequest}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::backend::native::{self, LocalProcess}; +use layerfs_sandbox::provider::{LocalRecord, SandboxRecord}; +use layerfs_sandbox::{CommandIdentity, ContainerId, Sandbox, WireFailure}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, @@ -26,6 +30,7 @@ use std::{ }; extern "C" { fn geteuid() -> u32; + fn getegid() -> u32; fn flock(fd: i32, operation: i32) -> i32; } const LOCK_EX: i32 = 2; @@ -45,7 +50,7 @@ pub(crate) fn home(args: &SdkValue) -> Result { directory(&home)?; Ok(home) } -fn private(path: &Path, dir: bool) -> Result<()> { +pub(crate) fn private(path: &Path, dir: bool) -> Result<()> { let meta = fs::symlink_metadata(path).map_err(failed)?; if meta.file_type().is_symlink() || meta.is_dir() != dir @@ -68,6 +73,11 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } +/// This process's effective uid and gid: a native Sandbox's command +/// identity (its commands run as this user). +pub(crate) fn user() -> (u32, u32) { + unsafe { (geteuid(), getegid()) } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -127,6 +137,8 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. +// Unmount exchanges run outside this guard; one shared readiness deadline +// covers the in-process turn and the cross-process discovery lock. const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); @@ -328,6 +340,20 @@ pub(crate) fn project(args: &SdkValue) -> Result { installed, }) } +/// The labels a record keeps, as the value map its reader expects. +fn label_values(labels: &[(String, String)]) -> SdkValue { + SdkValue::List( + (labels.iter()) + .map(|(key, value)| { + SdkValue::map([ + ("key", SdkValue::text(key)), + ("value", SdkValue::text(value)), + ]) + }) + .collect(), + ) +} + pub(crate) fn save_sandbox( directory: &Path, state: &SandboxState, @@ -339,41 +365,97 @@ pub(crate) fn save_sandbox( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "deploy"))?; + let mut fields = vec![ + ("id", SdkValue::text(&state.id)), + ("project", SdkValue::text(&state.project)), + ("name", SdkValue::text(&state.name)), + ( + "container", + SdkValue::text(state.runtime.identity().to_string()), + ), + ]; + match state.runtime.as_ref() { + Sandbox::Docker(runtime) => { + let record = runtime.request(); + fields.extend([ + ("docker", SdkValue::text(registry::text(deploy, "docker")?)), + ("image", SdkValue::text(&record.image)), + ("volume", SdkValue::text(&record.store_volume)), + ("labels", label_values(&record.labels)), + ]); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Sandbox::Native(runtime) => { + let record = runtime + .local_record() + .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + fields.extend(local_fields(args, state, &record, NATIVE, LOCAL)?); + } + #[cfg(target_os = "linux")] + Sandbox::Namespaces(runtime) => { + let Some(SandboxRecord::Namespaces(record)) = runtime.record() else { + return Err(refused("Failed", "namespace Sandbox has no init process")); + }; + let image = record.image().as_str().to_owned(); + fields.extend(local_fields( + args, + state, + record.local(), + NAMESPACES, + &image, + )?); + } + } + fields.extend([ + ("secret", SdkValue::Bytes(secret.to_vec())), + ("peer", SdkValue::Bytes(peer.to_vec())), + ("instance", SdkValue::Bytes(instance.to_vec())), + ]); save( &directory.join("sandbox.record"), - SdkValue::map([ - ("id", SdkValue::text(&state.id)), - ("project", SdkValue::text(&state.project)), - ("name", SdkValue::text(&state.name)), - ( - "container", - SdkValue::text(state.runtime.identity().to_string()), - ), - ("docker", SdkValue::text(registry::text(deploy, "docker")?)), - ("image", SdkValue::text(&state.runtime.request().image)), - ( - "volume", - SdkValue::text(&state.runtime.request().store_volume), - ), - ( - "labels", - SdkValue::List( - (state.runtime.request().labels.iter()) - .map(|(key, value)| { - SdkValue::map([ - ("key", SdkValue::text(key)), - ("value", SdkValue::text(value)), - ]) - }) - .collect(), - ), - ), - ("secret", SdkValue::Bytes(secret.to_vec())), - ("peer", SdkValue::Bytes(peer.to_vec())), - ("instance", SdkValue::Bytes(instance.to_vec())), - ]), + SdkValue::Map( + (fields.into_iter()) + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ), ) } +/// The fields a local Sandbox record keeps: its directory, its daemon or init +/// process identity, its listener port and the image it runs, under `runtime` +/// `native` (whose image is `local`) or `namespaces` (whose image is the +/// digest its root is an overlay of). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_fields( + args: &SdkValue, + state: &SandboxState, + record: &LocalRecord, + runtime: &'static str, + image: &str, +) -> Result> { + let boot = record + .boot + .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; + Ok(vec![ + ("runtime", SdkValue::text(runtime)), + ("uid", SdkValue::Int(i64::from(state.identity.uid))), + ("gid", SdkValue::Int(i64::from(state.identity.gid))), + ( + "directory", + SdkValue::text( + crate::local::directory(args, &state.id)? + .display() + .to_string(), + ), + ), + ("pid", SdkValue::Int(i64::from(record.pid))), + ("start", SdkValue::text(record.start_time.to_string())), + ("boot", SdkValue::text(format!("{boot:032x}"))), + ("port", SdkValue::Int(i64::from(record.port))), + ("image", SdkValue::text(image)), + ("labels", label_values(&record.labels)), + ]) +} + /// A Sandbox whose container no longer runs it: the Engine no longer has the /// container, or has it stopped (it closed when idle, or Docker restarted), /// which lost its daemon and every mount in it. Only its records, and a @@ -406,30 +488,260 @@ pub(crate) fn gone_container(refusal: &SdkRefusal) -> Option<(&str, bool)> { _ => None, } } -/// The Engine, container and deployment a Sandbox record names. -pub(crate) fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { +/// The labels a Sandbox record keeps; a record written before #373 has none. +fn labels(value: &SdkValue) -> Result> { + match value.get("labels") { + Some(SdkValue::List(items)) => items + .iter() + .map(|item| { + Ok(( + registry::text(item, "key")?.to_owned(), + registry::text(item, "value")?.to_owned(), + )) + }) + .collect(), + _ => Ok(Vec::new()), + } +} +/// The Engine, container and deployment a Docker Sandbox record names. +fn runtime(value: &SdkValue) -> Result<(DockerProvider, ContainerId, SandboxRecord)> { let docker = Docker::new(registry::text(value, "docker")?, Duration::from_secs(30)).map_err(failed)?; let request = SandboxRequest { image: registry::text(value, "image")?.to_owned(), store_volume: registry::text(value, "volume")?.to_owned(), port: 30421, - // a record written before #373 has no labels, nor has its container - labels: match value.get("labels") { - Some(SdkValue::List(items)) => items - .iter() - .map(|item| { - Ok(( - registry::text(item, "key")?.to_owned(), - registry::text(item, "value")?.to_owned(), - )) - }) - .collect::>()?, - _ => Vec::new(), - }, + labels: labels(value)?, }; let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; - Ok((docker, container, request)) + let provider = DockerProvider::new(docker); + Ok((provider, container, SandboxRecord::Docker(request.into()))) +} +/// What a Sandbox record names: a Docker container, a local daemon process +/// (`runtime` `native`) with its directory and port, or a namespace Sandbox +/// (`runtime` `namespaces`) with its own runtime, record and directory. +pub(crate) enum Recorded { + Docker(DockerProvider, ContainerId, SandboxRecord), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Local { + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + }, + #[cfg(target_os = "linux")] + Namespaces(layerfs_sandbox::Runtime, ContainerId, SandboxRecord), +} +/// Reads Sandbox `id`'s record's runtime. +pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + #[cfg(target_os = "linux")] + if value.get_text("runtime") == Some(NAMESPACES) { + return namespaces_record(args, id, value); + } + if matches!(value.get_text("runtime"), Some(LOCAL | NATIVE)) { + return local_record(args, id, value); + } + let (docker, container, request) = runtime(value)?; + Ok(Recorded::Docker(docker, container, request)) +} +/// A local record is used only where this deployment puts that Sandbox: +/// its directory must be exactly `/-` +/// and named so, so a damaged, edited or stale record (another +/// LAYERFS_LOCAL_DIR or prefix) never points a delete elsewhere. A record +/// without `boot` was written before boot ids were kept: its daemon is +/// matched by id and start time as then, and never signalled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, record) = local_record_of(args, id, value)?; + // with no placement (#428), a mount names its Sandbox: a record is used + // only by a native deployment, and only a native record is. A record of + // the `local` runtime (an earlier, unmerged build) is never used, so + // nothing it names is deleted + let native = args.get("deploy").is_some_and(crate::runtime::native); + if !native || value.get_text("runtime") != Some(NATIVE) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a {} Sandbox, but only a native deployment \ + (LAYERFS_SANDBOX=native) uses a native record and none uses another; \ + Sandboxes of other runtimes are never mixed, so the record is neither used \ + nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + value.get_text("runtime").unwrap_or_default(), + registry::text(args, "project")? + ), + )); + } + Ok(Recorded::Local { + directory, + id: ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + process: LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + }, + port: record.port, + labels: record.labels, + }) +} +/// A record's directory and physical facts, checked against where this +/// deployment puts that Sandbox. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record_of(args: &SdkValue, id: &str, value: &SdkValue) -> Result<(PathBuf, LocalRecord)> { + let directory = PathBuf::from(registry::text(value, "directory")?); + let expected = crate::local::directory(args, id)?; + // the Sandbox's own directory name, exactly as this deployment names it + let name = registry::text(value, "name")?; + if directory != expected || expected.file_name().and_then(|n| n.to_str()) != Some(name) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id}'s local record names {} (as {name}), but this deployment puts it at \ + {}: the record is damaged or from another LAYERFS_LOCAL_DIR or prefix, so it is \ + neither used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + directory.display(), + expected.display(), + registry::text(args, "project")? + ), + )); + } + let number = |key: &str| { + value + .get_int(key) + .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) + }; + let boot = match value.get_text("boot") { + None => None, + Some(hex) => Some( + (hex.len() == 32) + .then(|| u128::from_str_radix(hex, 16).ok()) + .flatten() + .ok_or_else(|| refused("InvalidInput", "local Sandbox record boot"))?, + ), + }; + let record = LocalRecord { + name: name.to_owned(), + pid: u32::try_from(number("pid")?).map_err(failed)?, + start_time: registry::text(value, "start")? + .parse::() + .map_err(failed)?, + boot, + port: u16::try_from(number("port")?).map_err(failed)?, + labels: labels(value)?, + }; + Ok((directory, record)) +} +/// A namespaces record is used only where this backend puts that Sandbox and +/// only by a deployment that selects it: another backend never reopens or +/// deletes what this one made. +#[cfg(target_os = "linux")] +fn namespaces_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, local) = local_record_of(args, id, value)?; + let image = + layerfs_sandbox::Image::parse(registry::text(value, "image")?).map_err(|error| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox whose record names no image digest \ + ({error}); it cannot be given a root, so it is neither reopened nor deleted" + ), + ) + })?; + let record = layerfs_sandbox::provider::NamespaceRecord { local, image }; + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + if crate::runtime::backend(deploy)? != crate::runtime::Backend::Namespaces { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox, but this deployment runs the other local \ + backend; Sandboxes of other runtimes are never mixed, so the record is neither \ + used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + registry::text(args, "project")? + ), + )); + } + let root = (directory.parent()) + .ok_or_else(|| refused("InvalidInput", "namespaces Sandbox directory"))?; + let provider = + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root).map_err(|error| { + refused( + "Unsupported", + format!( + "namespaces Sandbox at {} cannot run on this host: {error}", + directory.display() + ), + ) + })?; + Ok(Recorded::Namespaces( + layerfs_sandbox::Runtime::Namespaces(provider), + ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + SandboxRecord::Namespaces(record), + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { + Err(refused( + "Unsupported", + format!( + "Sandbox {id} is a native Sandbox (LAYERFS_SANDBOX=native), which needs Linux or macOS" + ), + )) +} +impl Recorded { + /// The runtime, observed running once (it starts nothing). + pub(crate) fn reopen(self) -> std::result::Result, Box> { + match self { + Self::Docker(provider, id, record) => provider + .reopen(id, &record) + .map(|s| Box::new(Sandbox::Docker(s))), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => native::Sandbox::reopen(directory, id, process, port, labels) + .map(|s| Box::new(Sandbox::Native(s))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => runtime.reopen(id, &record).map(Box::new), + } + } + /// The runtime found stopped: only its one Delete is admitted. + pub(crate) fn stopped(self) -> Box { + match self { + Self::Docker(provider, id, record) => { + let instance = provider + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"); + Box::new(Sandbox::Docker(instance)) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => Box::new(Sandbox::Native(native::Sandbox::stopped( + directory, id, process, port, labels, + ))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => Box::new( + runtime + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"), + ), + } + } } pub(crate) fn reopen(args: &SdkValue) -> Result> { let id = registry::text(args, "sandbox")?; @@ -447,8 +759,16 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let (docker, selector, request) = runtime(&value)?; - let runtime = Sandbox::reopen(docker, selector, request).map_err(|failure| { + let recorded = recorded(args, id, &value)?; + // a local daemon listens on this kernel's loopback; no other host reaches it + let host = match recorded { + Recorded::Docker(..) => crate::pool::daemon_host(args), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Recorded::Local { .. } => None, + #[cfg(target_os = "linux")] + Recorded::Namespaces(..) => None, + }; + let runtime = recorded.reopen().map_err(|failure| { if failure.container_gone() || failure.container_stopped() { gone(id, container, failure.container_stopped()) } else { @@ -457,7 +777,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { })?; let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, - host: crate::pool::daemon_host(args), + host, secret: bytes(&value, "secret")?, peer: bytes(&value, "peer")?, instance: bytes(&value, "instance")?, @@ -470,7 +790,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { name: value.get_text("name").unwrap_or_default().to_owned(), directory, runtime, - identity: CommandIdentity { uid: 501, gid: 20 }, + identity: recorded_identity(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -478,8 +798,23 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { refresh(&state)?; Ok(state) } +/// The command identity a Sandbox record names: a native Sandbox's is the +/// user who started it (`uid`, `gid`); every other runs as 501:20. +fn recorded_identity(value: &SdkValue) -> CommandIdentity { + let id = |key: &str| value.get_int(key).and_then(|n| u32::try_from(n).ok()); + match (id("uid"), id("gid")) { + (Some(uid), Some(gid)) => CommandIdentity { uid, gid }, + _ => CommandIdentity { uid: 501, gid: 20 }, + } +} /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { + refresh_except(state, &[]) +} +/// Explicit Sandbox close may dispose retained unmounts without asking their +/// daemon for a Ready receipt that no longer exists. Other records still +/// undergo the ordinary refresh and MountsActive admission. +pub(crate) fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); let mut processes = std::collections::BTreeMap::new(); for item in fs::read_dir(&state.directory).map_err(failed)? { @@ -490,6 +825,9 @@ pub(crate) fn refresh(state: &SandboxState) -> Result<()> { continue; }; let path = registry::text(&value, "path")?; + if retained.iter().any(|kept| kept == path) { + continue; + } let token = layerfs_bridge::control::WorkspaceToken { workspace: layerfs_history::WorkspaceId::from_slice(&bytes::<32>( &value, diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index 80a3120b2a..3982d74648 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -5,46 +5,38 @@ //! daemon is not involved. A background command is started in its own session //! with its output in a log file, and this binding keeps its process id. The //! Engine offers no per-exec cancel, so `cancel` signals the process group -//! from a second command and waits (bounded) until it is gone. -use crate::job::{failed, refused, Outcome}; +//! from a second command and waits (bounded) until it is gone. A local +//! Sandbox (`layerfs_sandbox::Sandbox::Native`) runs the same commands as +//! processes of this kernel, so the same probes and signals apply. +use crate::job::{refused, Outcome}; use crate::output::Spool; use crate::registry::{lock, naming, sandbox, sandbox_live, text, Process, SandboxState}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::time::Duration; -/// Home of the command identity, which has no account in the image. -const HOME: &str = "HOME=/tmp"; /// Checks that a cancelled process group is gone. const GONE_CHECKS: u32 = 50; /// Pause between those checks. const GONE_PAUSE: Duration = Duration::from_millis(100); /// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. +/// non-empty stream to its file instead of keeping it inline. Its home is +/// the Sandbox's scratch directory (`/tmp` in a container), since the +/// command identity has no account there. fn run( state: &SandboxState, directory: &str, arguments: Vec, spool: bool, ) -> Result<(i32, Spool, Spool), SdkRefusal> { - let created = layerfs_api::internal::exec( + crate::runtime::run( &state.runtime, state.identity, arguments, - vec![HOME.to_owned()], - directory.to_owned(), - false, + vec![format!("HOME={}", crate::runtime::scratch(&state.runtime))], + directory, + spool, ) - .map_err(failed)?; - let attached = created.start().map_err(failed)?; - let (_input, output, handle) = attached.into_parts().map_err(failed)?; - let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); - output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; - let inspected = handle.inspect().map_err(failed)?; - let code = inspected - .known_root_exit() - .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))?; - Ok((code, stdout, stderr)) } /// `workspace.exec(ws, command, { background?, spool? })`; with @@ -91,11 +83,17 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { state.id, layerfs_bridge::sdk_values::hex(&crate::registry::random()?[..12]) ); - let script = format!("setsid bash -c \"$0\" >/tmp/layerfs-{id}.log 2>&1 \"$1\" 2>&1 Outcome { "background process exited before its identity was observed", ) })?; + // its own session leaves the command's process group; named, it still + // ends with a local Sandbox. Unnamed it would outlive it: it ends here, once. + if let Err(refusal) = crate::runtime::track(&state.runtime, pid, &start) { + let end = format!("kill -KILL -- -{pid} 2>/dev/null; true"); + let _ = run( + &state, + &directory, + vec!["bash".into(), "-c".into(), end], + false, + ); + return Err(refusal); + } let process = Process { path: path.to_owned(), command: command.to_owned(), @@ -136,7 +146,7 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { /// The container's first process is the daemon, which does not reap orphans, /// so a finished member can stay a zombie that `kill -0` still finds. fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { - let (code, out, _) = run(state, "/tmp", vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; + let (code, out, _) = run(state, &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; if code == 0 { Ok(Some(String::from_utf8_lossy(out.kept()).into_owned())) } else { @@ -151,7 +161,7 @@ fn alive(state: &SandboxState, pid: u32) -> Result { ); let (code, _, _) = run( state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), probe], false, )?; @@ -283,7 +293,7 @@ pub(crate) fn cancel(args: &SdkValue) -> Outcome { } run( &state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec![ "bash".into(), "-c".into(), diff --git a/engine/crates/layerfs-node/src/lib.rs b/engine/crates/layerfs-node/src/lib.rs index 298d5020af..f7f89ae788 100644 --- a/engine/crates/layerfs-node/src/lib.rs +++ b/engine/crates/layerfs-node/src/lib.rs @@ -5,12 +5,16 @@ mod discovery; mod dispatch; mod exec; mod job; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local; mod output; mod pool; mod project; mod registry; +mod runtime; mod sandbox; mod trace; +mod unmount; mod workspace; use job::Job; diff --git a/engine/crates/layerfs-node/src/local.rs b/engine/crates/layerfs-node/src/local.rs new file mode 100644 index 0000000000..1556ed7efa --- /dev/null +++ b/engine/crates/layerfs-node/src/local.rs @@ -0,0 +1,440 @@ +//! Native Sandboxes (`deploy.sandbox` `native`, LAYERFS_SANDBOX=native): +//! the project's daemon runs as a process of this user instead of in a +//! container. It isolates nothing beyond the caller's own sandbox. +//! +//! Each Sandbox gets a directory `/-` +//! with what a container held at fixed paths: its setup +//! (`/layerfs-local/config`), overlay template (`/layerfs-local/overlay`) and +//! Workspace mounts (`/workspaces`). The daemon opens the installed Store +//! where this process opens it (`storePath`), so the daemon setup and the +//! Store's locator name that path instead of `/layerfs-store`. The daemon +//! listens on 127.0.0.1 only. macOS serves loopback NFS as this user; Linux +//! keeps the root FUSE daemon and nonroot commands (501:20) from #402. Start, +//! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's +//! daemon stopped and its directory deleted, each once. +use crate::job::{failed, refused}; +use crate::registry::{lock, text, SandboxState, SANDBOXES}; +use layerfs_api::{sandbox::ProviderCreate, SandboxApi}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::{CommandIdentity, Runtime, Sandbox}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +/// Starts the project's next Sandbox as a native daemon. Linux requires +/// root before any effect. Called by +/// [`crate::sandbox::start`] under the project's guard, with the project +/// and its installed manifest already read. +pub(crate) fn start( + args: &SdkValue, + directory: &Path, + project: (&crate::registry::Project, StoreManifest), +) -> Result, SdkRefusal> { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + // Inputs are read first, so a refused request leaves no creation intent. + if cfg!(target_os = "linux") && crate::discovery::user().0 != 0 { + return Err(refused( + "Unsupported", + "LAYERFS_SANDBOX=native on Linux needs root for FUSE and commands as uid 501, gid 20; run inside the VM or CI box that isolates it", + )); + } + let root = deploy.get_text("localDir").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a native Sandbox needs it", + ) + })?; + let backend = crate::runtime::backend(deploy)?; + let runtime = native_runtime(root, backend) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let executable = PathBuf::from(text(deploy, "daemon")?); + let store = match args.get_text("storePath") { + Some(path) if Path::new(path).is_absolute() => path.to_owned(), + _ => { + return Err(refused( + "InvalidInput", + "a local Sandbox's daemon opens the client's storePath (LAYERFS_STORE_PATH), an absolute path", + )) + } + }; + crate::sandbox::unresolved( + directory, + &format!("a directory under {root} named for it, and its daemon process"), + )?; + let (id, saved_directory) = crate::discovery::claim_next(args)?; + let made = start_claimed( + args, + directory, + &saved_directory, + &id, + project, + (runtime, backend, executable, store), + ); + if made.is_err() { + let _ = std::fs::remove_dir(&saved_directory); + } + made +} + +/// Starts the claimed Sandbox `id`: directory and daemon on the installed +/// Store, its first control connection, then its records. +fn start_claimed( + args: &SdkValue, + directory: &Path, + saved_directory: &Path, + id: &str, + (project, installed): (&crate::registry::Project, StoreManifest), + (runtime, backend, executable, store): (Runtime, crate::runtime::Backend, PathBuf, String), +) -> Result, SdkRefusal> { + let name = text(args, "project")?; + let sandbox_name = sandbox_name(args, id)?; + let paths = runtime.native_paths(&sandbox_name).map_err(failed)?; + let path_text = |path: &Path| { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_LOCAL_DIR must be UTF-8")) + }; + let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let mut setup = crate::sandbox::setup(daemon.private, controller.public); + // macOS uses the caller; Linux keeps #402's separate nonroot identity. + #[cfg(target_os = "macos")] + { + (setup.command_uid, setup.command_gid) = crate::discovery::user(); + } + setup.listen = "127.0.0.1:0".into(); + setup.store = store.clone(); + setup.overlay = path_text(&paths.overlay)?; + setup.mounts = path_text(&paths.mounts)?; + let mut existing = project.manifest.clone(); + existing.daemon_sqlite = installed.daemon_sqlite; + // the Store's path in the daemon's filesystem, which is this one + existing.locator = store; + setup.existing_store = Some(existing); + let identity = CommandIdentity { + uid: setup.command_uid, + gid: setup.command_gid, + }; + setup.encode().map_err(failed)?; + // Read only if this process ends before the intent is resolved. + crate::discovery::save( + &directory.join("create.intent"), + SdkValue::text(format!("creation of Sandbox {id} began and never finished")), + )?; + let labels = crate::sandbox::labels(name, id); + let api = SandboxApi::with_runtime(Arc::new(runtime)); + let options = launch_options(backend, identity, &setup.store, args)?; + let mut manifest = project.manifest.clone(); + manifest.locator = setup.store.clone(); + let created = api + .create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: sandbox_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), + labels, + startup_timeout: crate::sandbox::STARTUP, + options, + }, + setup, + controller_private: controller.private, + daemon_peer: daemon.public, + }, + &project.store, + &manifest, + ) + .map_err(|mut failure| { + drop(failure.control.take()); + abandon( + directory, + id, + format!( + "native sandbox stopped at {:?}: {:?}", + failure.phase, failure.cause + ), + failure.runtime.take(), + ) + })?; + let instance = created.startup.instance; + let (sandbox, control, identity) = layerfs_api::internal::split_provider(created); + let endpoint = match sandbox.endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + drop(control); + return Err(abandon( + directory, + id, + format!("native endpoint after startup: {error:?}"), + Some(sandbox), + )); + } + }; + let dial = crate::pool::Dial { + endpoint, + host: None, + secret: controller.private, + peer: daemon.public, + instance, + }; + let state = SandboxState { + id: id.to_owned(), + project: name.to_owned(), + name: sandbox_name, + directory: saved_directory.to_owned(), + runtime: sandbox, + identity, + controls: crate::pool::Controls::new(control, dial), + mounts: Default::default(), + processes: Default::default(), + }; + let recorded = (|| { + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed) + })(); + if let Err(refusal) = recorded { + // The daemon runs and is this start's own: it is stopped and its + // directory deleted like any failed start's, and the records written + // so far are removed, so its number is free again. + let SandboxState { runtime, .. } = state; + for record in ["sandbox.record", "container"] { + let _ = std::fs::remove_file(saved_directory.join(record)); + } + return Err(abandon( + directory, + id, + format!("local sandbox records: {}", refusal.message), + Some(runtime), + )); + } + let state = Arc::new(state); + lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); + Ok(state) +} + +/// The native runtime this host offers, over `root`: macOS loopback NFS as +/// this user, or Linux FUSE with a root daemon. The other platform's +/// provider, and another OS, is refused before root access. +#[cfg(target_os = "macos")] +fn native_runtime( + root: &str, + _: crate::runtime::Backend, +) -> Result { + layerfs_sandbox::backend::native::NativeMacosProvider::new(root).map(Runtime::MacosNative) +} + +#[cfg(target_os = "linux")] +fn native_runtime( + root: &str, + backend: crate::runtime::Backend, +) -> Result { + match backend { + crate::runtime::Backend::Native => { + layerfs_sandbox::backend::native::NativeLinuxProvider::new(root) + .map(Runtime::LinuxNative) + } + crate::runtime::Backend::Namespaces => { + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root) + .map(Runtime::Namespaces) + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn native_runtime( + _: &str, + _: crate::runtime::Backend, +) -> Result { + Err(layerfs_sandbox::RuntimeError::Unsupported( + "native Sandbox runtime platform", + )) +} + +/// How the daemon of `backend` is launched: the same local options, and for +/// the namespace backend the Store file its private root binds and the image +/// archive its root is unpacked from. +#[cfg(target_os = "linux")] +fn launch_options( + backend: crate::runtime::Backend, + identity: CommandIdentity, + store: &str, + args: &SdkValue, +) -> Result { + Ok(match backend { + crate::runtime::Backend::Native => ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }, + crate::runtime::Backend::Namespaces => { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + let (image, archive) = crate::runtime::namespaces_image(deploy)?; + ProviderLaunchOptions::Namespaces { + command_identity: identity, + environment: native_environment(), + store: PathBuf::from(store), + image, + archive, + } + } + }) +} + +#[cfg(not(target_os = "linux"))] +fn launch_options( + _: crate::runtime::Backend, + identity: CommandIdentity, + _: &str, + _: &SdkValue, +) -> Result { + Ok(ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }) +} + +/// What a native Sandbox's daemon starts with besides `PATH`: the NFS +/// route, and the mount helper and reference overrides this process was +/// given (`LAYERFS_NFS_*`), so a test host can name its own client. +#[cfg(target_os = "macos")] +fn native_environment() -> Vec<(String, String)> { + let mut environment = vec![("LAYERFS_KERNEL".to_owned(), "nfs".to_owned())]; + for name in [ + "LAYERFS_NFS_MOUNT", + "LAYERFS_NFS_UNMOUNT", + "LAYERFS_NFS_FORCE", + "LAYERFS_NFS_FLUSH", + "LAYERFS_NFS_REFRESH", + ] { + if let Ok(value) = std::env::var(name) { + environment.push((name.to_owned(), value)); + } + } + environment +} + +#[cfg(target_os = "linux")] +fn native_environment() -> Vec<(String, String)> { + vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())] +} + +/// A local Sandbox's directory name: `-`, the name its +/// container would have. +fn sandbox_name(args: &SdkValue, id: &str) -> Result { + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + Ok(format!("{prefix}{}-{id}", text(args, "project")?)) +} + +/// Where this deployment puts Sandbox `id`: +/// `/-`, computed from the +/// configuration alone, as a record's directory must be. +pub(crate) fn directory(args: &SdkValue, id: &str) -> Result { + let root = (args.get("deploy")) + .and_then(|deploy| deploy.get_text("localDir")) + .map(Path::new) + .ok_or_else(|| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a local Sandbox; its record is used only with \ + LAYERFS_LOCAL_DIR (deploy.localDir) configured" + ), + ) + })?; + let name = sandbox_name(args, id)?; + let plain = |path: &Path| { + !path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + }; + let single = matches!( + Path::new(&name).components().collect::>()[..], + [Component::Normal(_)] + ); + if !root.is_absolute() || !plain(root) || !single { + return Err(refused( + "InvalidInput", + format!("local Sandbox directory {}/{name}", root.display()), + )); + } + Ok(root.join(name)) +} + +/// The refusal for a failed start. Everything a local start makes is its +/// own (a directory and a child process), so each effect is known: the +/// daemon's last log lines are read once, then it is stopped and its +/// directory deleted, each once, and the intent removed. If that cleanup +/// fails, the intent stays and says what is left. +fn abandon( + directory: &Path, + id: &str, + mut cause: String, + sandbox: Option>, +) -> SdkRefusal { + let container = sandbox.as_ref().map(|s| s.identity().to_string()); + let mut log = None; + let mut left = None; + if let Some(mut sandbox) = sandbox { + match sandbox.log_tail(crate::sandbox::LOG_LINES, crate::sandbox::LOG_BYTES) { + Ok(bytes) => { + let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); + cause = format!("{cause}; its daemon's last log lines:\n{text}"); + log = Some(text); + } + Err(error) => cause = format!("{cause}; its daemon log was unreadable: {error:?}"), + } + let removed = sandbox.stop(1).and_then(|()| sandbox.delete()); + left = removed.err().map(|error| { + format!( + "its local Sandbox {} could not be removed: {error:?}", + sandbox.identity() + ) + }); + } + let intent = directory.join("create.intent"); + let (message, removed) = match left { + None => match std::fs::remove_file(&intent) { + Ok(()) => (cause, container.is_some()), + Err(error) => ( + format!("{cause}; its creation intent stays: {error}"), + container.is_some(), + ), + }, + Some(left) => { + let state = format!("creation of Sandbox {id} failed ({cause}); {left}"); + let kept = crate::discovery::save(&intent, SdkValue::text(&state)) + .err() + .map(|error| format!(" (recording it failed: {})", error.message)) + .unwrap_or_default(); + (format!("{state}; its creation intent stays{kept}"), false) + } + }; + SdkRefusal { + code: "Failed".to_owned(), + message, + detail: SdkValue::map([ + ("container", SdkValue::opt_text(container)), + ("removed", SdkValue::Bool(removed)), + ("log", SdkValue::opt_text(log)), + ]), + } +} diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..8bbbc2081c 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -2,7 +2,8 @@ //! project itself is only its home entry (`//`), so a project //! removed there can be initialized again without a restart. //! -//! A Sandbox is one container and its daemon. Its runtime runs commands +//! A Sandbox is one container and its daemon (or, with the local runtime, +//! one daemon process: `runtime`). Its runtime runs commands //! directly; its control connections (`pool`) each carry one exchange at a //! time, and concurrent calls take separate connections. The Sandbox runtime owner (this //! binding) also keeps the background processes it started; the daemon holds @@ -11,7 +12,7 @@ use crate::job::refused; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; +use layerfs_sandbox::CommandIdentity; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -42,10 +43,11 @@ pub(crate) struct Process { pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, - /// Its container's name, `-`. + /// Its container's name, `-` (a local Sandbox's + /// directory name). pub(crate) name: String, pub(crate) directory: std::path::PathBuf, - pub(crate) runtime: Sandbox, + pub(crate) runtime: Box, pub(crate) identity: CommandIdentity, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, @@ -76,7 +78,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// tables as they stand: no daemon call (#352 F-029). A call that names a /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes -/// [`sandbox_refreshed`]. +/// `discovery::refresh`. pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -129,14 +131,6 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { SdkValue::Map(fields) } -/// The Sandbox an operation names, its tables read again from the shared -/// records and the daemon (one status call per mount). -pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { - let state = sandbox(args)?; - crate::discovery::refresh(&state)?; - Ok(state) -} - impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { @@ -154,6 +148,7 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { + crate::unmount::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs new file mode 100644 index 0000000000..a4757f26ab --- /dev/null +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -0,0 +1,156 @@ +//! A Sandbox's runtime: Docker (default), or `LAYERFS_SANDBOX=native` on +//! Linux (root FUSE daemon, nonroot commands) and macOS (loopback NFS, +//! caller identity). Both use the same authenticated daemon control pipeline +//! and the same concrete `layerfs_sandbox::Sandbox` handle. +use crate::job::{failed, refused}; +use crate::output::Spool; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::provider::SandboxRecord; +#[cfg(target_os = "linux")] +use layerfs_sandbox::Image; +use layerfs_sandbox::{CommandIdentity, Sandbox}; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The image a native Sandbox records and reports, and the `runtime` of a +/// record from the former `local` runtime, which is never used (a Docker +/// record has no `runtime`). +pub(crate) const LOCAL: &str = "local"; +/// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local +/// daemon serving FUSE on Linux or loopback NFS on macOS. +pub(crate) const NATIVE: &str = "native"; +/// A namespace Sandbox record's `runtime` (LAYERFS_SANDBOX_BACKEND=namespaces, +/// Linux): a local daemon in its own mount, PID and IPC namespaces and cgroup. +pub(crate) const NAMESPACES: &str = "namespaces"; + +/// Which local backend a native deployment runs inside its Sandbox directory: +/// the plain native one, or the Linux namespace one. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Backend { + Native, + Namespaces, +} + +/// The local backend the deployment selects: `deploy.sandboxBackend` +/// (LAYERFS_SANDBOX_BACKEND), `native` when unset. The namespace backend is +/// Linux only; anywhere else selecting it is refused before any effect. +pub(crate) fn backend(deploy: &SdkValue) -> Result { + match deploy.get_text("sandboxBackend") { + None | Some(NATIVE) => Ok(Backend::Native), + Some(NAMESPACES) if cfg!(target_os = "linux") => Ok(Backend::Namespaces), + Some(NAMESPACES) => Err(refused( + "Unsupported", + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) namespaces is Linux only; use native", + )), + Some(other) => Err(refused( + "InvalidInput", + format!( + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) is native or namespaces, not {other}" + ), + )), + } +} + +/// Whether the deployment selects a local daemon: `deploy.sandbox` unset or +/// `docker` is Docker, `native` is a local daemon; anything else (the +/// former `local` included) is refused before any effect. +pub(crate) fn local(deploy: &SdkValue) -> Result { + match deploy.get_text("sandbox") { + None | Some("docker") => Ok(false), + Some(NATIVE) => Ok(true), + Some(other) => Err(refused( + "InvalidInput", + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not {other}"), + )), + } +} + +/// Whether the deployment selects the native route: FUSE on Linux, +/// loopback NFS on macOS, with no Docker. +pub(crate) fn native(deploy: &SdkValue) -> bool { + deploy.get_text("sandbox") == Some(NATIVE) +} + +/// The image a Sandbox reports: its container's, the digest a namespaces +/// Sandbox's root is an overlay of, or `local` for a native one. +pub(crate) fn image(runtime: &Sandbox) -> String { + match runtime.record() { + Some(SandboxRecord::Docker(docker)) => docker.image, + #[cfg(target_os = "linux")] + Some(SandboxRecord::Namespaces(record)) => record.image().as_str().to_owned(), + _ => LOCAL.to_owned(), + } +} + +/// The image a namespaces deployment's Sandbox root is an overlay of: +/// `deploy.image` (LAYERFS_IMAGE), the same `sha256:<64 hex>` digest the +/// Docker backend uses, and `deploy.imageArchive` (LAYERFS_IMAGE_ARCHIVE), +/// the docker-archive in the bundle that holds it. Both are read before any +/// effect, so a bundle without its image refuses the start. +#[cfg(target_os = "linux")] +pub(crate) fn namespaces_image(deploy: &SdkValue) -> Result<(Image, PathBuf), SdkRefusal> { + let digest = deploy.get_text("image").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.image is not configured (LAYERFS_IMAGE); a namespaces Sandbox's root is the image that digest names", + ) + })?; + let image = Image::parse(digest) + .map_err(|error| refused("InvalidInput", format!("deploy.image: {error}")))?; + let archive = deploy.get_text("imageArchive").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.imageArchive is not configured (LAYERFS_IMAGE_ARCHIVE); a namespaces Sandbox needs the docker-archive the bundle carries", + ) + })?; + let archive = PathBuf::from(archive); + if !archive.is_absolute() { + return Err(refused( + "InvalidInput", + "deploy.imageArchive (LAYERFS_IMAGE_ARCHIVE) must be an absolute path", + )); + } + Ok((image, archive)) +} + +/// Where its commands keep scratch files: `/tmp` in a container, the +/// Sandbox's own `tmp` directory for a native one (deleted with it), so a +/// native Sandbox leaves nothing in this machine's `/tmp`. +pub(crate) fn scratch(runtime: &Sandbox) -> String { + runtime.scratch_directory() +} + +/// Names a background process a command started in a session of its own +/// (`pid`, started at `start`), so it ends with a native Sandbox as it ends +/// with a container. A container's teardown already owns its descendants. +pub(crate) fn track(runtime: &Sandbox, pid: u32, start: &str) -> Result<(), SdkRefusal> { + if matches!(runtime, Sandbox::Docker(_)) { + return Ok(()); + } + let start = start.parse::().map_err(failed)?; + runtime.track(pid, start).map_err(failed) +} + +/// One command to its end: exit code, stdout, stderr. `spool` sends every +/// non-empty stream to its file instead of keeping it inline. +pub(crate) fn run( + runtime: &Sandbox, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, + spool: bool, +) -> Result<(i32, Spool, Spool), SdkRefusal> { + let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); + let request = layerfs_sandbox::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + let exit = runtime + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)?; + Ok((exit.code, stdout, stderr)) +} diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index edf23889a2..45e164f3b3 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -14,15 +14,15 @@ //! the caller's behalf. use crate::job::{failed, refused, Outcome}; use crate::registry::{lock, text, SandboxState, SANDBOXES}; -use layerfs_api::internal::split; -use layerfs_api::sandbox::values; -use layerfs_api::{ - OperationFailure, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase, -}; +use layerfs_api::internal::split_provider; +use layerfs_api::sandbox::{values, ProviderCreate, ProviderFailure}; +use layerfs_api::{OperationFailure, SandboxApi, SandboxCause, SandboxPhase}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::backend::docker::{CreatePhase, Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{CreatePhase, Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -30,12 +30,12 @@ use std::time::Duration; /// Port the daemon listens on inside its container. const PORT: u16 = 30421; /// Wait for the daemon's listener after Start. -const STARTUP: Duration = Duration::from_secs(30); +pub(crate) const STARTUP: Duration = Duration::from_secs(30); /// Wait for one Engine control response. const ENGINE: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. -const LOG_LINES: u16 = 20; -const LOG_BYTES: usize = 2048; +pub(crate) const LOG_LINES: u16 = 20; +pub(crate) const LOG_BYTES: usize = 2048; pub(crate) fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +pub(crate) fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -84,7 +84,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { } /// A container name prefix when the deployment sets none. -const PREFIX: &str = "layerfs-"; +pub(crate) const PREFIX: &str = "layerfs-"; /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. @@ -115,6 +115,15 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result state, - _ => "its record is unreadable".to_owned(), - }; - return Err(refused( - "Unknown", - format!( - "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ - is unknown, so it is neither retried nor taken as done. Remove any container it \ - left (docker ps -a --filter volume={store_volume}), then delete {}", - intent.display() - ), - )); - } + unresolved( + directory, + &format!("docker ps -a --filter volume={store_volume}"), + )?; // The number is claimed before anything is made; a failed start gives it // back (its directory holds no record yet), so the number is used again. let (id, saved_directory) = crate::discovery::claim_next(args)?; @@ -174,6 +170,29 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result<(), SdkRefusal> { + let intent = directory.join("create.intent"); + if intent.exists() { + let state = match crate::discovery::load(&intent) { + Ok(SdkValue::Text(state)) => state, + _ => "its record is unreadable".to_owned(), + }; + return Err(refused( + "Unknown", + format!( + "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ + is unknown, so it is neither retried nor taken as done. Remove any container it \ + left ({left}), then delete {}", + intent.display() + ), + )); + } + Ok(()) +} + /// Starts the claimed Sandbox `id`: container, daemon on the installed /// Store, then its records. fn start_claimed( @@ -204,25 +223,30 @@ fn start_claimed( .map_err(failed)? .naming(&container_name) .map_err(|_| refused("InvalidInput", format!("container name {container_name}")))?; + let provider = Arc::new(Runtime::Docker(DockerProvider::new(docker))); // Read only if this process ends before the intent is resolved. crate::discovery::save( &directory.join("create.intent"), SdkValue::text(format!("creation of Sandbox {id} began and never finished")), )?; - let created = SandboxApi::new(docker).create_file( - SandboxCreate { - deployment: SandboxRequest { - image, - store_volume, - port: PORT, + let created = SandboxApi::with_runtime(provider).create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: container_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), labels: labels(name, id), + startup_timeout: STARTUP, + options: ProviderLaunchOptions::Docker { + image, + store_volume, + port: PORT, + connect_host: crate::pool::daemon_host(args), + }, }, setup: daemon_setup, - executable, controller_private: controller.private, daemon_peer: daemon.public, - startup_wait: STARTUP, - connect_host: crate::pool::daemon_host(args), }, &project.store, &project.manifest, @@ -230,7 +254,7 @@ fn start_claimed( let owner = created.map_err(|failure| abandon(directory, id, failure))?; let container = owner.runtime.identity().to_string(); let instance = owner.startup.instance; - let (runtime, control, identity) = split(owner); + let (runtime, control, identity) = split_provider(owner); let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, host: crate::pool::daemon_host(args), @@ -267,33 +291,46 @@ fn start_claimed( /// Whether a failed create's every effect is known: nothing reached the /// Store volume, and any container it made is acknowledged by identity. -fn settled(failure: &SandboxFailure) -> bool { +fn settled(failure: &ProviderFailure) -> bool { // A Create the Engine refused (409: the container name is taken) made no container. - let unknown_container = matches!( - &failure.cause, - SandboxCause::Create(create) if create.phase == CreatePhase::Create - && !matches!(create.transfer.cause, layerfs_sandbox::RuntimeError::Http(409)) - ); + let unknown_container = match &failure.cause { + SandboxCause::Provider(launch) => match &launch.cause { + layerfs_sandbox::RuntimeError::Docker(docker_failure) => { + match docker_failure.as_ref() { + layerfs_sandbox::backend::docker::ProviderFailure::Create(create) => { + create.phase == CreatePhase::Create + && !matches!( + create.transfer.cause, + layerfs_sandbox::RuntimeError::Http(409) + ) + } + _ => false, + } + } + _ => false, + }, + _ => false, + }; failure.phase != SandboxPhase::Install && failure.installed.is_none() && !unknown_container } /// The refusal for a failed create. A settled one reads its container's log /// tail once, removes the container (stop, then delete, each once) and its /// intent; otherwise the intent stays, rewritten to say what is left. -fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { +fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { let mut cause = format!( "sandbox create stopped at {:?}: {:?}", failure.phase, failure.cause ); let container = failure - .sandbox + .runtime .as_ref() .map(|sandbox| sandbox.identity().to_string()); drop(failure.control.take()); let mut log = None; let left = if !settled(&failure) { Some("its effect on the Store volume or the Engine is unknown".to_owned()) - } else if let Some(sandbox) = failure.sandbox.as_mut() { + } else if let Some(sandbox) = failure.runtime.as_mut() { match sandbox.log_tail(LOG_LINES, LOG_BYTES) { Ok(bytes) => { let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); @@ -344,7 +381,8 @@ fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkR /// `sandbox.create({ image })`: under the project's guard, reclaims the /// project's idle-closed Sandboxes, then starts the lowest free number on /// the call's image (`deploy.image`) and waits for its daemon: -/// `{ sandbox, name, image }`. The only call that starts a Sandbox. Failures +/// `{ sandbox, name, image }`. With `deploy.sandbox` `native` (macOS) it +/// starts a daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures /// are `Failed` or `Unknown`, never retried. pub(crate) fn create(args: &SdkValue) -> Outcome { let project = crate::discovery::project_dir(args)?; @@ -356,7 +394,7 @@ pub(crate) fn create(args: &SdkValue) -> Outcome { ("name", SdkValue::text(&state.name)), ( "image", - SdkValue::text(state.runtime.request().image.as_str()), + SdkValue::text(crate::runtime::image(&state.runtime)), ), ])) } @@ -381,8 +419,16 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { } let state = crate::registry::sandbox_live(args)?; let _guard = crate::discovery::guard(&state.directory)?; - crate::discovery::refresh(&state)?; - let mounted = lock(&state.mounts)?.keys().cloned().collect::>(); + let recoverable = crate::unmount::check_all(&state)?; + crate::discovery::refresh_except(&state, &recoverable)?; + // Explicit close can take over an orphaned/retained unmount, but must + // still refuse every unrelated live Workspace. Stop detaches before + // killing the daemon; Delete independently checks physical detach. + let mounted = lock(&state.mounts)? + .keys() + .filter(|path| !recoverable.contains(path)) + .cloned() + .collect::>(); if !mounted.is_empty() { return Err(SdkRefusal { code: "MountsActive".to_owned(), @@ -397,8 +443,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { }); } let value = crate::discovery::load(&record)?; - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let mut owner = layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request) + let mut owner = crate::discovery::recorded(args, &id, &value)? + .reopen() .map_err(failed)?; owner.stop(STOP_S).map_err(failed)?; owner.delete().map_err(failed)?; @@ -413,7 +459,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { /// `backend.daemon.sandbox.get_log({ sandbox, lines })`: the last `lines` /// lines (1..=2000) of Sandbox `sandbox`'s daemon log, at most -/// [`GET_LOG_BYTES`], read once through the Engine: `{ sandbox, container, +/// [`GET_LOG_BYTES`], read once through the Engine (a local Sandbox's from +/// its log file): `{ sandbox, container, /// text, bytes }`. A stopped container's log is readable until its records /// are reclaimed. Refused `NoSuchSandbox` with no record or no container. /// Read-only: it starts, stops and reclaims nothing. @@ -437,8 +484,7 @@ pub(crate) fn get_log(args: &SdkValue) -> Outcome { crate::discovery::load(&record)? }; let container = text(&value, "container")?.to_owned(); - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let sandbox = layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request); + let sandbox = crate::discovery::recorded(args, &id, &value)?.stopped(); let bytes = match sandbox.log_tail(lines, GET_LOG_BYTES) { Ok(bytes) => bytes, Err(failure) if failure.container_gone() => { @@ -460,7 +506,7 @@ const GET_LOG_BYTES: usize = 64 * 1024; /// Seconds a closing Sandbox's daemon has to exit before the Engine kills it. const STOP_S: u16 = 5; -/// Reclaims Sandbox `id`, whose container a reopen found gone or stopped +/// Reclaims Sandbox `id`, whose container (or local daemon) a reopen found gone or stopped /// (`container`, `stopped`): one unforced Delete of a stopped container, then /// its records, so its number is free again. Called under the project's /// guard; the Sandbox's own guard is taken too. The record is read again @@ -485,8 +531,9 @@ pub(crate) fn reclaim(args: &SdkValue, id: &str, container: &str, stopped: bool) )); } if stopped { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - match layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request).delete() + match crate::discovery::recorded(args, id, &value)? + .stopped() + .delete() { Err(failure) if !failure.container_gone() => return Err(failed(failure)), _ => {} diff --git a/engine/crates/layerfs-node/src/unmount.rs b/engine/crates/layerfs-node/src/unmount.rs new file mode 100644 index 0000000000..71ca6dba05 --- /dev/null +++ b/engine/crates/layerfs-node/src/unmount.rs @@ -0,0 +1,45 @@ +//! Workspace-scoped unmount ownership slot; implemented by the native unmount lease. +use crate::{job::Outcome, registry::SandboxState}; +use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; + +pub(crate) struct Attempt; + +pub(crate) fn check(_state: &SandboxState, _path: &str) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn find(_state: &SandboxState, _path: &str) -> Result, SdkRefusal> { + Ok(None) +} + +pub(crate) fn check_all(_state: &SandboxState) -> Result, SdkRefusal> { + Ok(Vec::new()) +} + +pub(crate) fn begin( + _state: &SandboxState, + _token: WorkspaceToken, + _path: &str, +) -> Result { + Ok(Attempt) +} + +pub(crate) fn finish( + _attempt: &Attempt, + _path: &str, + _outcome: &Outcome, +) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn cancel(_attempt: &Attempt) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn cleanup_failure(outcome: Outcome, _error: SdkRefusal) -> Outcome { + outcome +} + +pub(crate) fn repeat(value: &SdkValue) -> Outcome { + Ok(value.clone()) +} diff --git a/engine/crates/layerfs-node/src/workspace.rs b/engine/crates/layerfs-node/src/workspace.rs index 829df67f71..8dcb0c4ca9 100644 --- a/engine/crates/layerfs-node/src/workspace.rs +++ b/engine/crates/layerfs-node/src/workspace.rs @@ -8,9 +8,7 @@ //! by a mount and reclaimed by the next `sandbox.create` or //! `runtime.sandbox.reclaim`. use crate::job::{failed, Outcome}; -use crate::registry::{ - lock, naming, random, sandbox, sandbox_live, sandbox_refreshed, text, Mount, -}; +use crate::registry::{lock, naming, random, sandbox, sandbox_live, text, Mount}; use crate::sandbox::{operation, WORKSPACES}; use layerfs_api::internal::workspace; use layerfs_api::sandbox::{values, Unmounted}; @@ -96,8 +94,10 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { if let SdkValue::Map(fields) = &mut value { fields.push(("sandbox".to_owned(), SdkValue::text(state.id.as_str()))); fields.push(("container".to_owned(), SdkValue::text(state.name.as_str()))); - let image = state.runtime.request().image.as_str(); - fields.push(("image".to_owned(), SdkValue::text(image))); + fields.push(( + "image".to_owned(), + SdkValue::text(crate::runtime::image(&state.runtime)), + )); } crate::discovery::save_workspace(&state, &mounted, branch)?; lock(&state.mounts)?.insert( @@ -112,15 +112,21 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { /// One recorded Sandbox of the project: running (`Ok(true)`), or found /// gone or stopped (`Ok(false)`, with its container and whether it stopped). -/// Its container is inspected once; no daemon is dialed. -fn observed(record: &SdkValue, directory: &Path) -> Result, SdkRefusal> { +/// Its container (or local daemon process) is inspected once; no daemon +/// is dialed. +fn observed( + args: &SdkValue, + id: &str, + record: &SdkValue, + directory: &Path, +) -> Result, SdkRefusal> { let cached = lock(&crate::registry::SANDBOXES)?.get(directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => { - let (docker, selector, request) = crate::discovery::runtime(record)?; - layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request).map(drop) - } + // a container, or with the local runtime a daemon process + None => crate::discovery::recorded(args, id, record)? + .reopen() + .map(drop), }; match observed { Ok(()) => Ok(None), @@ -152,7 +158,7 @@ fn records(args: &SdkValue) -> Result Result<(), SdkRefusal> { for (id, directory, value) in records(args)? { - if let Some((container, stopped)) = observed(&value, &directory)? { + if let Some((container, stopped)) = observed(args, &id, &value, &directory)? { crate::sandbox::reclaim(args, &id, &container, stopped)?; } } @@ -168,7 +174,7 @@ pub(crate) fn reclaim_gone(args: &SdkValue) -> Result<(), SdkRefusal> { pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { let mut items = Vec::new(); for (id, directory, value) in records(args)? { - if observed(&value, &directory)?.is_some() { + if observed(args, &id, &value, &directory)?.is_some() { continue; } items.push(SdkValue::map([ @@ -198,7 +204,7 @@ pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { pub(crate) fn reclaim(args: &SdkValue) -> Outcome { let mut reclaimed = Vec::new(); for (id, directory, value) in records(args)? { - let Some((container, stopped)) = observed(&value, &directory)? else { + let Some((container, stopped)) = observed(args, &id, &value, &directory)? else { continue; }; let _placing = crate::discovery::guard(&crate::discovery::project_dir(args)?)?; @@ -252,10 +258,19 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { /// `workspace.unmount(ws, { discard?, force? })`: `Busy` while a background /// process this binding started still runs there, then the daemon's answer. pub(crate) fn unmount(args: &SdkValue) -> Outcome { - let state = sandbox_refreshed(args)?; - let _guard = crate::discovery::guard(&state.directory)?; + let state = sandbox(args)?; let path = text(args, "path")?; + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } + let guard = crate::discovery::guard(&state.directory)?; + // Another process may have registered while this call waited for the + // short admission section. Check again before sending anything. + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } let token = state.token(path)?; + crate::discovery::refresh(&state)?; // The runtime owner knows its own processes; the daemon's kernel probe // still answers Busy for any other holder. let running = crate::exec::running_in(&state, path)?; @@ -273,22 +288,59 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { )]), }); } - let value = { + let attempt = crate::unmount::begin(&state, token, path)?; + drop(guard); + // Only this Workspace's lease remains held. Other Workspaces and + // processes can take Sandbox ownership while the daemon helper waits. + let mut sent = false; + let outcome = (|| { let mut control = crate::trace::control(&state.controls)?; + sent = true; let mut api = workspace(&mut control); // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. if args.get_bool("force") { match api.unmount(token, true).map_err(operation)? { - Some(outcome) => values::force_unmounted(&outcome), - None => values::unmounted(&Unmounted::Closed)?, + Some(outcome) => Ok(values::force_unmounted(&outcome)), + None => values::unmounted(&Unmounted::Closed), } } else { - let discard = args.get_bool("discard"); - values::unmounted(&api.unmount_checked(token, discard).map_err(operation)?)? + match api.unmount_checked(token, args.get_bool("discard")) { + Ok(answer) => values::unmounted(&answer), + Err(error) => Err(operation(error)), + } } + })(); + let outcome = match outcome { + Err(error) if error.code == "Missing" => { + Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])) + } + other => other, }; - std::fs::remove_file(crate::discovery::workspace_file(&state, token)).map_err(failed)?; - lock(&state.mounts)?.remove(path); - Ok(value) + // Preserve the daemon's known outcome before any discovery-lock wait. + // A file/cleanup failure is separate from an already successful detach. + let publication = if !sent { + crate::unmount::cancel(&attempt) + } else { + crate::unmount::finish(&attempt, path, &outcome) + }; + if let Err(error) = publication { + return crate::unmount::cleanup_failure(outcome, error); + } + if outcome.is_ok() { + let cleanup = (|| { + let _guard = crate::discovery::guard(&state.directory)?; + match std::fs::remove_file(crate::discovery::workspace_file(&state, token)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + lock(&state.mounts)?.remove(path); + Ok(()) + })(); + if let Err(error) = cleanup { + return crate::unmount::cleanup_failure(outcome, error); + } + } + outcome } diff --git a/engine/crates/layerfs-node/tests/native_discovery.rs b/engine/crates/layerfs-node/tests/native_discovery.rs index f0afde037f..8498ae9424 100644 --- a/engine/crates/layerfs-node/tests/native_discovery.rs +++ b/engine/crates/layerfs-node/tests/native_discovery.rs @@ -32,11 +32,19 @@ fn another_404_is_a_failure_that_removes_nothing() { let home = Home::new(); let engine = engine(&home.0, 404, r#"{"message":"page not found"}"#); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(404\)/); + assert.match(answer.refused.message, /page not found/); + "#, None, Duration::from_secs(5), ); - engine.join().unwrap(); + assert!(engine + .join() + .unwrap() + .starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } @@ -113,11 +121,21 @@ fn a_stopped_container_the_engine_will_not_delete_keeps_its_records() { ], ); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(409\)/); + assert.match(answer.refused.message, /cannot remove container: container is running/); + "#, None, Duration::from_secs(5), ); - assert_eq!(engine.join().unwrap().len(), 2); + let requests = engine.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); + assert!(requests[1].starts_with(&format!( + "DELETE /v1.54/containers/{CID}?force=false&v=false HTTP/1.1\r\n" + ))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } diff --git a/engine/crates/layerfs-node/tests/support/engine.rs b/engine/crates/layerfs-node/tests/support/engine.rs index 6fb656c2d4..302c2f8e38 100644 --- a/engine/crates/layerfs-node/tests/support/engine.rs +++ b/engine/crates/layerfs-node/tests/support/engine.rs @@ -1,5 +1,5 @@ use super::*; -/// The Engine at `home/docker.sock`, answering one request with `status` and `body`. +/// The Engine at `home/docker.sock`, answering one business request with `status` and `body`. pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { let engine = engines(home, vec![(status, body.to_owned())]); thread::spawn(move || engine.join().unwrap().remove(0)) @@ -7,6 +7,7 @@ pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { /// The Engine at `home/docker.sock`, answering each request, one connection /// each, with the next `(status, body)`; it returns their request heads. +/// Version probes get API 1.54 without consuming a response or calling `before`. pub fn engines(home: &Path, responses: Vec<(u16, String)>) -> JoinHandle> { engines_before(home, responses, || {}) } @@ -22,11 +23,9 @@ pub fn engines_before( let mut before = Some(before); let deadline = Instant::now() + Duration::from_secs(3); let mut requests = Vec::new(); - for (status, body) in responses { - let response = format!( - "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); + let mut responses = responses.into_iter().peekable(); + while responses.peek().is_some() { + assert!(Instant::now() < deadline, "bounded engine request deadline"); let stream = loop { match listener.accept() { Ok((stream, _)) => break stream, @@ -43,6 +42,9 @@ pub fn engines_before( stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); let mut request = String::new(); loop { @@ -52,11 +54,23 @@ pub fn engines_before( } request.push_str(&line); } - if let Some(before) = before.take() { - before(); - } + let probe = request.split_whitespace().nth(1) == Some("/version"); + let (status, body) = if probe { + (200, r#"{"ApiVersion":"1.54"}"#.to_owned()) + } else { + if let Some(before) = before.take() { + before(); + } + responses.next().unwrap() + }; + let response = format!( + "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); (&stream).write_all(response.as_bytes()).unwrap(); - requests.push(request); + if !probe { + requests.push(request); + } } requests }) diff --git a/engine/crates/layerfs-sandbox/Cargo.toml b/engine/crates/layerfs-sandbox/Cargo.toml index e75b3fd4ce..a1f1141f86 100644 --- a/engine/crates/layerfs-sandbox/Cargo.toml +++ b/engine/crates/layerfs-sandbox/Cargo.toml @@ -7,4 +7,13 @@ rust-version.workspace = true license.workspace = true [dependencies] -nix = { version = "=0.31.3", features = ["poll", "socket", "fs"] } +nix = { version = "=0.31.3", features = [ + "poll", + "socket", + "fs", + "signal", + "mount", + "user", + "sched", + "process", +] } diff --git a/engine/crates/layerfs-sandbox/PROVIDERS.md b/engine/crates/layerfs-sandbox/PROVIDERS.md new file mode 100644 index 0000000000..85c740797c --- /dev/null +++ b/engine/crates/layerfs-sandbox/PROVIDERS.md @@ -0,0 +1,134 @@ +# Sandbox runtimes + +Status: Current general guide. + +`backend::runtime` holds the concrete runtime choice and the launched owner. +There are no provider or instance traits: each backend is an ordinary concrete +type and the only dispatch is one `match` per method. + +- `Runtime` is one deployment's selection and is kept for every Sandbox it + starts: `Docker(DockerProvider)` (the default), `MacosNative(NativeMacosProvider)`, + `LinuxNative(NativeLinuxProvider)`, `Namespaces(NamespaceProvider)` (Linux + only) and `Windows`, a placeholder with no backend. + `Runtime::{capabilities, local_directory, native_paths, launch, reopen, + reopen_stopped}` dispatch to them. +- `Sandbox` is what `Runtime::launch`/`reopen` return: `Docker(docker::Sandbox)`, + `Native(native::Sandbox)` or `Namespaces(Box)`, with + `identity`, `endpoint`, `run`, `track`, `log_tail`, `inspect`, `stop`, + `delete`, `scratch_directory`, `local_record` and `record`. + +`layerfs_sandbox::{Runtime, Sandbox}` re-export both. `provider` keeps the +inputs and observations they share: `LaunchRequest`, `ProviderLaunchOptions`, +`LocalRecord`, `DockerRecord`, `SandboxRecord`, `ExecExit`, +`SandboxInspection`, `ProviderCapabilities` and the `LaunchPhase`/`LaunchFailure` +pair. The runtime layer has no daemon control dependency. `backend::HelperSpec` +is the namespace helper's line format; it is compiled and tested on every +platform, while only the helper that reads it is Linux only. + +## Docker + +`Runtime::Docker(DockerProvider::new(docker))` adapts the existing Engine client +and `SandboxRequest`. It shares its lifecycle admission, API 1.48–1.54 +negotiation cache, streamed private archive, logs, exact deployment inspection, +and explicit stop/delete. Call `runtime.launch(LaunchRequest { .. })` with the +executable, private configuration, name, explicit startup wait, and +`ProviderLaunchOptions::Docker { image, store_volume, port, connect_host }`; +Docker validates the immutable image digest, existing volume name, port, +connect_host format and labels before any Engine effect. The runtime does not +decode daemon configuration or perform authentication. + +`Sandbox::run` streams stdout/stderr to caller sinks with bounded windows and +returns `ExecExit` only after one inspection knows the root process exit. It +rejects `stdin=true` before effects: interactive users keep the existing +concrete `create_exec/start/into_parts` stream API. EOF alone is not an exit and +no automatic poll follows an unknown exit. `track` needs no host registration +because container teardown owns descendants; scratch is `/tmp` and +`local_record()` is absent. `inspect` checks the recorded deployment but neither +adopts it nor changes lifecycle admission; unknown PID/exit/storage fields stay +absent. + +## Native + +`Runtime::MacosNative(NativeMacosProvider::new(root))` runs the daemon as this +user and serves loopback NFS; `Runtime::LinuxNative(NativeLinuxProvider::new(root))` +runs a root daemon serving FUSE with ordinary commands as a nonroot identity. +Each refuses the wrong platform before root access, and Linux refuses a non-root +caller the same way. `backend::native` owns the one lifecycle both use; only the +platform facts differ: + +- `backend::macos`: `darwin.rs` (`proc_pidinfo`, `sysctl`, the `mount` table and + `umount -f`) and `mounts.rs` (force detach, deepest first). +- `backend::linux`: `process.rs` (`/proc//stat`, the boot id and the + process table) and `mounts.rs` (`/proc/self/mountinfo`, lazy `MNT_DETACH`). + +The provider validates the command identity, the environment it is given and +the daemon executable, then starts one daemon in a private directory with the +platform's kernel marker (`LAYERFS_KERNEL=fuse` on Linux, `nfs` on macOS); it +rejects conflicting overrides. Launch preserves the original single-attempt +process lifecycle and partial owner on failure. `inspect` reports exact process +liveness; unknown exit/storage facts stay absent. Records remain the Node +binding's responsibility; runtimes neither scan directories nor infer project +ownership. + +## Namespaces (Linux) + +`Runtime::Namespaces(NamespaceProvider::new(root))` is the same local root and +daemon control as `LinuxNative`, with each Sandbox's daemon in its own mount, +PID and IPC namespaces and its own cgroup v2 directory. The VM that holds this +kernel is the boundary (one per Store); the namespaces keep Sandboxes on one +Store out of each other. + +- `backend::namespaces::create` makes the Sandbox's private directory, copies + the daemon to `/bin/layerfs-daemon` (0755), writes the setup (0600) + and a `HelperSpec`, makes `/sys/fs/cgroup/layerfs/`, then spawns the + helper's init stage. +- `backend::namespaces::sandbox` owns the launched handle: the init process's + identity, `cgroup.kill` for stop (a `SIGKILL` to everything the Sandbox + started, `populated 0` awaited), cgroup and directory removal on delete, and + reopen by the recorded process plus its cgroup membership. +- `backend::namespaces::exec` runs one command through the helper's exec stage. +- `src/bin/layerfs-sandbox-helper` performs the calls that act on the calling + process, so the product crate stays `unsafe`-free: the init stage creates the + namespaces, joins the cgroup and starts the inner stage (PID 1), which mounts + the private root (the VM's userland read-only, a private `/proc`, `/dev`, + `/dev/pts`, `/dev/shm`, `/tmp`, plus the Sandbox's directory and the Store's + directory at their own paths), pivots, starts the daemon and reaps; the exec + stage enters the namespaces, sets no-new-privs and runs the command as the + requested uid:gid (its own exit is a signal, never a command's status). + +Preconditions are checked before any effect and refused with the existing +`RuntimeError::Unsupported`, never replaced by a weaker Sandbox: root, the +unified cgroup v2 hierarchy (and `cgroup.kill`), `/dev/fuse`, and +`user.max_user_namespaces=0` — without the last one a command could unshare a +user namespace and hold its own copy of the Workspace mount. The runtime never +sets that limit itself. There is no image: the private root binds the VM's own +userland, so the Sandbox runs the daemon and libraries the VM already has. +`ProviderLaunchOptions::Namespaces { command_identity, environment, store }` +names the Store file whose directory is bound in; `SandboxRecord::Namespaces` +keeps the same physical facts as `LocalRecord`. + +## Windows + +`Runtime::Windows` is a placeholder: `capabilities`, `local_directory`, +`native_paths`, `launch`, `reopen` and `reopen_stopped` all refuse with the +existing `RuntimeError::Unsupported` before any effect, and no `Sandbox` variant +exists for it. No Windows host runtime is implemented; see +[`src/backend/windows/README.md`](src/backend/windows/README.md) for the planned +routes. + +## Failure and records + +A `LaunchFailure` retains the acknowledged `Sandbox`, when any. Its cause may be +`RuntimeError::Docker(Box)`: match the typed variant for +the original Create/Upload/Start/Listener receipt, partial IDs, upload work and +pending source bytes. Upload is within existing `LaunchPhase::Create`; inspect +the typed cause to distinguish it. A partially decoded Create ID is never +adopted, and retaining a handle permits an explicit later observation, never a +replay of creation or Start. + +`Runtime::reopen(id, &SandboxRecord)` validates the exact existing deployment or +process and endpoint without starting it; `reopen_stopped` requires the caller's +prior exact stopped observation and admits only non-forcing Delete. Both accept +`SandboxRecord`, which represents `LocalRecord`, `Namespaces(LocalRecord)` or +`DockerRecord`; each runtime refuses another runtime's record before effects. `Sandbox::record()` exposes the persisted deployment facts. No disk +format changes occur: a record without a `runtime` field is Docker's. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs index 0b1ceaa8a6..88ba0e4eb1 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs @@ -1,11 +1,11 @@ //! One-attempt create/start/stop/delete, with no implicit adoption or volume deletion. use super::{ container_types::{CreatePhase, Sandbox, SandboxCreateFailure, SandboxRequest}, - json::{once, Json}, - strings::quoted, topology::SECCOMP, Docker, }; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; use crate::{ContainerId, RuntimeError, WireFailure}; use std::io::{self, Read, Write}; impl SandboxRequest { diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs index 0d207f1049..16ddfe2e62 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs @@ -1,8 +1,6 @@ //! Structurally observed exact container and loopback-published daemon endpoint. -use super::{ - json::{once, Json}, - Sandbox, -}; +use super::Sandbox; +use crate::backend::json::{once, Json}; use crate::{ContainerId, RuntimeError, WireFailure}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; impl Sandbox { @@ -61,6 +59,20 @@ impl Sandbox { if !self.started || self.stop_attempted || self.delete_attempted { return Err(self.invalid("running endpoint admission")); } + let (_, endpoint) = self.observe(true)?; + endpoint.ok_or_else(|| self.invalid("missing daemon endpoint")) + } + /// One exact deployment/state observation; absent optional facts remain absent. + pub fn inspect(&self) -> Result> { + self.observe(false).map(|(inspection, _)| inspection) + } + fn observe( + &self, + require_running: bool, + ) -> Result<(crate::provider::SandboxInspection, Option), Box> { + if self.deleted { + return Err(self.invalid("deleted container observation")); + } let mut r = self .docker .exchange( @@ -80,6 +92,8 @@ impl Sandbox { let mut topology = true; let mut running = None; let mut endpoint = None; + let mut exit_code = None; + let mut process_id = None; let mut j = Json::new(&mut r.body); j.object(|j, key| { if key.equals("Id") { @@ -108,6 +122,20 @@ impl Sandbox { if key.equals("Running") { once(&mut bits, 1)?; running = Some(j.boolean()?); + } else if key.equals("ExitCode") { + once(&mut bits, 16)?; + if !j.null()? { + exit_code = + Some(i32::try_from(j.integer()?).map_err(|_| { + RuntimeError::Protocol("container exit width") + })?); + } + } else if key.equals("Pid") { + once(&mut bits, 32)?; + process_id = Some( + u32::try_from(j.integer()?) + .map_err(|_| RuntimeError::Protocol("container PID width"))?, + ); } else if key.equals("Paused") || key.equals("Restarting") || key.equals("Dead") @@ -126,7 +154,7 @@ impl Sandbox { } Ok(()) })?; - topology &= bits == 15; + topology &= bits & 15 == 15; } else if key.equals("NetworkSettings") { once(&mut fields, 4)?; let mut bits = 0; @@ -137,6 +165,9 @@ impl Sandbox { j.object(|j, key| { if key.equals(&format!("{}/tcp", self.request.port)) { once(&mut selected, 1)?; + if j.null()? { + return Ok(()); + } let mut count = 0; j.array(|j| { count += 1; @@ -213,27 +244,36 @@ impl Sandbox { Ok(()) })?; j.finish()?; - // the exact recorded container, intact but not running: its daemon - // and every mount in it are gone (Docker restarted, for instance) - if observed == Some(self.id) - && fields == 127 - && image - && topology - && running == Some(false) - { - return Err(RuntimeError::Protocol(crate::types::STOPPED)); - } if observed != Some(self.id) || fields != 127 || !image || !topology - || running != Some(true) + || running.is_none() { return Err(RuntimeError::Protocol( "exact running container observation", )); } - endpoint.ok_or(RuntimeError::Protocol("missing daemon endpoint")) + if require_running && running == Some(false) { + return Err(RuntimeError::Protocol(crate::types::STOPPED)); + } + if running == Some(true) && endpoint.is_none() { + return Err(RuntimeError::Protocol("missing daemon endpoint")); + } + Ok(( + crate::provider::SandboxInspection { + identity: self.id, + running: running.expect("required"), + exit_code: if running == Some(false) { + exit_code + } else { + None + }, + process_id: process_id.filter(|pid| *pid != 0), + scratch_bytes: None, + }, + endpoint, + )) })(); result.map_err(|e| { let mut f = self.selected(r.fail(e, None)); diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs index f4d5e88b4a..1a81fc2baf 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs @@ -1,8 +1,6 @@ //! One exact runtime observation, structurally decoded without retaining Arguments. -use super::{ - json::{once, Json}, - Docker, -}; +use super::Docker; +use crate::backend::json::{once, Json}; use crate::{ContainerId, ExecId, ExecInspection, RuntimeError, WireFailure}; impl Docker { /// Reads one coherent runtime snapshot. PID is diagnostic, never a signal target. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs index e0eedb87b1..e02b4a03b2 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs @@ -7,13 +7,14 @@ mod endpoint; mod headers; mod http; mod inspect; -mod json; mod listener; mod mux; +mod provider; +mod provider_exec; +mod provider_failure; mod request; mod socket; mod streams; -mod strings; mod topology; mod version; pub use archive::{UploadFailure, UploadWork}; @@ -22,6 +23,9 @@ pub use container_types::{ }; pub use http::Docker; pub use listener::{ListenerFailure, ListenerObservation}; +pub use provider::{DockerInstance, DockerProvider}; +pub(crate) use provider_exec::run as run_command; +pub use provider_failure::ProviderFailure; pub use request::{ExecCreateFailure, ExecCreated, ExecRequest}; pub use streams::{ Attached, ExecHandle, InputCloseFailure, InputEnd, OutputFailure, OutputProgress, RuntimeInput, diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs new file mode 100644 index 0000000000..d4171e4002 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs @@ -0,0 +1,124 @@ +//! Docker's existing one-attempt runtime as a concrete backend. +use super::{Docker, Sandbox, SandboxRequest}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::fs::File; + +/// Negotiation and all transfers use the supplied client's original policy and version cache. +#[derive(Clone, Debug)] +pub struct DockerProvider { + docker: Docker, +} +/// The original container owner, including its exact attempted/acknowledged lifecycle state. +pub type DockerInstance = Sandbox; +impl DockerProvider { + /// Negotiation and all transfers use the supplied client's original policy and version cache. + pub fn new(docker: Docker) -> Self { + Self { docker } + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "docker", + isolation: IsolationModel::Container, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let mut phase = LaunchPhase::Validate; + let mut instance = None; + let result = (|| { + let ProviderLaunchOptions::Docker { + image, + store_volume, + port, + connect_host, + } = request.options + else { + return Err(RuntimeError::Unsupported( + "Docker requires Docker launch options", + )); + }; + if let Some(host) = &connect_host { + if host.is_empty() || host.contains(['\0', ' ', '\t', '\r', '\n']) { + return Err(RuntimeError::Protocol("Docker connect_host format")); + } + } + let deployment = SandboxRequest { + image, + store_volume, + port, + labels: request.labels, + }; + deployment.check()?; + if request.startup_timeout.is_zero() + || request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + { + return Err(RuntimeError::Protocol("Docker startup wait/private config")); + } + let docker = self.docker.clone().naming(&request.name)?; + // Opening the executable precedes any Engine effect, as in the original API route. + let mut file = File::open(request.daemon_executable)?; + phase = LaunchPhase::Create; + instance = + Some(docker.create_sandbox(deployment).map_err(|f| { + RuntimeError::Docker(Box::new(super::ProviderFailure::Create(f))) + })?); + let owner = instance.as_mut().expect("acknowledged container"); + owner + .upload(&mut file, &request.daemon_config) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Upload(f))))?; + phase = LaunchPhase::Start; + owner + .start() + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Start(f))))?; + phase = LaunchPhase::Listener; + owner + .wait_listener(request.startup_timeout) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Listener(f))))?; + Ok(()) + })(); + match result { + Ok(()) => Ok(instance.expect("launched")), + Err(cause) => Err(Box::new(LaunchFailure { + phase, + instance: instance + .map(|sandbox| Box::new(crate::backend::Sandbox::Docker(sandbox))), + cause, + })), + } + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + Sandbox::reopen(self.docker.clone(), id, record.clone().into()) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + let request: SandboxRequest = record.clone().into(); + request.check().map_err(WireFailure::local)?; + Ok(Sandbox::stopped(self.docker.clone(), id, request)) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs new file mode 100644 index 0000000000..59a415d3a5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs @@ -0,0 +1,100 @@ +//! Ordinary noninteractive provider execution, with streamed output and exact failure custody. +use super::{ProviderFailure, Sandbox}; +use crate::{provider::ExecExit, ExecRequest, RuntimeError, WireFailure}; +use std::io::Write; +fn failure(container: crate::ContainerId, cause: ProviderFailure) -> Box { + // Scalar facts describe only the deciding exchange. Owned pending bytes, + // diagnostics and fence errors remain in the original nested receipt. + let wire = match &cause { + ProviderFailure::ExecCreate(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecStart(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecInspect { failure, .. } => Some(failure.as_ref()), + _ => None, + }; + let mut outer = WireFailure::local(RuntimeError::Protocol("provider execution")); + if let Some(wire) = wire { + outer.attempted = wire.attempted; + outer.sent_bytes = wire.sent_bytes; + outer.request_header_bytes = wire.request_header_bytes; + outer.sent_body_bytes = wire.sent_body_bytes; + outer.status = wire.status; + outer.observed_exec = wire.observed_exec; + outer.observed_container = wire.observed_container; + outer.requested_exec = wire.requested_exec; + } else { + // Attachment/splitting, output and exit observation follow an attempted Start. + outer.attempted = true; + } + outer.requested_container = Some(container); + outer.requested_exec = outer.requested_exec.or_else(|| match &cause { + ProviderFailure::ExecStart(f) => Some(f.created.identity().1), + ProviderFailure::ExecOutput { handle, .. } + | ProviderFailure::ExecInspect { handle, .. } + | ProviderFailure::ExecExitUnknown { handle, .. } => Some(handle.identity().1), + _ => None, + }); + outer.cause = RuntimeError::Docker(Box::new(cause)); + outer +} +pub(crate) fn run( + sandbox: &Sandbox, + request: &ExecRequest, + mut stdout: &mut (dyn Write + Send), + mut stderr: &mut (dyn Write + Send), +) -> Result> { + if request.stdin { + return Err( + sandbox.invalid("provider run has no stdin source; use ordinary Docker streams") + ); + } + let id = sandbox.identity(); + let created = sandbox + .create_exec(request.clone()) + .map_err(|f| failure(id, ProviderFailure::ExecCreate(f)))?; + let attached = created + .start() + .map_err(|f| failure(id, ProviderFailure::ExecStart(f)))?; + let (input, output, handle) = attached + .into_parts() + .map_err(|f| failure(id, ProviderFailure::ExecSplit(f)))?; + // Stdin was disabled in the creation request; dropping it sends no cancellation. + drop(input); + let progress = match output.copy_to(&mut stdout, &mut stderr) { + Ok(progress) => progress, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecOutput { failure: f, handle }, + )) + } + }; + let inspection = match handle.inspect() { + Ok(inspection) => inspection, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + )) + } + }; + let Some(code) = inspection.known_root_exit() else { + return Err(failure( + id, + ProviderFailure::ExecExitUnknown { + inspection, + handle, + progress, + }, + )); + }; + Ok(ExecExit { + code, + pid: inspection.pid, + stdout_bytes: progress.stdout, + stderr_bytes: progress.stderr, + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs new file mode 100644 index 0000000000..c97c9a596d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs @@ -0,0 +1,32 @@ +//! Original Docker stage failures retained across the provider's common error port. +use super::{ + ExecCreateFailure, ExecHandle, ListenerFailure, OutputFailure, OutputProgress, + SandboxCreateFailure, SplitFailure, StartFailure, UploadFailure, +}; +use crate::{ExecInspection, WireFailure}; +/// No failed creation/Start/stream owner is reconstructed, replayed or silently cleaned up. +#[derive(Debug)] +pub enum ProviderFailure { + Create(Box), + Upload(Box), + Start(Box), + Listener(Box), + ExecCreate(Box), + ExecStart(Box), + ExecSplit(Box), + ExecOutput { + failure: Box, + handle: ExecHandle, + }, + ExecInspect { + failure: Box, + handle: ExecHandle, + progress: OutputProgress, + }, + /// EOF is not proof of process exit. The original handle permits explicit observation. + ExecExitUnknown { + inspection: ExecInspection, + handle: ExecHandle, + progress: OutputProgress, + }, +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs index b7622d7efc..5b283c9c85 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs @@ -1,58 +1,11 @@ //! One ordinary non-TTY Exec creation; exact acknowledged runtime custody. -use super::{ - json::{once, Json}, - strings::quoted, - Docker, -}; -use crate::{CommandIdentity, ContainerId, ExecId, RuntimeError, WireFailure}; -use std::{ - fmt, - io::{self, Write}, -}; -/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. -#[derive(Clone)] -pub struct ExecRequest { - pub identity: CommandIdentity, - pub arguments: Vec, - pub environment: Vec, - pub directory: String, - pub stdin: bool, -} -impl fmt::Debug for ExecRequest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ExecRequest") - .field("identity", &self.identity) - .field("arguments_count", &self.arguments.len()) - .field("environment_count", &self.environment.len()) - .field("directory", &self.directory) - .field("stdin", &self.stdin) - .finish() - } -} +use super::Docker; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; +pub use crate::ExecRequest; +use crate::{ContainerId, ExecId, RuntimeError, WireFailure}; +use std::io::{self, Write}; impl ExecRequest { - fn check(&self) -> Result<(), RuntimeError> { - self.identity.check()?; - if self.arguments.is_empty() - || self.arguments[0].is_empty() - || !std::path::Path::new(&self.directory).is_absolute() - || self - .arguments - .iter() - .chain(&self.environment) - .chain(std::iter::once(&self.directory)) - .any(|v| v.as_bytes().contains(&0)) - { - return Err(RuntimeError::Protocol("ordinary command fields")); - } - if self - .environment - .iter() - .any(|v| !v.contains('=') || v.starts_with('=')) - { - return Err(RuntimeError::Protocol("ordinary environment field")); - } - Ok(()) - } fn encode(&self, out: &mut dyn Write) -> io::Result<()> { out.write_all(b"{\"AttachStdout\":true,\"AttachStderr\":true,\"AttachStdin\":")?; out.write_all(if self.stdin { b"true" } else { b"false" })?; diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs index 9ee9ae628d..3626ab3bbe 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs @@ -1,8 +1,6 @@ //! Exact observed daemon identity, security and kernel-access settings, and volume placement. -use super::{ - json::{once, Json}, - SandboxRequest, -}; +use super::SandboxRequest; +use crate::backend::json::{once, Json}; use crate::RuntimeError; use std::io::Read; /// The container's whole system-call filter. It denies creation of a user diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs index ae3504b098..973623a478 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs @@ -1,9 +1,9 @@ //! The Engine API version a client speaks: the daemon's, up to the newest this client targets. use super::{ http::{failure, Mode}, - json::{once, Json}, Docker, }; +use crate::backend::json::{once, Json}; use crate::{ErrorDiagnostic, RuntimeError, WireFailure}; /// The oldest Engine API minor version (1.N) this client speaks: Engine 28.0. /// Before it, uploading the daemon fails (`getent unable to find entry "0:0"`: diff --git a/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs b/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs new file mode 100644 index 0000000000..6e1f4d4e08 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs @@ -0,0 +1,159 @@ +//! The init stage's inputs: absolute paths only, one per line, in a 0600 file: +//! what it mounts, the three directories of the Sandbox root's overlay (the +//! unpacked image as its read-only lower layer, and the Sandbox's own upper +//! and work directories), the daemon it starts, and the cgroup it joins. +//! +//! A format, not a platform behaviour: it is compiled wherever the crate is and +//! tested without root; only the helper that reads it is Linux only. +//! +//! The daemon's setup (its keys and every secret in it) is a separate file the +//! helper never reads; this spec names only what the init stage mounts and +//! runs, so it carries no secret. Values may contain `=` (the key ends at the +//! first one); a path may not contain a control character, because the format +//! is line based. +use crate::RuntimeError; +use std::{ + os::unix::ffi::OsStrExt, + path::{Component, Path, PathBuf}, +}; + +/// The fields, in the order they are written. +const FIELDS: [&str; 9] = [ + "root", "sandbox", "store", "daemon", "config", "cgroup", "lower", "upper", "work", +]; + +/// One init stage's decoded inputs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HelperSpec { + /// The private root the mount namespace pivots to; a tmpfs is mounted on + /// it by the init stage. Sibling of `sandbox`, never inside it. + pub root: PathBuf, + /// The Sandbox directory, bound at its own path inside the private root, + /// so the daemon's setup, overlay and Workspace mounts keep one naming. + pub sandbox: PathBuf, + /// The Store's directory, bound read-write: SQLite's WAL and the writers' + /// directory lock then see the same inode as every other opener. + pub store: PathBuf, + /// The daemon executable inside the private root. + pub daemon: PathBuf, + /// The daemon's setup file, 0600. + pub config: PathBuf, + /// The Sandbox's cgroup v2 directory. + pub cgroup: PathBuf, + /// The unpacked image: the read-only lower layer of the Sandbox's root. + pub lower: PathBuf, + /// The Sandbox's own writable layer, and the overlay's work directory; + /// both inside its private directory, deleted with it. + pub upper: PathBuf, + pub work: PathBuf, +} + +impl HelperSpec { + /// Encodes the six fields; refuses a path that is not absolute, has `.` + /// or `..` components, or holds a control character. + pub fn encode(&self) -> Result, RuntimeError> { + let mut out = Vec::new(); + for (index, (field, path)) in self.fields().into_iter().enumerate() { + check(path)?; + if index >= LAYERS { + check_overlay(path)?; + } + out.extend_from_slice(field.as_bytes()); + out.push(b'='); + let text = path.to_str().ok_or(RuntimeError::Protocol(PATH))?; + out.extend_from_slice(text.as_bytes()); + out.push(b'\n'); + } + Ok(out) + } + + /// Decodes a spec written by [`Self::encode`]: exactly the six fields, + /// each once, in any order. + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MAX { + return Err(RuntimeError::Protocol(PATH)); + } + let text = std::str::from_utf8(bytes).map_err(|_| RuntimeError::Protocol(PATH))?; + let mut values: [Option; 9] = Default::default(); + for line in text.lines() { + let (key, value) = line.split_once('=').ok_or(RuntimeError::Protocol(FIELD))?; + let index = FIELDS + .iter() + .position(|name| *name == key) + .ok_or(RuntimeError::Protocol(FIELD))?; + if values[index].is_some() { + return Err(RuntimeError::Protocol(FIELD)); + } + let path = PathBuf::from(value); + check(&path)?; + values[index] = Some(path); + } + let mut take = |index: usize| values[index].take().ok_or(RuntimeError::Protocol(FIELD)); + let spec = Self { + root: take(0)?, + sandbox: take(1)?, + store: take(2)?, + daemon: take(3)?, + config: take(4)?, + cgroup: take(5)?, + lower: take(6)?, + upper: take(7)?, + work: take(8)?, + }; + for path in [&spec.lower, &spec.upper, &spec.work] { + check_overlay(path)?; + } + Ok(spec) + } + + fn fields(&self) -> [(&'static str, &Path); 9] { + [ + (FIELDS[0], &self.root), + (FIELDS[1], &self.sandbox), + (FIELDS[2], &self.store), + (FIELDS[3], &self.daemon), + (FIELDS[4], &self.config), + (FIELDS[5], &self.cgroup), + (FIELDS[6], &self.lower), + (FIELDS[7], &self.upper), + (FIELDS[8], &self.work), + ] + } +} + +/// The largest spec accepted: six absolute paths, each well under a page. +const MAX: usize = 8192; + +/// Rejects a path the line format cannot carry or the init stage cannot +/// mount: relative, `..`/`.`, a control character, or longer than [`MAX`]. +fn check(path: &Path) -> Result<(), RuntimeError> { + let text = path.to_str().ok_or(RuntimeError::Protocol(PATH))?; + if !path.is_absolute() + || text.len() > MAX + || text + .chars() + .any(|c| c == '\n' || c == '\r' || c == '\0' || c == '\t') + || (path.components()).any(|c| matches!(c, Component::CurDir | Component::ParentDir)) + { + return Err(RuntimeError::Protocol(PATH)); + } + Ok(()) +} + +/// The three fields from here on are the Sandbox root's overlay. +const LAYERS: usize = 6; + +/// The three overlay paths travel in one mount option string, where `,` +/// separates options and `:` would separate layers, so neither may appear. +fn check_overlay(path: &Path) -> Result<(), RuntimeError> { + let bytes = path.as_os_str().as_bytes(); + if bytes.iter().any(|byte| matches!(byte, b',' | b':')) { + return Err(RuntimeError::Protocol("an overlay path holds a `,` or `:`")); + } + Ok(()) +} + +/// Path text is UTF-8, absolute and free of control characters. +const PATH: &str = "helper spec path"; +/// A line that is not one known field, or a field given twice or not at all. +const FIELD: &str = "helper spec field"; diff --git a/engine/crates/layerfs-sandbox/src/backend/identity.rs b/engine/crates/layerfs-sandbox/src/backend/identity.rs new file mode 100644 index 0000000000..8e1c174c73 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/identity.rs @@ -0,0 +1,16 @@ +//! A fresh random identity in the Engine's form (64 lowercase hex, not all +//! zero), so records, names and the cgroup directory treat a local Sandbox +//! like a container. +use crate::{ContainerId, RuntimeError}; +use std::{fs::File, io::Read}; + +pub(crate) fn random() -> Result { + let mut bytes = [0u8; 32]; + File::open("/dev/urandom")?.read_exact(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let text: String = (bytes.iter()) + .flat_map(|b| [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]]) + .map(char::from) + .collect(); + ContainerId::parse(&text) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/json.rs b/engine/crates/layerfs-sandbox/src/backend/json.rs similarity index 100% rename from engine/crates/layerfs-sandbox/src/backend/docker/json.rs rename to engine/crates/layerfs-sandbox/src/backend/json.rs diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs new file mode 100644 index 0000000000..46c121526d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs @@ -0,0 +1,7 @@ +//! Linux platform facts for the native runtime: the daemon runs as root and +//! serves FUSE, so its process facts come from `/proc` and its mounts are +//! detached lazily. Only facts live here; the Sandbox lifecycle is shared +//! with macOS in [`super::native`]. +pub(crate) mod mounts; +mod process; +pub(crate) use process::{boot, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs new file mode 100644 index 0000000000..35f67d00ed --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs @@ -0,0 +1,20 @@ +//! Linux mount table and lazy detach slot; implemented by the Linux native backend. +use std::{ + io, + path::{Path, PathBuf}, +}; + +const UNSUPPORTED: &str = "Linux native mount table"; + +pub(crate) fn under(_directory: &Path) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn filesystem(_path: &Path) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + let _ = filesystem(directory); + under(directory).map(drop) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/process.rs b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs new file mode 100644 index 0000000000..53b1661565 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs @@ -0,0 +1,17 @@ +//! Linux `/proc` process facts slot; implemented by the Linux native backend. +use super::super::native::process::Stat; +use std::io; + +const UNSUPPORTED: &str = "Linux native process table"; + +pub(crate) fn boot() -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn stat(_pid: u32) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn processes() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/listener.rs b/engine/crates/layerfs-sandbox/src/backend/listener.rs new file mode 100644 index 0000000000..c8fe639ce1 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/listener.rs @@ -0,0 +1,68 @@ +//! The daemon's startup line: a bounded wait for the loopback address it +//! prints once it listens. Both local backends spawn a daemon whose output is +//! a log file, so both read that same marker from it, each look taking only +//! what was appended since the last one. Nothing is retried: the wait ends at +//! the marker, at the daemon's exit, or at the caller's deadline. +use crate::RuntimeError; +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom}, + path::Path, + time::{Duration, Instant}, +}; + +/// The daemon's startup line; the address follows it. +pub(crate) const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; +/// Pause between two reads of a starting daemon's output. +const PAUSE: Duration = Duration::from_millis(10); + +/// Waits until `log` names the daemon's loopback address, the daemon is no +/// longer alive (as `alive` reports), or `wait` ends. +pub(crate) fn await_listener( + log: &Path, + wait: Duration, + alive: &dyn Fn() -> bool, +) -> Result { + let deadline = Instant::now() + wait; + let mut file = File::open(log)?; + let mut offset = 0u64; + let mut line = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + file.seek(SeekFrom::Start(offset))?; + loop { + let n = file.read(&mut chunk)?; + if n == 0 { + break; + } + offset += n as u64; + for &byte in &chunk[..n] { + if byte != b'\n' { + // a startup line is short; a longer one is not the marker + if line.len() < 256 { + line.push(byte); + } + continue; + } + if let Some(address) = line.strip_prefix(MARKER) { + let address = std::str::from_utf8(address) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|a| a.ip().is_loopback() && a.port() != 0) + .ok_or(RuntimeError::Protocol("local daemon loopback listener"))?; + return Ok(address.port()); + } + line.clear(); + } + } + if !alive() { + return Err(RuntimeError::Protocol( + "local daemon exited before its listener line", + )); + } + if Instant::now() >= deadline { + return Err(io::Error::from(io::ErrorKind::TimedOut).into()); + } + std::thread::sleep(PAUSE); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs new file mode 100644 index 0000000000..23b45ea94b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs @@ -0,0 +1,28 @@ +//! macOS Darwin process and mount facts slot; implemented by the macOS native backend. +use crate::backend::native::process::Stat; +use std::{ + io, + path::{Path, PathBuf}, +}; + +const UNSUPPORTED: &str = "macOS native process and mount table"; + +pub(crate) fn stat(_pid: u32) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn processes() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn boot() -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn mount_points() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn force_unmount(_point: &Path) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs new file mode 100644 index 0000000000..f9bf26868c --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs @@ -0,0 +1,7 @@ +//! macOS platform facts for the native runtime: the daemon serves loopback +//! NFS as this user, so its process facts come from `proc_pidinfo`/`sysctl` +//! and its mounts from `mount`/`umount -f`. Only facts live here; the Sandbox +//! lifecycle is shared with Linux in [`super::native`]. +mod darwin; +pub(crate) mod mounts; +pub(crate) use darwin::{boot, force_unmount, mount_points, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs new file mode 100644 index 0000000000..2b05e21e2d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs @@ -0,0 +1,16 @@ +//! macOS mount table and forced detach slot; implemented by the macOS native backend. +use std::{ + io, + path::{Path, PathBuf}, +}; + +pub(crate) fn under(_directory: &Path) -> io::Result> { + super::mount_points() +} + +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + for point in under(directory)? { + super::force_unmount(&point)?; + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/mod.rs b/engine/crates/layerfs-sandbox/src/backend/mod.rs index 509d7e29fa..75d331e04a 100644 --- a/engine/crates/layerfs-sandbox/src/backend/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,2 +1,22 @@ -//! Concrete ordinary runtime providers. +//! Concrete ordinary runtime backends and the enum that selects one. pub mod docker; +mod helper_spec; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod identity; +mod json; +#[cfg(target_os = "linux")] +pub(crate) mod linux; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod listener; +#[cfg(target_os = "macos")] +pub(crate) mod macos; +#[cfg(target_os = "linux")] +pub mod namespaces; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub mod native; +pub mod runtime; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod streams; +mod strings; +pub use helper_spec::HelperSpec; +pub use runtime::{Runtime, Sandbox}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs new file mode 100644 index 0000000000..bc72e71cd7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs @@ -0,0 +1,84 @@ +//! One cgroup v2 directory per Sandbox: the whole Sandbox ends with one +//! write to `cgroup.kill`, and nothing it started can leave that group, so a +//! process that left its process group still ends with it. The hierarchy is +//! the host's unified one; there is no second, nested controller setup. +use crate::ContainerId; +use std::{ + fs, + io::{self, Write}, + os::unix::fs::DirBuilderExt, + path::{Path, PathBuf}, +}; + +/// This host's unified cgroup v2 hierarchy. +pub(crate) const ROOT: &str = "/sys/fs/cgroup"; +/// The directory this runtime owns under it, one child per Sandbox. +const GROUP: &str = "layerfs"; + +/// Sandbox `id`'s cgroup directory. +pub(crate) fn path(id: &ContainerId) -> PathBuf { + Path::new(ROOT).join(GROUP).join(id.as_str()) +} + +/// Makes this runtime's own directory once (idempotent) and returns Sandbox +/// `id`'s directory, made here and empty, with the files this runtime needs. +pub(crate) fn create(id: &ContainerId) -> io::Result { + let parent = Path::new(ROOT).join(GROUP); + match fs::DirBuilder::new().mode(0o755).create(&parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + let directory = path(id); + fs::DirBuilder::new().mode(0o755).create(&directory)?; + // a kernel without cgroup.kill (before 5.14) cannot end a Sandbox as a whole + if !directory.join("cgroup.kill").exists() { + let _ = fs::remove_dir(&directory); + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "this kernel's cgroup v2 has no cgroup.kill", + )); + } + Ok(directory) +} + +/// Ends every process of the Sandbox with one write: the kernel sends +/// `SIGKILL` to all of them, descendants included, however they left their +/// process groups. +pub(crate) fn kill(directory: &Path) -> io::Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .open(directory.join("cgroup.kill"))?; + file.write_all(b"1") +} + +/// The number of processes the Sandbox's group still has. +pub(crate) fn populated(directory: &Path) -> io::Result { + let events = fs::read_to_string(directory.join("cgroup.events"))?; + let mut count = None; + for line in events.lines() { + if let Some(value) = line.strip_prefix("populated ") { + count = value.trim().parse().ok(); + } + } + count.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "cgroup.events populated")) +} + +/// Removes the Sandbox's directory once nothing is in it. A group that still +/// has processes is refused rather than reported as gone. +pub(crate) fn remove(directory: &Path) -> io::Result<()> { + match populated(directory) { + Ok(0) => {} + Ok(_) => { + return Err(io::Error::other( + "a process of the namespaces Sandbox still runs after SIGKILL", + )) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + match fs::remove_dir(directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + other => other, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs new file mode 100644 index 0000000000..0ad0c7c5c9 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs @@ -0,0 +1,283 @@ +//! One namespace Sandbox start: its private directory (setup, executable, +//! overlay, mounts), its cgroup and one spawn of the init stage, then the +//! wait for its daemon's listener. Nothing is retried, and a failure returns +//! what it made so the caller can end and delete it once. +use super::{ + cgroup, host, + paths::Paths, + rootfs::{self, ImageRoot}, + sandbox::Sandbox, +}; +use crate::backend::listener; +use crate::backend::native::{search::search_path, Local, LocalPhase, LocalProcess}; +use crate::backend::{identity, HelperSpec}; +use crate::{CommandIdentity, Image, RuntimeError}; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, +}; + +/// What one namespaces launch was asked for, after validation. +#[derive(Clone, Debug)] +pub(crate) struct Options { + pub(crate) command_identity: CommandIdentity, + pub(crate) environment: Vec<(String, String)>, + /// The exact Store file the daemon opens; its directory is bound in. + pub(crate) store: PathBuf, + /// The image digest the Sandbox's root is an overlay of, and the + /// docker-archive in the bundle that holds it. + pub(crate) image: Image, + pub(crate) archive: PathBuf, +} + +/// A failed start. `sandbox` holds what it made (its directory, cgroup and, +/// once spawned, its init stage) for the caller to end and delete once. +#[derive(Debug)] +pub(crate) struct CreateFailure { + pub(crate) phase: LocalPhase, + pub(crate) sandbox: Option, + pub(crate) cause: RuntimeError, +} + +/// The init stage of `request`'s Sandbox, started once over `local`'s root. +pub(crate) fn create( + local: &Local, + helper: &Path, + request: &crate::provider::LaunchRequest, + options: &Options, +) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(CreateFailure { + phase, + sandbox, + cause, + }) + }; + let (paths, image) = start_request(local, helper, request, options) + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let cgroup = cgroup::path(&id); + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.local.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted( + id, + paths.clone(), + cgroup.clone(), + request.labels.clone(), + helper.to_owned(), + image.clone(), + ); + let log = match prepare(&paths, request, options, &id, &image) { + Ok(log) => log, + Err(cause) => return Err(fail(LocalPhase::Create, Some(sandbox), cause)), + }; + match cgroup::create(&id) { + Ok(_) => {} + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + } + let spawned = log.try_clone().and_then(|output| { + Command::new(helper) + .arg("init") + .arg("--spec") + .arg(&paths.spec) + .env_clear() + .env("PATH", search_path()) + .envs(options.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.local.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + match LocalProcess::observe(child.id()) { + Ok(init) => sandbox.init = Some(init), + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + } + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.local.log, request.startup_timeout, &|| { + sandbox.reap(); + sandbox.init.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } +} + +/// Everything a start refuses before it makes anything: the request's own +/// fields, this host's runtime prerequisites, the Store's directory against +/// the private root's, the helper, and the image the bundle carries. Getting +/// the image's root may unpack it, once per digest for the whole runtime root. +fn start_request( + local: &Local, + helper: &Path, + request: &crate::provider::LaunchRequest, + options: &Options, +) -> Result<(Paths, ImageRoot), RuntimeError> { + let paths = { + let named = local.clone().naming(&request.name)?; + Paths::of(named.paths()?)? + }; + host::require_root()?; + host::require_user_namespaces_blocked()?; + host::require_cgroup_v2()?; + host::require_fuse()?; + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + host::require_overlay_upper(root)?; + options.command_identity.check()?; + let executable = &request.daemon_executable; + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("namespaces daemon executable")); + } + let helper_metadata = fs::metadata(helper)?; + if !helper.is_absolute() || !helper_metadata.is_file() || helper_metadata.len() == 0 { + return Err(RuntimeError::Protocol("namespaces helper executable")); + } + if request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + || request.startup_timeout.is_zero() + { + return Err(RuntimeError::Protocol( + "namespaces daemon setup/startup wait", + )); + } + check_environment(&options.environment)?; + check_store(&paths, &options.store)?; + for path in [&paths.local.directory, &paths.root] { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + Ok(_) => return Err(RuntimeError::Protocol("namespaces Sandbox path exists")), + } + } + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + let image = rootfs::ensure(&options.archive, &options.image, &rootfs::cache(root))?; + Ok((paths, image)) +} + +/// The daemon starts with the search path and nothing else but these: no +/// `PATH` override, and only this backend's kernel choice. +fn check_environment(environment: &[(String, String)]) -> Result<(), RuntimeError> { + let invalid = environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != "fuse") + }); + if invalid { + return Err(RuntimeError::Protocol("namespaces environment")); + } + Ok(()) +} + +/// The Store's directory is bound into every Sandbox, so it must not hold the +/// runtime root (which would expose every Sandbox's directory to all of them) +/// and must be a real directory that is not the filesystem root. +fn check_store(paths: &Paths, store: &Path) -> Result<(), RuntimeError> { + let directory = (store.parent()) + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or(RuntimeError::Protocol("namespaces Store file"))?; + let metadata = fs::metadata(directory)?; + let root = paths + .local + .directory + .parent() + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + if !store.is_absolute() + || !metadata.is_dir() + || directory == Path::new("/") + || (store.components()).any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + || root.starts_with(directory) + || paths.local.directory.starts_with(directory) + || paths.root.starts_with(directory) + { + return Err(RuntimeError::Protocol( + "namespaces Store directory (must not hold the runtime root)", + )); + } + Ok(()) +} + +/// The Sandbox's own directories and files, its copied executable, its init +/// spec and the empty log; returns the log file. Its overlay's writable layer +/// and work directory are made here, empty: the init stage mounts them with +/// the image underneath. +fn prepare( + paths: &Paths, + request: &crate::provider::LaunchRequest, + options: &Options, + id: &crate::ContainerId, + image: &ImageRoot, +) -> Result { + fs::set_permissions(&paths.local.directory, Permissions::from_mode(0o755))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + for file in [&paths.local.config, &paths.local.overlay] { + let parent = file + .parent() + .ok_or(RuntimeError::Protocol("namespaces path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + for directory in [&paths.bin_directory, &paths.local.mounts] { + fs::DirBuilder::new().mode(0o755).create(directory)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.local.tmp)?; + fs::set_permissions(&paths.local.tmp, Permissions::from_mode(0o1777))?; + fs::DirBuilder::new().mode(0o755).create(&paths.root)?; + for directory in [&paths.upper, &paths.work] { + fs::DirBuilder::new().mode(0o700).create(directory)?; + } + private(&paths.local.config)?.write_all(&request.daemon_config)?; + fs::copy(&request.daemon_executable, &paths.bin)?; + fs::set_permissions(&paths.bin, Permissions::from_mode(0o755))?; + let spec = HelperSpec { + root: paths.root.clone(), + sandbox: paths.local.directory.clone(), + store: (options.store.parent()) + .ok_or(RuntimeError::Protocol("namespaces Store directory"))? + .to_owned(), + daemon: paths.bin.clone(), + config: paths.local.config.clone(), + cgroup: cgroup::path(id), + lower: image.directory().to_owned(), + upper: paths.upper.clone(), + work: paths.work.clone(), + }; + private(&paths.spec)?.write_all(&spec.encode()?)?; + Ok(private(&paths.local.log)?) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs new file mode 100644 index 0000000000..d17372a70e --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs @@ -0,0 +1,120 @@ +//! One ordinary command in a namespace Sandbox: the helper enters the +//! Sandbox's mount, PID and IPC namespaces, becomes uid:gid with no +//! capabilities and no-new-privs, and runs the command there. The streams are +//! the helper's own pipes; nothing of this process's environment but the +//! search path reaches it, and no stdin is offered. +use super::sandbox::Sandbox; +use crate::backend::native::search::search_path; +use crate::backend::streams; +use crate::provider::ExecExit; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +impl Sandbox { + /// Runs `request` once while this Sandbox's init stage runs, as its + /// uid:gid, in its directory inside the Sandbox, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + let init = self + .init + .ok_or_else(|| self.invalid("running command admission"))?; + self.endpoint()?; + let mut command = Command::new(&self.helper); + command + .arg("exec") + .arg("--init") + .arg(init.pid.to_string()) + .arg("--cgroup") + .arg(&self.cgroup) + .arg("--uid") + .arg(request.identity.uid.to_string()) + .arg("--gid") + .arg(request.identity.gid.to_string()) + .arg("--cwd") + .arg(&request.directory) + .arg("--") + .args(&request.arguments) + .env_clear() + .env("PATH", search_path()) + .current_dir(self.directory()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = (status.code()) + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("namespaces command exit status"))?; + if status.code().is_none() { + // the helper ends itself with a signal when it cannot set the + // command up (or the Sandbox ended under it), so this is a + // reported failure, never a command's exit code + return Err(self.invalid("namespaces exec: the helper ended without a status")); + } + Ok(ExecExit { + code, + pid: Some(pid), + stdout_bytes: out_bytes, + stderr_bytes: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs new file mode 100644 index 0000000000..0107cf02bd --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs @@ -0,0 +1,115 @@ +//! What this host must provide before a namespace Sandbox may start, and the +//! two host facts its lifecycle reads: whether user namespaces are blocked +//! and which cgroup a process is in. Every condition is checked before any +//! effect; a host that cannot meet them is refused, never quietly given a +//! weaker Sandbox. +use crate::RuntimeError; +use std::{fs, os::unix::fs::FileTypeExt, path::Path}; + +/// The kernel's limit on user namespaces; `0` blocks them host-wide. +const MAX_USER_NAMESPACES: &str = "/proc/sys/user/max_user_namespaces"; +/// This host's unified cgroup v2 hierarchy. +const CGROUP_ROOT: &str = "/sys/fs/cgroup"; + +/// The init stage must create mount, PID and IPC namespaces and mount the +/// daemon's private root, which needs the real root. +pub(crate) fn require_root() -> Result<(), RuntimeError> { + if nix::unistd::Uid::effective().is_root() { + Ok(()) + } else { + Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime runs its daemon as root (mount, PID and IPC namespaces, \ + /dev/fuse); start it as root inside the project's VM", + )) + } +} + +/// Refuses to start when a command could create a user namespace: it would +/// hold its own copy of the Workspace mount, outside the daemon's unmount. +/// The block is host-wide (`user.max_user_namespaces = 0`), set by whoever +/// owns the VM; this runtime never sets it. +pub(crate) fn require_user_namespaces_blocked() -> Result<(), RuntimeError> { + let text = fs::read_to_string(MAX_USER_NAMESPACES).map_err(|_| { + RuntimeError::Unsupported( + "the namespaces Sandbox runtime cannot read /proc/sys/user/max_user_namespaces, so it \ + cannot prove that user namespaces are blocked", + ) + })?; + match text.trim().parse::() { + Ok(0) => Ok(()), + Ok(_) => Err(RuntimeError::Unsupported( + "user namespaces are not blocked; a command could unshare one and keep its own copy of \ + the Workspace mount. Set user.max_user_namespaces=0 in the project's VM before a \ + namespaces Sandbox starts", + )), + Err(_) => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime cannot parse /proc/sys/user/max_user_namespaces, so it \ + cannot prove that user namespaces are blocked", + )), + } +} + +/// The unified cgroup v2 hierarchy, which ends a Sandbox as a whole. +pub(crate) fn require_cgroup_v2() -> Result<(), RuntimeError> { + let controllers = Path::new(CGROUP_ROOT).join("cgroup.controllers"); + match fs::read_to_string(&controllers) { + Ok(_) => Ok(()), + Err(_) => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs the unified cgroup v2 hierarchy at /sys/fs/cgroup", + )), + } +} + +/// The daemon serves FUSE from inside the Sandbox. +pub(crate) fn require_fuse() -> Result<(), RuntimeError> { + match fs::metadata("/dev/fuse") { + Ok(metadata) if metadata.file_type().is_char_device() => Ok(()), + _ => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs a character device /dev/fuse in the project's VM", + )), + } +} + +/// The Sandbox's root is an overlay whose writable layer is the Sandbox's own +/// directory under this runtime root, so the root must be on a filesystem that +/// can be an overlay upper. overlayfs refuses another overlay or a network +/// filesystem there, and a host that cannot hold it is refused before any +/// Sandbox is made rather than failing inside the Sandbox's mount. +pub(crate) fn require_overlay_upper(root: &Path) -> Result<(), RuntimeError> { + let resolved = fs::canonicalize(root)?; + match super::super::linux::mounts::filesystem(&resolved) { + Ok(kind) if !UNSUITABLE.contains(&kind.as_str()) && !kind.starts_with("fuse") => Ok(()), + // an unreadable mount table is no proof either way: the mount itself + // then reports what is wrong, and nothing is invented here + Err(_) => Ok(()), + Ok(_) => Err(RuntimeError::Unsupported( + "the namespaces runtime root is on a filesystem that cannot hold a Sandbox root's writable layer (overlayfs or a network filesystem); put LAYERFS_LOCAL_DIR on a local filesystem such as the project VM's ext4, xfs or btrfs disk", + )), + } +} + +/// Filesystems an overlay upper layer cannot live on. +const UNSUITABLE: [&str; 7] = ["overlay", "nfs", "nfs4", "cifs", "smb3", "9p", "virtiofs"]; + +/// The cgroup of `pid`, as the kernel reports it, e.g. `0::/layerfs/`. +pub(crate) fn cgroup_of(pid: u32) -> std::io::Result { + fs::read_to_string(format!("/proc/{pid}/cgroup")) +} + +/// Whether the live process `pid` is in `directory`'s cgroup: its recorded +/// start time can outlive a reboot-free id reuse, but its cgroup membership +/// cannot be inherited from a sandbox that never made it. +pub(crate) fn in_cgroup(pid: u32, directory: &Path) -> bool { + let Ok(table) = cgroup_of(pid) else { + return false; + }; + let relative = directory + .strip_prefix(CGROUP_ROOT) + .ok() + .and_then(|r| r.to_str()); + let Some(relative) = relative else { + return false; + }; + let suffix = format!("/{relative}"); + (table.lines()).any(|line| line.ends_with(&suffix)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs new file mode 100644 index 0000000000..72a70747a3 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs @@ -0,0 +1,23 @@ +//! Linux namespace Sandboxes: each Sandbox is one daemon inside its own +//! mount, PID and IPC namespaces and its own cgroup v2 directory, on the one +//! kernel and one Store disk of the project's VM. Commands enter those +//! namespaces as a nonroot identity with no capabilities and no-new-privs, +//! and `cgroup.kill` ends everything the Sandbox started. +//! +//! The lifecycle is the local one (`crate::backend::native`): a private +//! directory per Sandbox, one daemon start and one listener wait, a record of +//! the daemon's exact process and its port. What this backend adds is outside +//! that daemon: the helper that builds the private root, the namespaces and +//! the cgroup. Nothing here is retried and no effect is undone implicitly. +mod cgroup; +mod create; +mod exec; +mod host; +mod paths; +mod provider; +pub mod rootfs; +mod sandbox; + +pub use provider::NamespaceProvider; +pub use rootfs::ImageRoot; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs new file mode 100644 index 0000000000..469f5f4f0f --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs @@ -0,0 +1,56 @@ +//! Where one namespace Sandbox keeps its parts: the same private directory a +//! local Sandbox uses, plus the executable it runs and the private root its +//! mount namespace pivots to. +use crate::backend::native::LocalPaths; +use crate::RuntimeError; +use std::path::PathBuf; + +/// The daemon executable inside the Sandbox's own `bin` directory, copied +/// there once per Sandbox so nothing about it is read from the host later. +pub(crate) const DAEMON: &str = "layerfs-daemon"; + +/// One Sandbox's paths, all fixed from its directory name. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Paths { + /// The Sandbox directory with the setup, overlay, mounts, log and `tmp`. + pub(crate) local: LocalPaths, + /// `/bin`, 0755. + pub(crate) bin_directory: PathBuf, + /// `/bin/`, 0755 and root-owned. + pub(crate) bin: PathBuf, + /// The init stage's inputs, `/config/helper.spec`, 0600. + pub(crate) spec: PathBuf, + /// The private root, `.root`; the init stage mounts the + /// Sandbox's overlay on it and pivots to it, so it is empty again once the + /// namespaces end. + pub(crate) root: PathBuf, + /// The Sandbox's own writable layer of that overlay, + /// `/overlay/upper`, and the overlay's work directory beside + /// it: root-owned 0700, and deleted with the directory. + pub(crate) upper: PathBuf, + pub(crate) work: PathBuf, +} + +impl Paths { + /// The paths of the Sandbox named by `local`: its directory name is the + /// Sandbox's name, and its sibling `.root` is the private root. + pub(crate) fn of(local: LocalPaths) -> Result { + if (local.directory.file_name()) + .and_then(|name| name.to_str()) + .is_none() + { + return Err(RuntimeError::Protocol("namespaces Sandbox directory name")); + } + let mut root = local.directory.clone().into_os_string(); + root.push(".root"); + Ok(Self { + bin_directory: local.directory.join("bin"), + bin: local.directory.join("bin").join(DAEMON), + spec: local.directory.join("config").join("helper.spec"), + root: PathBuf::from(root), + upper: local.directory.join("overlay").join("upper"), + work: local.directory.join("overlay").join("work"), + local, + }) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs new file mode 100644 index 0000000000..794a5c9ffa --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs @@ -0,0 +1,229 @@ +//! The Linux namespace runtime as a concrete backend: the same local runtime +//! root and daemon control as the native one, with each Sandbox's daemon in +//! its own mount, PID and IPC namespaces and its own cgroup, and its commands +//! entering them as a nonroot identity. The VM that holds this kernel is the +//! boundary; the namespaces keep Sandboxes on one Store out of each other. +use super::{ + cgroup, + create::{self, CreateFailure, Options}, + host, + paths::Paths, + rootfs, + sandbox::Sandbox, +}; +use crate::backend::native::{Local, LocalPaths}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::{ + env, + path::{Path, PathBuf}, +}; + +/// The helper executable, installed beside the daemon it starts. +const HELPER: &str = "layerfs-sandbox-helper"; +/// Names the helper explicitly, when it is not beside the daemon. +const HELPER_ENVIRONMENT: &str = "LAYERFS_SANDBOX_HELPER"; + +/// A Linux namespace runtime over one local root directory. +#[derive(Clone, Debug)] +pub struct NamespaceProvider { + local: Local, +} + +impl NamespaceProvider { + /// Selects `root`, the parent of every Sandbox directory, and refuses a + /// host that cannot hold namespace Sandboxes before any effect: not root, + /// user namespaces not blocked, no cgroup v2, or no `/dev/fuse`. + pub fn new(root: impl AsRef) -> Result { + let local = Local::new(&root)?; + host::require_root()?; + host::require_user_namespaces_blocked()?; + host::require_cgroup_v2()?; + host::require_fuse()?; + host::require_overlay_upper(root.as_ref())?; + Ok(Self { local }) + } + + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + + /// What this backend supports. The isolation boundary is the VM that + /// holds this one kernel (one per Store); a Sandbox inside it owns its + /// mount, PID and IPC namespaces and its cgroup, not a VM of its own. + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "namespaces-linux", + isolation: IsolationModel::MicroVm, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + + /// Starts one Sandbox once; a failed start keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Namespaces { + command_identity, + environment, + store, + image, + archive, + } = request.options.clone() + else { + return Err(fail(RuntimeError::Unsupported( + "namespaces provider requires namespaces launch options", + ))); + }; + let helper = helper(Some(&request.daemon_executable)).map_err(fail)?; + self.local.clone().naming(&request.name).map_err(fail)?; + let options = Options { + command_identity, + environment, + store, + image, + archive, + }; + create::create(&self.local, &helper, &request, &options).map_err(|failure| { + let CreateFailure { + phase, + sandbox, + cause, + } = *failure; + Box::new(LaunchFailure { + phase: match phase { + crate::backend::native::LocalPhase::Validate => LaunchPhase::Validate, + crate::backend::native::LocalPhase::Create => LaunchPhase::Create, + crate::backend::native::LocalPhase::Start => LaunchPhase::Start, + crate::backend::native::LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: sandbox.map(|sandbox| { + Box::new(crate::backend::Sandbox::Namespaces(Box::new(sandbox))) + }), + cause, + }) + }) + } + + /// Reopen starts nothing: it checks the recorded init process, its cgroup + /// and its listener. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let record = record.namespaces().ok_or_else(|| { + WireFailure::local(RuntimeError::Unsupported( + "namespaces provider requires a namespaces record", + )) + })?; + let paths = self + .record_paths(&record.local().name) + .map_err(WireFailure::local)?; + let image = self.image(record).map_err(WireFailure::local)?; + let helper = helper(None).map_err(WireFailure::local)?; + let recorded = super::sandbox::Recorded { + init: super::sandbox::process(record.local()), + port: record.local().port, + labels: record.local().labels.clone(), + image, + }; + Sandbox::reopen(paths, cgroup::path(&id), id, helper, recorded) + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let record = record.namespaces().ok_or_else(|| { + WireFailure::local(RuntimeError::Unsupported( + "namespaces provider requires a namespaces record", + )) + })?; + let paths = self + .record_paths(&record.local().name) + .map_err(WireFailure::local)?; + let image = self.image(record).map_err(WireFailure::local)?; + let helper = helper(None).map_err(WireFailure::local)?; + let recorded = super::sandbox::Recorded { + init: super::sandbox::process(record.local()), + port: record.local().port, + labels: record.local().labels.clone(), + image, + }; + Ok(Sandbox::stopped( + paths, + cgroup::path(&id), + id, + helper, + recorded, + )) + } + + fn record_paths(&self, name: &str) -> Result { + Paths::of(self.paths(name)?) + } + + /// The record's image, as the root this runtime already holds: a reopen + /// starts and unpacks nothing, so a record whose image is gone is refused + /// rather than given a different root. + fn image( + &self, + record: &crate::provider::NamespaceRecord, + ) -> Result { + let paths = self.record_paths(&record.local().name)?; + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + rootfs::lookup(record.image(), &rootfs::cache(root)) + } +} + +/// The helper this backend runs: `LAYERFS_SANDBOX_HELPER` when set, else +/// beside `daemon` or beside this process (or its Cargo test runner). +fn helper(daemon: Option<&Path>) -> Result { + if let Some(path) = env::var_os(HELPER_ENVIRONMENT) { + let path = PathBuf::from(path); + if let Ok(metadata) = std::fs::metadata(&path) { + if path.is_absolute() && metadata.is_file() && metadata.len() > 0 { + return Ok(path); + } + } + } + let mut candidates = Vec::new(); + if let Some(parent) = daemon.and_then(Path::parent) { + candidates.push(parent.join(HELPER)); + } + if let Ok(exe) = env::current_exe() { + if let Some(parent) = exe.parent() { + candidates.push(parent.join(HELPER)); + if let Some(grandparent) = parent.parent() { + candidates.push(grandparent.join(HELPER)); + } + } + } + for candidate in candidates { + if let Ok(metadata) = std::fs::metadata(&candidate) { + if candidate.is_absolute() && metadata.is_file() && metadata.len() > 0 { + return Ok(candidate); + } + } + } + Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs the layerfs-sandbox-helper executable beside the daemon (or LAYERFS_SANDBOX_HELPER set)", + )) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs new file mode 100644 index 0000000000..3afd573689 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs @@ -0,0 +1,330 @@ +//! One docker-archive: its `manifest.json`, the digest it names its config by, +//! and its layers, unpacked into the tree bottom first. +use super::{ + layers::apply, + refuse, + tar::{path_of, Kind, Tape}, + PATH_MAX, +}; +use crate::{backend::json::Json, Image, RuntimeError}; +use std::{ + fs::{self, File, OpenOptions}, + io::{self, BufReader, Read, Seek, SeekFrom}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, +}; + +/// The largest `manifest.json` accepted: it names a config and a few layers. +const MANIFEST_MAX: usize = 1 << 20; +/// The largest number of layers one archive may name. +const LAYERS_MAX: usize = 1024; + +/// One image archive's decided layout: the config blob and the layers, in the +/// order they are applied. +#[derive(Debug)] +struct Manifest { + config: PathBuf, + layers: Vec, +} + +/// Where one blob of the archive is: the offset its bytes start at, and how +/// many there are. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Blob { + offset: u64, + size: u64, +} + +/// What a layer's first bytes say it is. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Magic { + Tar, + Gzip, + Zstd, +} + +/// Unpacks `archive`'s layers, bottom first, into `root`. +pub(super) fn unpack(archive: &Path, image: &Image, root: &Path) -> Result<(), RuntimeError> { + let manifest = manifest(archive, image)?; + let mut wanted = Vec::with_capacity(manifest.layers.len() + 1); + wanted.push(manifest.config.clone()); + wanted.extend(manifest.layers.iter().cloned()); + let blobs = locate(archive, &wanted)?; + // the config digest is the name the archive gave that blob; it was checked + // before this point, so only its presence is still open here + if blobs.first().copied().flatten().is_none() { + return Err(refuse(format!( + "the image archive has no config blob {}", + manifest.config.display() + ))); + } + for (index, layer) in manifest.layers.iter().enumerate() { + let blob = blobs.get(index + 1).copied().flatten().ok_or_else(|| { + refuse(format!( + "the image archive has no layer {}", + layer.display() + )) + })?; + extract_layer(archive, blob, &layer.display().to_string(), root)?; + } + Ok(()) +} + +/// The archive's `manifest.json`, as the entry whose config digest is `image`. +fn manifest(archive: &Path, image: &Image) -> Result { + let located = locate(archive, &[PathBuf::from("manifest.json")])?; + let blob = located + .first() + .copied() + .flatten() + .ok_or_else(|| refuse("the image archive has no manifest.json"))?; + if blob.size > MANIFEST_MAX as u64 { + return Err(refuse("the image archive's manifest.json is too long")); + } + read_blob(archive, blob, MANIFEST_MAX).and_then(|bytes| parse_manifest(&bytes, image)) +} + +/// Reads `manifest.json` and keeps the entry whose config digest is `image`. +fn parse_manifest(bytes: &[u8], image: &Image) -> Result { + let mut found: Option = None; + let mut named: Vec = Vec::new(); + let mut json = Json::new(bytes); + json.array(|element| { + let mut config: Option> = None; + let mut layers: Option>> = None; + let mut seen: u8 = 0; + element.object(|reader, key| { + if key.overflow { + return Ok(()); + } + match key.text() { + "Config" => { + crate::backend::json::once(&mut seen, 1)?; + let mut text = Vec::new(); + reader.text_into(&mut text, PATH_MAX)?; + config = Some(text); + } + "Layers" => { + crate::backend::json::once(&mut seen, 2)?; + let mut items = Vec::new(); + reader.array(|entry| { + if items.len() == LAYERS_MAX { + return Err(refuse("the image archive names too many layers")); + } + let mut text = Vec::new(); + entry.text_into(&mut text, PATH_MAX)?; + items.push(text); + Ok(()) + })?; + layers = Some(items); + } + _ => reader.skip(0)?, + } + Ok(()) + })?; + let (Some(config), Some(layers)) = (config, layers) else { + return Ok(()); + }; + let config = path_of(&config)?; + if named.len() < 8 { + named.push(config.display().to_string()); + } + if found.is_none() && check_config(image, &config).is_ok() { + found = Some(Manifest { + config, + layers: (layers.iter()) + .map(|layer| path_of(layer)) + .collect::, _>>()?, + }); + } + Ok(()) + })?; + json.finish()?; + found.ok_or_else(|| { + refuse(format!( + "the image archive's config digest is {}, not the requested {}: the bundle does not \ + hold that image", + if named.is_empty() { + "missing".to_owned() + } else { + named.join(", ") + }, + image.as_str() + )) + }) +} + +/// The config digest check: the archive names its config blob by that digest +/// (`<64 hex>.json` or `blobs/sha256/<64 hex>`), and it must be `image`'s. +fn check_config(image: &Image, config: &Path) -> Result<(), RuntimeError> { + let name = config + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| refuse("the image archive's config name"))?; + let digest = name.strip_suffix(".json").unwrap_or(name); + if digest != image.hex() { + return Err(refuse(format!( + "the image archive's config digest is {digest}, not the requested {}", + image.as_str() + ))); + } + Ok(()) +} + +/// One scan of the archive for the entries named by `wanted`, in that order; +/// everything else is skipped by seeking, never read. +fn locate(archive: &Path, wanted: &[PathBuf]) -> Result>, RuntimeError> { + let mut found = vec![None; wanted.len()]; + let mut remaining = wanted.len(); + let mut tape = Tape::new(BufReader::with_capacity(64 * 1024, open(archive)?)); + while remaining > 0 { + let Some(entry) = tape.next()? else { break }; + if entry.kind != Kind::Regular { + continue; + } + for (index, name) in wanted.iter().enumerate() { + if found[index].is_none() && *name == entry.path { + found[index] = Some(Blob { + offset: tape.data_offset(), + size: entry.size, + }); + remaining -= 1; + break; + } + } + } + if remaining > 0 { + let missing: Vec = (wanted.iter().enumerate()) + .filter(|(index, _)| found[*index].is_none()) + .map(|(_, name)| name.display().to_string()) + .collect(); + return Err(refuse(format!( + "the image archive is missing {}", + missing.join(", ") + ))); + } + Ok(found) +} + +/// The first bytes of `blob`, up to `max`. +fn read_blob(archive: &Path, blob: Blob, max: usize) -> Result, RuntimeError> { + let mut file = open(archive)?; + file.seek(SeekFrom::Start(blob.offset))?; + let mut bytes = Vec::new(); + let limit = blob.size.min(max as u64 + 1); + file.take(limit).read_to_end(&mut bytes)?; + if bytes.len() > max { + return Err(refuse("the image archive's blob is longer than allowed")); + } + Ok(bytes) +} + +/// Applies one layer, whose bytes are the `size` at `blob`. +fn extract_layer(archive: &Path, blob: Blob, name: &str, root: &Path) -> Result<(), RuntimeError> { + let mut file = open(archive)?; + file.seek(SeekFrom::Start(blob.offset))?; + match magic(&mut file)? { + Magic::Tar => {} + Magic::Gzip => { + return Err(refuse(format!( + "the image archive's layer {name} is gzip-compressed; a docker-archive must hold \ + uncompressed layers (docker save writes them so)" + ))) + } + Magic::Zstd => { + return Err(refuse(format!( + "the image archive's layer {name} is zstd-compressed; a docker-archive must hold \ + uncompressed layers (docker save writes them so)" + ))) + } + } + file.seek(SeekFrom::Start(blob.offset))?; + let mut tape = Tape::new(Window { + input: file, + left: blob.size, + }); + while let Some(entry) = tape.next()? { + let mut data = (&mut tape).take(entry.size); + apply(root, &entry, &mut data)?; + } + Ok(()) +} + +/// What the first bytes at the current position say the layer is. +fn magic(file: &mut File) -> Result { + let mut header = [0u8; 4]; + let mut read = 0; + while read < header.len() { + let count = file.read(&mut header[read..])?; + if count == 0 { + break; + } + read += count; + } + Ok(match &header[..read] { + [0x1f, 0x8b, ..] => Magic::Gzip, + [0x28, 0xb5, 0x2f, 0xfd] => Magic::Zstd, + _ => Magic::Tar, + }) +} + +/// A bounded window of an archive: one layer's bytes. Seeking forward is what +/// skipping an entry needs; anything else is refused. +struct Window { + input: R, + left: u64, +} + +impl Read for Window { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.left == 0 || buffer.is_empty() { + return Ok(0); + } + let wanted = (buffer.len() as u64).min(self.left) as usize; + let read = self.input.read(&mut buffer[..wanted])?; + self.left -= read as u64; + Ok(read) + } +} + +impl Seek for Window { + fn seek(&mut self, position: SeekFrom) -> io::Result { + let SeekFrom::Current(skip) = position else { + return Err(io::Error::from(io::ErrorKind::Unsupported)); + }; + if skip < 0 { + return Err(io::Error::from(io::ErrorKind::Unsupported)); + } + let skip = (skip as u64).min(self.left); + self.input.seek(SeekFrom::Current(skip as i64))?; + self.left -= skip; + Ok(self.left) + } +} + +/// An archive opened for reading, refused when it is not a regular file (a +/// symbolic link would let the archive be swapped after the check). +fn open(archive: &Path) -> Result { + let metadata = fs::symlink_metadata(archive).map_err(|error| { + refuse(format!( + "the image archive {} cannot be read: {error}", + archive.display() + )) + })?; + if !metadata.file_type().is_file() { + return Err(refuse(format!( + "the image archive {} is not a regular file", + archive.display() + ))); + } + OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(archive) + .map_err(|error| { + refuse(format!( + "the image archive {} cannot be opened: {error}", + archive.display() + )) + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs new file mode 100644 index 0000000000..90db445af5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs @@ -0,0 +1,149 @@ +//! Where unpacked images live and how one digest is unpacked exactly once: +//! `/images/<64 hex>`, published by a rename, guarded by a per-digest +//! `flock`. +use super::{archive::unpack, refuse, ImageRoot}; +use crate::{Image, RuntimeError}; +use nix::fcntl::{Flock, FlockArg}; +use std::{ + fs::{self, DirBuilder, File, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +/// The cache's own name under a runtime root: `/images`. +const CACHE: &str = "images"; + +/// The cache directory below a runtime root, `/images`, where unpacked +/// images and their locks live. +pub fn cache(root: &Path) -> PathBuf { + root.join(CACHE) +} + +/// The image's root under `cache`, unpacking `archive` first when this host +/// has no complete copy of that digest yet. One unpack per digest, however +/// many Sandboxes ask for it at once. +pub fn ensure(archive: &Path, image: &Image, cache: &Path) -> Result { + let cache = beneath(cache)?; + let lock = lock(&cache, image)?; + let directory = cache.join(image.hex()); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => { + drop(lock); + return Ok(ImageRoot::new(image.clone(), directory)); + } + Ok(_) => { + return Err(refuse(format!( + "the image cache {} is not a directory", + directory.display() + ))) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + // a crash of an earlier unpack left only its temporary directory: it is + // never the real name, so it is removed before this one is made + remove_temporaries(&cache, image)?; + let staging = cache.join(format!( + ".tmp-{}-{}", + image.hex(), + crate::backend::identity::random()?.as_str() + )); + DirBuilder::new().mode(0o700).create(&staging)?; + if let Err(cause) = unpack(archive, image, &staging) { + let _ = fs::remove_dir_all(&staging); + return Err(cause); + } + fs::set_permissions(&staging, fs::Permissions::from_mode(0o555))?; + match fs::rename(&staging, &directory) { + Ok(()) => {} + Err(error) => { + // another Sandbox published the same digest first: its copy is + // this one's too, and this one is the duplicate + let _ = fs::remove_dir_all(&staging); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + _ => return Err(error.into()), + } + } + } + drop(lock); + Ok(ImageRoot::new(image.clone(), directory)) +} + +/// The image's root under `cache` when this host already holds it, unpacking +/// nothing. A record whose image is gone cannot be reopened as the same root. +pub fn lookup(image: &Image, cache: &Path) -> Result { + let directory = cache.join(image.hex()); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => Ok(ImageRoot::new(image.clone(), directory)), + _ => Err(refuse(format!( + "the unpacked image {} is not at {}: this Sandbox's root cannot be found, so it is \ + neither reopened nor deleted", + image.as_str(), + directory.display() + ))), + } +} + +/// The cache directory, made 0700 and owned by this (root) process when it is +/// not there yet, verified to be a real directory when it is. +fn beneath(cache: &Path) -> Result { + if !cache.is_absolute() { + return Err(RuntimeError::Protocol( + "image cache must be an absolute path", + )); + } + match fs::symlink_metadata(cache) { + Ok(metadata) if metadata.is_dir() => Ok(cache.to_owned()), + Ok(_) => Err(refuse(format!( + "the image cache {} is not a directory", + cache.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let parent = cache + .parent() + .ok_or(RuntimeError::Protocol("image cache parent"))?; + fs::symlink_metadata(parent)?; + DirBuilder::new().mode(0o700).create(cache)?; + Ok(cache.to_owned()) + } + Err(error) => Err(error.into()), + } +} + +/// The per-digest lock, held by the caller until the returned lock is dropped. +/// The file stays on disk: another process may be waiting on it. +fn lock(cache: &Path, image: &Image) -> Result, RuntimeError> { + let path = cache.join(format!("{}.lock", image.hex())); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(path)?; + Flock::lock(file, FlockArg::LockExclusive).map_err(|(_, error)| RuntimeError::Io(error.into())) +} + +/// Removes the temporary directories a crashed unpack of `image` left; only +/// this digest's, and only while holding its lock. +fn remove_temporaries(cache: &Path, image: &Image) -> Result<(), RuntimeError> { + let prefix = format!(".tmp-{}-", image.hex()); + for entry in fs::read_dir(cache)? { + let entry = entry?; + let name = entry.file_name(); + let Some(text) = name.to_str() else { continue }; + if !text.starts_with(&prefix) { + continue; + } + let path = entry.path(); + if fs::symlink_metadata(&path)?.file_type().is_dir() { + fs::remove_dir_all(&path)?; + } else { + fs::remove_file(&path)?; + } + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs new file mode 100644 index 0000000000..4d27e27743 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs @@ -0,0 +1,220 @@ +//! What one layer does to the unpacked tree: a whiteout hides what the layers +//! below placed, and a regular file, directory, symbol link or hard link is +//! written inside the root — never through a symbolic link, and never outside +//! the root it belongs to. +use super::{ + refuse, + tar::{Entry, Kind}, +}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + ffi::OsStr, + fs::{self, DirBuilder, OpenOptions}, + io, + os::unix::{ + ffi::OsStrExt, + fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + }, + path::{Component, Path, PathBuf}, +}; + +/// One entry, applied to the unpacked tree: a whiteout, or a file, directory, +/// symbol link or hard link inside it. +pub(super) fn apply( + root: &Path, + entry: &Entry, + data: &mut dyn io::Read, +) -> Result<(), RuntimeError> { + let (Some(parent), Some(name)) = (entry.path.parent(), entry.path.file_name()) else { + return Err(refuse("the image archive's entry has no name")); + }; + let name = name.as_bytes(); + if entry.kind == Kind::Regular && name.starts_with(b".wh.") { + return whiteout(root, parent, &name[b".wh.".len()..]); + } + let target = beneath_root(root, &entry.path)?; + match &entry.kind { + Kind::Directory => make_directory(&target, entry.mode), + Kind::Regular => { + remove_any(&target)?; + write_file(&target, entry.mode, data) + } + Kind::Symlink(link) => { + remove_any(&target)?; + std::os::unix::fs::symlink(link, &target)?; + Ok(()) + } + Kind::HardLink(link) => { + let source = beneath_root(root, link)?; + if !fs::symlink_metadata(&source)?.file_type().is_file() { + return Err(refuse(format!( + "the image archive links {} to {}, which is not a file in it", + entry.path.display(), + link.display() + ))); + } + remove_any(&target)?; + fs::hard_link(&source, &target)?; + Ok(()) + } + } +} + +/// Applies `.wh.`/`.wh..wh..opq` to `root`: the named entry, or everything in +/// the directory, is hidden from the layers below. A whiteout with nothing +/// below it removes nothing. +fn whiteout(root: &Path, parent: &Path, name: &[u8]) -> Result<(), RuntimeError> { + // a whiteout hides what a lower layer placed; it never creates the + // directory it names, so a layer that only hides a path changes nothing + let Some(directory) = beneath_existing(root, parent)? else { + return Ok(()); + }; + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + return Err(refuse( + "the image archive's whiteout is outside a directory", + )) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + } + if name == b".wh..opq" { + for entry in fs::read_dir(&directory)? { + remove_any(&entry?.path())?; + } + return Ok(()); + } + if name.starts_with(b".wh..wh.") { + // aufs bookkeeping of an opaque directory's own metadata: not content + return Ok(()); + } + if name.is_empty() || name.contains(&b'/') { + return Err(refuse("the image archive's whiteout name")); + } + let hidden = directory.join(OsStr::from_bytes(name)); + match fs::symlink_metadata(&hidden) { + Ok(_) => remove_any(&hidden), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +/// `relative`'s place inside `root`, with every directory above it verified to +/// be one of this archive's own directories and never a symbolic link: an +/// entry can then only ever write inside the root it belongs to. +fn beneath_root(root: &Path, relative: &Path) -> Result { + let components: Vec = relative.components().collect(); + if components.is_empty() { + return Err(refuse("the image archive's entry has no name")); + } + let mut path = root.to_owned(); + for component in &components[..components.len() - 1] { + let Component::Normal(name) = component else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + path.push(name); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(refuse(format!( + "the image archive's entry {} passes through a symbolic link", + relative.display() + ))) + } + Ok(_) => { + return Err(refuse(format!( + "the image archive's entry {} passes through a non-directory", + relative.display() + ))) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + DirBuilder::new().mode(0o755).create(&path)?; + } + Err(error) => return Err(error.into()), + } + } + let Some(Component::Normal(name)) = components.last() else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + Ok(path.join(name)) +} + +/// `relative`'s place inside `root` when every component of it is already +/// there: `None` when one is not, and nothing is made. +fn beneath_existing(root: &Path, relative: &Path) -> Result, RuntimeError> { + let mut path = root.to_owned(); + for component in relative.components() { + let Component::Normal(name) = component else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + path.push(name); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(refuse( + "the image archive's entry passes through a symbolic link", + )) + } + Ok(_) => return Ok(None), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + } + Ok(Some(path)) +} + +/// A directory at `target`, with `mode` (0755 for a mode of zero). +fn make_directory(target: &Path, mode: u32) -> Result<(), RuntimeError> { + let mode = if mode & 0o777 == 0 { + 0o755 + } else { + mode & 0o777 + }; + match fs::symlink_metadata(target) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + remove_any(target)?; + DirBuilder::new().mode(mode).create(target)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + DirBuilder::new().mode(mode).create(target)?; + } + Err(error) => return Err(error.into()), + } + fs::set_permissions(target, fs::Permissions::from_mode(mode))?; + Ok(()) +} + +/// A regular file at `target`, whose bytes are all of `data`. +fn write_file(target: &Path, mode: u32, data: &mut dyn io::Read) -> Result<(), RuntimeError> { + let mode = mode & 0o777; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(mode) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(target)?; + io::copy(data, &mut file)?; + fs::set_permissions(target, fs::Permissions::from_mode(mode))?; + Ok(()) +} + +/// Removes whatever is at `path` without following it: a symbolic link is +/// removed as the link it is, never as what it names. +fn remove_any(path: &Path) -> Result<(), RuntimeError> { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(metadata) if metadata.is_dir() => Ok(fs::remove_dir_all(path)?), + Ok(_) => Ok(fs::remove_file(path)?), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs new file mode 100644 index 0000000000..729621a071 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs @@ -0,0 +1,85 @@ +//! The Sandbox image as a read-only root filesystem: one archive per bundle, +//! unpacked once per config digest and shared by every Sandbox that names it. +//! +//! A namespace Sandbox's root is an overlay of two things: this unpacked image +//! (the lower, read-only layer) and the Sandbox's own `upper` and `work` +//! directories (its writable layer, deleted with it). The image is what the +//! commands see as `/`; the host's own userland is not part of that root. +//! +//! # The archive +//! +//! The bundle ships one **docker-archive** (`docker save`'s output) per +//! Sandbox image, named `/images/<64 hex>.tar`, where `<64 hex>` is +//! `deploy.image` without its `sha256:` prefix. The bundle builder sets +//! `LAYERFS_IMAGE` (the digest) and `LAYERFS_IMAGE_ARCHIVE` (the absolute path +//! of that file) for the worker that runs this runtime. +//! +//! `docker-archive` is chosen over an OCI image layout because +//! - `deploy.image` is Docker's own local image identity on the Docker backend, +//! and a docker-archive's `manifest.json` names its config by exactly that +//! digest, so one value addresses the image on both backends; +//! - `docker save` is available everywhere the Docker backend already works, +//! which is where the image comes from; +//! - its layer blobs are uncompressed tars, so unpacking needs no decompressor +//! and this crate keeps its single dependency. +//! +//! A compressed layer blob (`podman save` and some other writers gzip them) is +//! refused before anything is written; the archive must be produced by +//! `docker save` or converted to that shape. +//! +//! # The digest +//! +//! `deploy.image` is `sha256:<64 lowercase hex>` and must be the archive's +//! config digest: the docker-archive names that blob `<64 hex>.json` or +//! `blobs/sha256/<64 hex>`, and [`archive`] compares that name with the +//! requested digest. An archive whose config digest is something else is +//! refused before anything is unpacked, so a Sandbox never runs a root other +//! than the one the deployment named. The config blob is not re-hashed here: +//! the archive arrives inside a sealed bundle that the provider copies in +//! whole, and this crate has no hashing dependency of its own. +//! +//! # One unpack per digest +//! +//! The unpacked image lives under `/images/<64 hex>`, named +//! by its digest only, so two Sandboxes that name one image share one copy. +//! The first Sandbox to need it takes a per-digest lock +//! (`images/<64 hex>.lock`, `flock`), unpacks into `images/.tmp-<64 hex>-…` +//! and renames that directory into place once it is complete, so a second +//! Sandbox either waits for that rename or finds the finished directory, and a +//! crashed unpack never leaves a half image under the real name. The lock file +//! stays: another process may still hold it. +//! +//! A command never gets write access to the image. The unpacked directory is +//! outside the Sandbox's mount tree — only the image's *contents* are composed +//! into its root, by the kernel, as a read-only lower layer — so no path a +//! command can name reaches the cache, and the cache directory itself is +//! root-owned and not writable by the command identity. +//! +//! # Extraction safety +//! +//! Every layer is applied in order, with the archive's own whiteouts +//! (`.wh.` removes that name, `.wh..wh..opq` empties its directory). +//! Entries are refused, not repaired, when they are +//! - an absolute path or one with a `.`/`..` component, +//! - reachable only through a symbolic link or a non-directory (so a symlink +//! in an early entry cannot redirect a later entry's write outside the root), +//! - a device node, a FIFO or any other entry type this runtime does not +//! extract (regular files, directories, symlinks and hard links only). +//! +//! File ownership becomes root (the archive's numeric owners are not applied, +//! and a rootfs layer is root's), permission bits are applied as `0o777` — +//! setuid, setgid, sticky and xattrs (and so file capabilities) are not +//! restored — and a command runs with no-new-privs anyway. +//! +//! The pieces are the cache ([`cache`]), one archive's own reading +//! ([`archive`]), the tar format ([`tar`]), what one layer does to the tree +//! ([`layers`]) and the types they share ([`types`]). +mod archive; +mod cache; +mod layers; +mod tar; +mod types; + +pub use cache::{cache, ensure, lookup}; +pub use types::ImageRoot; +pub(crate) use types::{refuse, PATH_MAX}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs new file mode 100644 index 0000000000..e480994601 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs @@ -0,0 +1,366 @@ +//! The tar format as far as an image archive needs it: one header at a time, +//! the entry's data read through the same handle, everything else skipped by +//! seeking. Paths are checked here, so nothing downstream sees one that could +//! leave the root. +use super::{refuse, PATH_MAX}; +use crate::RuntimeError; +use std::{ + ffi::OsStr, + io::{self, Read, Seek, SeekFrom}, + os::unix::ffi::OsStrExt, + path::{Component, Path, PathBuf}, +}; + +/// A tar block, and the header's own size. +const BLOCK: u64 = 512; +/// The largest pax or GNU extended header accepted. +const EXTENDED_MAX: usize = 1 << 20; + +/// One entry of a tar archive, its path already checked to be a plain relative +/// path inside the root. +#[derive(Debug)] +pub(super) struct Entry { + pub(super) path: PathBuf, + pub(super) kind: Kind, + pub(super) mode: u32, + pub(super) size: u64, +} + +/// What an entry places in the root. Every other tar entry type (device +/// nodes, FIFOs, …) is refused when the header is read. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum Kind { + Regular, + Directory, + Symlink(PathBuf), + HardLink(PathBuf), +} + +/// A streaming tar reader. Nothing is collected: an entry's bytes are streamed +/// to where they go, and skipped by seeking when the entry is not wanted. +pub(super) struct Tape { + input: R, + /// Bytes read from the start of this archive. + position: u64, + /// Bytes of the current entry's data not read yet. + left: u64, + /// Padding after that data, to the next block. + padding: u64, + /// Where the current entry's data starts. + data_offset: u64, + /// A pax or GNU name waiting for the entry it belongs to. + path: Option, + /// A pax or GNU link target waiting for the same entry. + link: Option, +} + +impl Tape { + pub(super) fn new(input: R) -> Self { + Self { + input, + position: 0, + left: 0, + padding: 0, + data_offset: 0, + path: None, + link: None, + } + } + + /// Where the current entry's bytes start. + pub(super) fn data_offset(&self) -> u64 { + self.data_offset + } + + /// The next entry, or `None` at the archive's end. Extended headers (pax + /// `x`, GNU `L`/`K`) are applied to the entry that follows them; pax + /// global headers and unknown fixed fields are read past. + pub(super) fn next(&mut self) -> Result, RuntimeError> { + loop { + self.skip_data()?; + let Some(header) = self.header()? else { + return Ok(None); + }; + let kind = header[156]; + let size = number(&header[124..136])?; + self.left = size; + self.padding = (BLOCK - size % BLOCK) % BLOCK; + self.data_offset = self.position; + match kind { + b'x' => { + let (path, link) = pax(&self.data(EXTENDED_MAX)?)?; + if path.is_some() { + self.path = path; + } + if link.is_some() { + self.link = link; + } + continue; + } + b'g' => continue, + b'L' => { + self.path = Some(long_name(&self.data(PATH_MAX)?)?); + continue; + } + b'K' => { + self.link = Some(long_name(&self.data(PATH_MAX)?)?); + continue; + } + _ => {} + } + let path = match self.path.take() { + Some(path) => path, + None => own_name(&header)?, + }; + let link = match self.link.take() { + Some(link) => link, + None => link_name(&header)?, + }; + let kind = kind_of(kind, link)?; + return Ok(Some(Entry { + path, + kind, + mode: u32::try_from(number(&header[100..108])?).unwrap_or(0o644), + size, + })); + } + } + + /// One block of header, or `None` at the end of the archive (an all-zero + /// block, or nothing left). + fn header(&mut self) -> Result, RuntimeError> { + let mut header = [0u8; 512]; + let read = self.fill(&mut header)?; + if read == 0 || header.iter().all(|byte| *byte == 0) { + return Ok(None); + } + if read < header.len() { + return Err(refuse("the image archive's header is truncated")); + } + if checksum(&header) != number(&header[148..156])? { + return Err(refuse("the image archive's header checksum")); + } + Ok(Some(header)) + } + + /// Reads the current entry's data (at most `max` bytes) and returns it. + fn data(&mut self, max: usize) -> Result, RuntimeError> { + if self.left > max as u64 { + return Err(refuse("the image archive's extended header is too long")); + } + let left = self.left; + let mut bytes = Vec::new(); + let mut source = Read::by_ref(self).take(left); + source.read_to_end(&mut bytes)?; + Ok(bytes) + } + + /// Skips whatever is left of the current entry and its padding. + fn skip_data(&mut self) -> Result<(), RuntimeError> { + let skip = (self.left) + .checked_add(self.padding) + .ok_or_else(|| refuse("the image archive's entry length"))?; + self.left = 0; + self.padding = 0; + if skip == 0 { + return Ok(()); + } + let distance = + i64::try_from(skip).map_err(|_| refuse("the image archive's entry length"))?; + self.input.seek(SeekFrom::Current(distance))?; + self.position += skip; + Ok(()) + } + + /// Fills `buffer` as far as the input allows, returning how many bytes were + /// read (zero only at the end). + fn fill(&mut self, buffer: &mut [u8]) -> Result { + let mut filled = 0; + while filled < buffer.len() { + let read = self.input.read(&mut buffer[filled..])?; + if read == 0 { + break; + } + filled += read; + self.position += read as u64; + } + Ok(filled) + } +} + +impl Read for Tape { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.left == 0 || buffer.is_empty() { + return Ok(0); + } + let wanted = (buffer.len() as u64).min(self.left) as usize; + let read = self.input.read(&mut buffer[..wanted])?; + self.left -= read as u64; + self.position += read as u64; + Ok(read) + } +} + +/// The entry's own name: its `name` and, in a ustar archive, its `prefix`. +fn own_name(header: &[u8; 512]) -> Result { + let name = text(&header[0..100]); + let prefix = if &header[257..262] == b"ustar".as_slice() { + text(&header[345..500]) + } else { + &[] + }; + let mut bytes = Vec::with_capacity(prefix.len() + 1 + name.len()); + if !prefix.is_empty() { + bytes.extend_from_slice(prefix); + bytes.push(b'/'); + } + bytes.extend_from_slice(name); + checked(&bytes) +} + +/// The entry's own link target, for the two kinds that carry one. +fn link_name(header: &[u8; 512]) -> Result { + Ok(PathBuf::from(OsStr::from_bytes(text(&header[157..257])))) +} + +/// The entry's kind, refusing every type this runtime does not extract. +fn kind_of(flag: u8, link: PathBuf) -> Result { + match flag { + b'0' | b'\0' | b'7' => Ok(Kind::Regular), + b'5' => Ok(Kind::Directory), + b'2' => Ok(Kind::Symlink(link)), + b'1' => Ok(Kind::HardLink(checked(link.as_os_str().as_bytes())?)), + b'3' | b'4' => Err(refuse( + "the image archive holds a device node, which a Sandbox root does not extract", + )), + b'6' => Err(refuse( + "the image archive holds a FIFO, which a Sandbox root does not extract", + )), + _ => Err(refuse( + "the image archive holds an entry type it does not extract", + )), + } +} + +/// A path checked to be plain and relative: no `/` at its start, no `.` or +/// `..` component, at most one kernel path's worth of bytes. +fn checked(bytes: &[u8]) -> Result { + let path = Path::new(OsStr::from_bytes(bytes)); + if bytes.len() > PATH_MAX || bytes.contains(&0) { + return Err(refuse( + "the image archive's path is not a safe relative path", + )); + } + let mut cleaned = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(name) => cleaned.push(name), + Component::CurDir => {} + Component::RootDir | Component::ParentDir | Component::Prefix(_) => { + return Err(refuse( + "the image archive's path is absolute or holds a `..` component", + )) + } + } + } + if cleaned.as_os_str().is_empty() { + return Err(refuse("the image archive's path is empty")); + } + Ok(cleaned) +} + +/// A manifest entry's path, from the UTF-8 text the archive's JSON holds. +pub(super) fn path_of(bytes: &[u8]) -> Result { + std::str::from_utf8(bytes) + .map_err(|_| refuse("the image archive's manifest is not UTF-8")) + .and_then(|text| checked(text.as_bytes())) +} + +/// A pax extended header's `path` and `linkpath` records, if it has them. +fn pax(data: &[u8]) -> Result<(Option, Option), RuntimeError> { + let (mut path, mut link) = (None, None); + let mut rest = data; + while !rest.is_empty() { + let space = rest + .iter() + .position(|byte| *byte == b' ') + .ok_or_else(|| refuse("the image archive's pax record"))?; + let length: usize = std::str::from_utf8(&rest[..space]) + .ok() + .and_then(|text| text.parse().ok()) + .filter(|length: &usize| *length > space + 1 && *length <= rest.len()) + .ok_or_else(|| refuse("the image archive's pax record length"))?; + let record = &rest[space + 1..length - 1]; + if let Some(equals) = record.iter().position(|byte| *byte == b'=') { + let (key, value) = (&record[..equals], &record[equals + 1..]); + match key { + b"path" => path = Some(checked(value)?), + b"linkpath" => link = Some(checked(value)?), + _ => {} + } + } + rest = &rest[length..]; + } + Ok((path, link)) +} + +/// A GNU long name's data: the name, up to its first NUL. +fn long_name(data: &[u8]) -> Result { + let name = match data.iter().position(|byte| *byte == 0) { + Some(end) => &data[..end], + None => data, + }; + checked(name) +} + +/// A header's NUL- or space-padded octal (or GNU base-256) number. +fn number(field: &[u8]) -> Result { + if let Some(first) = field.first() { + if first & 0x80 != 0 { + let mut value = u64::from(first & 0x7f); + for byte in &field[1..] { + value = (value) + .checked_mul(256) + .and_then(|value| value.checked_add(u64::from(*byte))) + .ok_or_else(|| refuse("the image archive's number"))?; + } + return Ok(value); + } + } + let mut value = 0u64; + for byte in field { + match byte { + b'0'..=b'7' => { + value = (value) + .checked_mul(8) + .and_then(|value| value.checked_add(u64::from(*byte - b'0'))) + .ok_or_else(|| refuse("the image archive's number"))?; + } + b' ' | 0 => {} + _ => return Err(refuse("the image archive's number")), + } + } + Ok(value) +} + +/// A header's checksum: the sum of its bytes with the checksum field itself +/// read as spaces. +fn checksum(header: &[u8; 512]) -> u64 { + (header.iter().enumerate()) + .map(|(index, byte)| match index { + 148..=155 => u64::from(b' '), + _ => u64::from(*byte), + }) + .sum() +} + +/// `field` up to its first NUL, trimmed of trailing spaces. +fn text(field: &[u8]) -> &[u8] { + let end = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = &field[..end]; + let end = (text.iter().rposition(|byte| *byte != b' ')).map_or(0, |index| index + 1); + &text[..end] +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs new file mode 100644 index 0000000000..eaa2b36215 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs @@ -0,0 +1,36 @@ +//! The image's unpacked root handle and errors: [`ImageRoot`] and [`refuse`]. +use crate::{Image, RuntimeError}; +use std::{io, path::PathBuf}; + +/// The kernel's own limit on one path. +pub(crate) const PATH_MAX: usize = 4096; + +/// One image's unpacked root, and the digest it is the root of. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ImageRoot { + image: Image, + directory: PathBuf, +} + +impl ImageRoot { + /// The digest this root is the unpacked image of. + pub fn image(&self) -> &Image { + &self.image + } + + /// The read-only directory the Sandbox's overlay takes as its lower layer. + pub fn directory(&self) -> &std::path::Path { + &self.directory + } + + /// The root of `image` at `directory`, which is already unpacked there. + pub(super) fn new(image: Image, directory: PathBuf) -> Self { + Self { image, directory } + } +} + +/// A refusal with the exact reason, as the runtime's I/O cause: the runtime's +/// protocol causes are static and these name paths and digests. +pub(crate) fn refuse(message: impl Into) -> RuntimeError { + RuntimeError::Io(io::Error::new(io::ErrorKind::InvalidData, message.into())) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs new file mode 100644 index 0000000000..4b650396e7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs @@ -0,0 +1,365 @@ +//! One namespace Sandbox: its directory, the init stage that owns its +//! namespaces, its cgroup and its exact attempts. Nothing is retried and no +//! Drop cleans up; a Drop only reaps an init process this process started +//! that has already exited, so it never stays a zombie of this process. +use super::{cgroup, host, paths::Paths, rootfs::ImageRoot}; +use crate::backend::native::LocalProcess; +use crate::provider::{LocalRecord, NamespaceRecord, SandboxRecord}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::Path, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The recorded process facts of a Sandbox, as a live process to check. +pub(super) fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} + +/// The facts a record carries into a reopen: the init stage it saw, the port +/// it listened on, its labels and the image its root is an overlay of. +pub(crate) struct Recorded { + pub(crate) init: LocalProcess, + pub(crate) port: u16, + pub(crate) labels: Vec<(String, String)>, + pub(crate) image: ImageRoot, +} + +/// Pause between two looks at a killed init process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for the group's processes to be gone after `cgroup.kill`. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged namespace Sandbox: the init stage of its namespaces, its +/// listener and the cgroup that owns everything it started. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) paths: Paths, + pub(super) cgroup: std::path::PathBuf, + pub(super) labels: Vec<(String, String)>, + /// The helper executable this backend's init and exec stages run. + pub(super) helper: std::path::PathBuf, + /// The unpacked image its root is an overlay of, and its digest. + pub(super) image: ImageRoot, + /// The init stage this process started, kept so its exit is reaped here. + /// `None` after a reopen. + pub(super) child: Mutex>, + pub(super) init: Option, + pub(super) port: u16, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// The owner a start returns once its init stage is spawned and its + /// cgroup exists; the listener port follows. + pub(super) fn unstarted( + id: ContainerId, + paths: Paths, + cgroup: std::path::PathBuf, + labels: Vec<(String, String)>, + helper: std::path::PathBuf, + image: ImageRoot, + ) -> Self { + Self { + id, + paths, + cgroup, + labels, + helper, + image, + child: Mutex::new(None), + init: None, + port: 0, + stop_attempted: false, + stopped: false, + delete_attempted: false, + deleted: false, + } + } + + /// Reopens a recorded Sandbox whose init stage still runs: the same + /// process (id, start time and boot), in its own cgroup, and its loopback + /// port. It starts nothing. + pub(crate) fn reopen( + paths: Paths, + cgroup: std::path::PathBuf, + id: ContainerId, + helper: std::path::PathBuf, + recorded: Recorded, + ) -> Result> { + let Recorded { + init, + port, + labels, + image, + } = recorded; + let mut owner = Self::unstarted(id, paths, cgroup, labels, helper, image); + owner.init = Some(init); + owner.port = port; + if !owner.paths.local.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved namespaces Sandbox")); + } + if !init.alive() || !host::in_cgroup(init.pid, &owner.cgroup) { + let mut failure = owner.invalid(crate::types::STOPPED); + failure.observed_container = Some(owner.id); + return Err(failure); + } + owner.endpoint()?; + Ok(owner) + } + + /// A recorded Sandbox whose init stage is gone + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub(crate) fn stopped( + paths: Paths, + cgroup: std::path::PathBuf, + id: ContainerId, + helper: std::path::PathBuf, + recorded: Recorded, + ) -> Self { + let mut owner = Self::unstarted(id, paths, cgroup, recorded.labels, helper, recorded.image); + owner.init = Some(recorded.init); + owner.port = recorded.port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + + /// Its private directory (setup, executable, overlay, mounts, log). + pub fn directory(&self) -> &Path { + &self.paths.local.directory + } + + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> std::path::PathBuf { + self.paths.local.tmp.clone() + } + + /// Its init stage process, once started. + pub fn process(&self) -> Option { + self.init + } + + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + + /// The image its root is an overlay of. + pub fn image(&self) -> &ImageRoot { + &self.image + } + + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let init = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: init.pid, + start_time: init.start, + boot: init.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + + /// Its record, naming this backend so another one never reopens it, and + /// the image digest so a reopen finds the same root again. + pub fn record(&self) -> Option { + let local = self.local_record()?; + Some(SandboxRecord::Namespaces(NamespaceRecord { + local, + image: self.image.image().clone(), + })) + } + + /// The daemon's endpoint while its init stage runs. A Sandbox whose init + /// stage exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(init) = self.init else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !init.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + + /// Ends the Sandbox once: one write to `cgroup.kill` asks the kernel to + /// `SIGKILL` every process of its group, descendants and commands + /// included, and no graceful drain exists to wait for. A Sandbox whose + /// init stage already ended is the same stopped state. + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original namespaces Stop admission")); + } + self.stop_attempted = true; + cgroup::kill(&self.cgroup).map_err(|error| self.io(error))?; + let wait = Duration::from_secs(u64::from(timeout_seconds)).max(KILL_WAIT); + let deadline = Instant::now() + wait; + loop { + self.reap(); + if self.init.is_none_or(|init| !init.alive()) { + match cgroup::populated(&self.cgroup) { + Ok(0) => break, + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => break, + Err(error) => return Err(self.io(error)), + } + } + if Instant::now() >= deadline { + return Err( + self.invalid("a process of the namespaces Sandbox runs after cgroup.kill") + ); + } + std::thread::sleep(PAUSE); + } + self.stopped = true; + Ok(()) + } + + /// Deletes its cgroup and its directory once, after a known stop: a + /// group that still holds a process is refused, so nothing is reported + /// gone while it runs. The private root it made is a mount of the + /// Sandbox's own namespaces and is gone with them. The shared Store is + /// outside the directory and is never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.init.is_some() && !self.stopped) { + return Err(self.invalid("original namespaces Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.init.is_some_and(LocalProcess::alive) { + return Err(self.invalid("namespaces Sandbox init runs again; nothing was deleted")); + } + cgroup::remove(&self.cgroup).map_err(|error| self.io(error))?; + match fs::remove_dir_all(self.directory()) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + // the private root holds nothing once its namespaces ended + match fs::remove_dir(&self.paths.root) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(&self.paths.local.log)?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + + /// Nothing to name: `cgroup.kill` owns every process the Sandbox started, + /// including one a command left in a session of its own. + pub fn track(&self, _pid: u32, _start_time: u64) -> Result<(), Box> { + Ok(()) + } + + /// Reaps the init stage this process started, if it exited. True when no + /// init stage this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// An init stage this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running one is + /// left alone: ending it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/create.rs b/engine/crates/layerfs-sandbox/src/backend/native/create.rs new file mode 100644 index 0000000000..c814869456 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/create.rs @@ -0,0 +1,284 @@ +//! One local daemon start: the Sandbox's private directory, its 0600 setup, +//! one spawn and its listener line. Nothing is retried. +use super::{ + process::LocalProcess, + sandbox::{Sandbox, LOG}, + search::search_path, +}; +use crate::backend::{identity, listener}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::Duration, +}; + +/// A local runtime root: one directory per Sandbox under it. +#[derive(Clone, Debug)] +pub struct Local { + root: PathBuf, + name: Option, + /// Variables the daemon starts with besides `PATH`. + environment: Vec<(String, String)>, +} + +/// Where one Sandbox keeps its parts, all under its directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalPaths { + /// The Sandbox directory, `/`, 0755. + pub directory: PathBuf, + /// Its daemon setup, `config/daemon.setup`, 0600 in a 0700 directory. + pub config: PathBuf, + /// Its daemon overlay template, `overlay/overlay.sqlite`, in a 0700 directory. + pub overlay: PathBuf, + /// Its Workspace mount home, `workspaces`, made 0755 by the daemon. + pub mounts: PathBuf, + /// Its daemon's output, `daemon.log`, 0600. + pub log: PathBuf, + /// Its commands' scratch directory, `tmp`, 1777 like a container's + /// `/tmp`, so what they leave there goes with the Sandbox. + pub tmp: PathBuf, +} + +/// The boundary one failed create reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LocalPhase { + /// Local checks; nothing made. + Validate, + /// The directory, setup and log file. + Create, + /// The daemon spawn. + Start, + /// The wait for the daemon's listener line. + Listener, +} + +/// A failed create. `sandbox` holds what it made (its directory, and its +/// daemon once spawned), for the caller to stop and delete once. +#[derive(Debug)] +pub struct LocalCreateFailure { + pub phase: LocalPhase, + pub sandbox: Option, + pub cause: RuntimeError, +} + +impl Local { + /// Selects `root`: an existing absolute directory owned by this user, + /// writable by it alone and traversable by others, as is every directory + /// above it (commands enter their Workspace below it only after taking + /// another identity, so one closed directory on the way, such as a 0700 + /// `/root`, would fail every command at spawn). + pub fn new(root: impl AsRef) -> Result { + let local = Self::native(root.as_ref())?; + let root = root.as_ref(); + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } + Ok(local) + } + /// Selects `root` for the native route, whose commands run as this user + /// (no other identity): the root's own checks as in [`Self::new`], but + /// the directories above it need not be traversable by others (a macOS + /// home is 0750). + pub fn native(root: impl AsRef) -> Result { + let root = root.as_ref(); + if !root.is_absolute() + || root + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return Err(RuntimeError::Protocol("absolute local runtime root")); + } + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() + || metadata.uid() != nix::unistd::Uid::effective().as_raw() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & 0o001 == 0 + { + return Err(RuntimeError::Protocol( + "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", + )); + } + Ok(Self { + root: root.to_owned(), + name: None, + environment: Vec::new(), + }) + } + /// Starts the next daemon with `environment` besides its search path (the + /// native route's `LAYERFS_KERNEL=nfs` and its mount helpers). Nothing + /// else of this process's environment reaches it. + pub fn environment(mut self, environment: Vec<(String, String)>) -> Self { + self.environment = environment; + self + } + /// Names the next Sandbox's directory `name`, under the Docker name rule + /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at + /// most 128), so it is one plain path component. + pub fn naming(mut self, name: &str) -> Result { + let bytes = name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 128 + || !bytes[0].is_ascii_alphanumeric() + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) + { + return Err(RuntimeError::Protocol("local Sandbox name")); + } + self.name = Some(name.to_owned()); + Ok(self) + } + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self) -> Result { + let name = self + .name + .as_deref() + .ok_or(RuntimeError::Protocol("local Sandbox name"))?; + let directory = self.root.join(name); + Ok(LocalPaths { + config: directory.join("config").join("daemon.setup"), + overlay: directory.join("overlay").join("overlay.sqlite"), + mounts: directory.join("workspaces"), + log: directory.join(LOG), + tmp: directory.join("tmp"), + directory, + }) + } + /// Makes the named Sandbox's directory, writes `config` (the encoded + /// daemon setup naming [`Self::paths`]) 0600, spawns `executable + /// --config ` in its own process group with no inherited + /// environment but the search path, and waits up to `wait` for its + /// listener line. Once anything is made, a failure returns it. + pub fn create( + &self, + executable: &Path, + config: &[u8], + wait: Duration, + labels: Vec<(String, String)>, + ) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(LocalCreateFailure { + phase, + sandbox, + cause, + }) + }; + let paths = self + .paths() + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let checked = (|| { + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("local daemon executable")); + } + if config.is_empty() || config.len() > 8192 || wait.is_zero() { + return Err(RuntimeError::Protocol("local daemon setup/startup wait")); + } + match fs::symlink_metadata(&paths.directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(_) => Err(RuntimeError::Protocol("local Sandbox directory exists")), + } + })(); + checked.map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted(id, paths.directory.clone(), labels); + let log = match prepare(&paths, config) { + Ok(log) => log, + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + }; + let spawned = log.try_clone().and_then(|output| { + Command::new(executable) + .arg("--config") + .arg(&paths.config) + .env_clear() + .env("PATH", search_path()) + .envs(self.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + let process = match LocalProcess::observe(child.id()) { + Ok(process) => process, + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + }; + sandbox.process = Some(process); + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.log, wait, &|| { + sandbox.reap(); + sandbox.process.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } + } +} + +/// The setup and overlay directories (0700), the scratch directory (1777), +/// the setup file (0600, written once) and the empty log file (0600) it +/// returns. +fn prepare(paths: &LocalPaths, config: &[u8]) -> io::Result { + // the process umask may narrow the create mode + fs::set_permissions(&paths.directory, Permissions::from_mode(0o755))?; + for file in [&paths.config, &paths.overlay] { + let parent = file + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "local path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.tmp)?; + fs::set_permissions(&paths.tmp, Permissions::from_mode(0o1777))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + private(&paths.config)?.write_all(config)?; + private(&paths.log) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/exec.rs b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs new file mode 100644 index 0000000000..1478054771 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs @@ -0,0 +1,133 @@ +//! One ordinary command to its end in a local Sandbox: the command identity, +//! the Workspace directory and a search path (on macOS this process's own); +//! nothing else of this process's environment, no shell of its own and no +//! implicit timeout. Each command leads a process group of its own, named in +//! the Sandbox until it is empty, so what it leaves running ends with the +//! Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox, search::search_path}; +use crate::backend::streams; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use nix::sys::signal::{killpg, Signal}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +/// One command's end: its exit code (128 + the signal for one a signal +/// ended, as the Engine reports it), its process id (diagnostic only) and the +/// bytes each stream delivered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalExit { + pub code: i32, + pub pid: u32, + pub stdout: u64, + pub stderr: u64, +} + +impl Sandbox { + /// Runs `request` once while this Sandbox's daemon runs: as its uid/gid + /// with no supplementary groups, in its directory, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. Standard input is not offered. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + self.endpoint()?; + let mut command = Command::new(&request.arguments[0]); + command + .args(&request.arguments[1..]) + .env_clear() + .env("PATH", search_path()) + .current_dir(Path::new(&request.directory)) + // std drops supplementary groups (setgroups 0) when it sets a uid as root + .uid(request.identity.uid) + .gid(request.identity.gid) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + // named before anything is waited for: the leader is not reaped + // before the wait below, so it is observable even once it exited + let named = LocalProcess::observe(pid) + .and_then(|leader| groups::name(self.directory(), leader).map(|()| leader)); + let leader = match named { + Ok(leader) => leader, + Err(error) => { + // an unnamed group would outlive the Sandbox: it ends here, once + if let Ok(group) = process::pid(pid) { + let _ = killpg(group, Signal::SIGKILL); + } + let _ = child.wait(); + let mut failure = self.io(error); + failure.attempted = true; + return Err(failure); + } + }; + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + // a name left behind is harmless: its group is checked again when ended + let _ = groups::forget_if_empty(self.directory(), leader); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("local command exit status"))?; + Ok(LocalExit { + code, + pid, + stdout: out_bytes, + stderr: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/groups.rs b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs new file mode 100644 index 0000000000..6f18869381 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs @@ -0,0 +1,149 @@ +//! The process groups a local Sandbox's processes run in. A container's +//! processes end with it; a local Sandbox's would outlive it on this kernel, +//! so each command starts its own process group, named in the Sandbox's +//! `groups` directory while it may have members, and every named group (and +//! the daemon's own) is killed when the Sandbox stops or is deleted. +//! +//! A group is named by its leader: id, start time and boot. Its id stays the +//! group's while any member lives, so a group whose leader is gone is still +//! known by it; a leader found with another start time means the id was +//! reused and the group is not this Sandbox's. A process that leaves its +//! group (`setsid`) is not followed unless it is named here too +//! ([`super::Sandbox::track`]). +use super::platform; +use super::process::{self, LocalProcess}; +use nix::{fcntl::OFlag, sys::signal::Signal}; +use std::{ + fs::{self, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::Path, + time::{Duration, Instant}, +}; + +/// The directory of a Sandbox that names its groups, one empty file each. +pub(super) const GROUPS: &str = "groups"; +/// Pause between two looks at a killed group. +const PAUSE: Duration = Duration::from_millis(20); + +/// Names the group `leader` leads (its id is the group's), once. +pub(super) fn name(directory: &Path, leader: LocalProcess) -> io::Result<()> { + let groups = directory.join(GROUPS); + // a Sandbox started before groups were named has no directory yet + match fs::DirBuilder::new().mode(0o700).create(&groups) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(groups.join(entry(leader)?)) + .map(drop) +} + +/// Forgets the group `leader` led once no member of it is left. Called after +/// the leader was reaped; a group that still has members stays named. +pub(super) fn forget_if_empty(directory: &Path, leader: LocalProcess) -> io::Result<()> { + match nix::sys::signal::killpg(process::pid(leader.pid)?, None) { + Err(nix::errno::Errno::ESRCH) => {} + _ => return Ok(()), + } + match fs::remove_file(directory.join(GROUPS).join(entry(leader)?)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Kills every group the Sandbox at `directory` named, and the group of +/// `daemon` (its leader), once with KILL, as a container's end kills what +/// ran in it, then waits up to `wait` for their members to be gone. Groups +/// of another boot ended with it. +pub(super) fn end( + directory: &Path, + daemon: Option, + wait: Duration, +) -> io::Result<()> { + let mut leaders: Vec = daemon.into_iter().collect(); + match fs::read_dir(directory.join(GROUPS)) { + Ok(items) => { + for item in items { + let name = item?.file_name(); + let leader = name.to_str().and_then(parse).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "local Sandbox group name") + })?; + leaders.push(leader); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let this_boot = process::boot()?; + leaders.retain(|leader| leader.boot == Some(this_boot)); + let table = processes()?; + let mut killed = Vec::new(); + for leader in leaders { + if !members(&table, leader).is_empty() { + match nix::sys::signal::killpg(process::pid(leader.pid)?, Signal::SIGKILL) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => killed.push(leader), + Err(errno) => return Err(errno.into()), + } + } + } + let deadline = Instant::now() + wait; + loop { + let table = processes()?; + if killed + .iter() + .all(|leader| members(&table, *leader).is_empty()) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::other( + "a process of the local Sandbox still runs after KILL", + )); + } + std::thread::sleep(PAUSE); + } +} + +/// The running members of `leader`'s group: none when the leader's id now +/// names another process (the id was reused, so the group is another's). +fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { + let reused = (table.iter()).any(|(pid, stat)| *pid == leader.pid && stat.start != leader.start); + if reused { + return Vec::new(); + } + (table.iter()) + .filter(|(_, stat)| stat.group == leader.pid && !stat.ended()) + .map(|(pid, _)| *pid) + .collect() +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +fn processes() -> io::Result> { + platform::processes() +} + +/// A group's file name: `--`. +fn entry(leader: LocalProcess) -> io::Result { + let boot = leader + .boot + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "a group leader's boot"))?; + Ok(format!("{boot:032x}-{}-{}", leader.pid, leader.start)) +} + +fn parse(name: &str) -> Option { + let mut parts = name.split('-'); + let boot = parts.next().filter(|b| b.len() == 32)?; + let process = LocalProcess { + boot: Some(u128::from_str_radix(boot, 16).ok()?), + pid: parts.next()?.parse().ok()?, + start: parts.next()?.parse().ok()?, + }; + parts.next().is_none().then_some(process) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/mod.rs b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs new file mode 100644 index 0000000000..1f1fb4ad81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs @@ -0,0 +1,40 @@ +//! Native process runtime: each Sandbox is one daemon process on this kernel, +//! started from a private directory, with no container, image or Engine. +//! +//! It is for a machine that is itself the isolation boundary (a microVM, a +//! CI job, a developer's own Mac). It isolates nothing: the daemon and its +//! commands share this kernel's processes, network, users and filesystem with +//! everything else on it. Commands still run as their own nonroot identity +//! where the platform allows it, and what they leave running ends with the +//! Sandbox. One attempt per operation, as in the Docker runtime; nothing is +//! retried. +//! +//! The lifecycle here is one implementation for both platforms. What differs +//! is read where it lives: [`super::macos`] supplies Darwin process facts, the +//! `mount` table and `umount -f` for the loopback NFS the macOS daemon serves; +//! [`super::linux`] supplies `/proc` facts, the `mountinfo` table and a lazy +//! `MNT_DETACH` for the FUSE mount the root daemon serves. +//! +//! The daemon and its commands start with one search path and nothing else of +//! this process's environment: on Linux a container's, whatever this process +//! has; on macOS this process's own `PATH`, since a native command is this +//! user's own tool (Homebrew's `/opt/homebrew/bin`, `~/.local/bin`, a version +//! manager's `node`) and no image supplies it. +mod create; +mod exec; +mod groups; +pub(crate) mod process; +mod provider; +mod sandbox; +pub(crate) mod search; +#[cfg(target_os = "linux")] +pub(crate) use super::linux as platform; +#[cfg(target_os = "macos")] +pub(crate) use super::macos as platform; +pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; +pub use exec::LocalExit; +pub use process::LocalProcess; +pub use provider::{ + NativeLinuxInstance, NativeLinuxProvider, NativeMacosInstance, NativeMacosProvider, +}; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/native/process.rs b/engine/crates/layerfs-sandbox/src/backend/native/process.rs new file mode 100644 index 0000000000..74d58bf34d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/process.rs @@ -0,0 +1,85 @@ +//! A local process named by its id, kernel start time and boot, so a reused +//! id, or the same id and start time after a reboot or snapshot restore, is +//! never taken for the process that was started. The three facts come from +//! `/proc` on Linux and from `proc_pidinfo`/`sysctl` on macOS (see +//! [`super::macos`]). +pub(crate) use super::platform::{boot, stat}; +use nix::{sys::signal::Signal, unistd::Pid}; +use std::io; + +/// One started process: its id, its start time in clock ticks since boot +/// (`/proc//stat` field 22) and the boot it started in +/// (`/proc/sys/kernel/random/boot_id`). All must match for it to count as +/// alive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalProcess { + pub pid: u32, + pub start: u64, + /// The boot id as a number. `None` only for a record written before boot + /// ids were kept: it is matched by id and start time alone, as it was + /// then, and never signalled. + pub boot: Option, +} + +impl LocalProcess { + /// The process running now under `pid`, read once from this kernel, in + /// this boot. + pub fn observe(pid: u32) -> io::Result { + let stat = stat(pid)?; + Ok(Self { + pid, + start: stat.start, + boot: Some(boot()?), + }) + } + /// True while this exact process exists and has not exited: a zombie + /// (or a dead entry) is not alive, and another process under the same id + /// has another start time or another boot. + pub fn alive(self) -> bool { + let this_boot = self + .boot + .is_none_or(|recorded| boot().ok() == Some(recorded)); + match stat(self.pid) { + Ok(stat) => this_boot && stat.start == self.start && !stat.ended(), + Err(_) => false, + } + } + /// Sends `signal` once, only after this exact process was observed alive. + pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if self.boot.is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "a local process recorded without its boot is never signalled", + )); + } + if !self.alive() { + return Ok(()); + } + match nix::sys::signal::kill(pid(self.pid)?, signal) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), + Err(errno) => Err(errno.into()), + } + } +} + +/// What this kernel says of one process. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Stat { + pub(crate) state: u8, + pub(crate) group: u32, + pub(crate) start: u64, +} + +impl Stat { + /// A zombie or a dead entry: it runs nothing any more. + pub(crate) fn ended(self) -> bool { + matches!(self.state, b'Z' | b'X' | b'x') + } +} + +/// A process id as the kernel's signed type. +pub(super) fn pid(id: u32) -> io::Result { + i32::try_from(id) + .map(Pid::from_raw) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width")) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/provider.rs b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs new file mode 100644 index 0000000000..b5e48d9916 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs @@ -0,0 +1,178 @@ +//! The existing native runtime as a concrete backend; no alternate process engine. +use super::{Local, LocalPhase, LocalProcess, Sandbox}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, LocalRecord, MountDriver, + ProviderCapabilities, ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct NativeProvider { + local: Local, +} +/// Native macOS processes with loopback NFS and the caller's identity. +pub type NativeMacosProvider = NativeProvider; +/// Native Linux processes with FUSE and a separate nonroot command identity. +pub type NativeLinuxProvider = NativeProvider; +/// The existing local owner, using Darwin process facts. +pub type NativeMacosInstance = Sandbox; +/// The existing local owner, using Linux `/proc` process facts. +pub type NativeLinuxInstance = Sandbox; +impl NativeProvider { + /// Refuses other platforms before accessing the root or launching anything. + pub fn new(root: impl AsRef) -> Result { + if (LINUX && !cfg!(target_os = "linux")) || (!LINUX && !cfg!(target_os = "macos")) { + return Err(RuntimeError::Unsupported("native provider platform")); + } + if LINUX && !nix::unistd::Uid::effective().is_root() { + return Err(RuntimeError::Unsupported( + "native Linux provider requires root", + )); + } + Ok(Self { + local: if LINUX { + Local::new(root)? + } else { + Local::native(root)? + }, + }) + } + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: if LINUX { + "native-linux" + } else { + "native-macos" + }, + isolation: IsolationModel::HostProcess, + mount_driver: if LINUX { + MountDriver::Fuse + } else { + MountDriver::LoopbackNfs + }, + requires_image: false, + search_path_policy: if LINUX { + SearchPathPolicy::FixedContainer + } else { + SearchPathPolicy::InheritHost + }, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Local { + command_identity, + mut environment, + } = request.options + else { + return Err(fail(RuntimeError::Unsupported( + "native provider requires local launch options", + ))); + }; + command_identity.check().map_err(fail)?; + if (!LINUX + && (command_identity.uid != nix::unistd::Uid::effective().as_raw() + || command_identity.gid != nix::unistd::Gid::effective().as_raw())) + || environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != if LINUX { "fuse" } else { "nfs" }) + }) + { + return Err(fail(RuntimeError::Protocol( + "native command identity/environment", + ))); + } + environment.retain(|(key, _)| key != "LAYERFS_KERNEL"); + environment.push(( + "LAYERFS_KERNEL".into(), + if LINUX { "fuse" } else { "nfs" }.into(), + )); + let local = self + .local + .clone() + .naming(&request.name) + .map_err(fail)? + .environment(environment); + local + .create( + &request.daemon_executable, + &request.daemon_config, + request.startup_timeout, + request.labels, + ) + .map_err(|failure| { + Box::new(LaunchFailure { + phase: match failure.phase { + LocalPhase::Validate => LaunchPhase::Validate, + LocalPhase::Create => LaunchPhase::Create, + LocalPhase::Start => LaunchPhase::Start, + LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: failure + .sandbox + .map(|sandbox| Box::new(crate::backend::Sandbox::Native(sandbox))), + cause: failure.cause, + }) + }) + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Sandbox::reopen( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + ) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Ok(Sandbox::stopped( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + )) + } +} +fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs new file mode 100644 index 0000000000..8c67717882 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs @@ -0,0 +1,335 @@ +//! One local Sandbox: its directory, its daemon process and its exact +//! attempts. No Drop cleanup or replay, as for a container; a Drop only +//! reaps a daemon this process started that has already exited. A stop +//! returns only once a daemon this process started is reaped, so it never +//! stays a zombie of this process. +use super::{groups, platform, process::LocalProcess}; +use crate::provider::LocalRecord; +use crate::{ContainerId, RuntimeError, WireFailure}; +use nix::sys::signal::Signal; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The daemon's output file in a Sandbox directory, stdout and stderr together. +pub(super) const LOG: &str = "daemon.log"; +/// Pause between two looks at a stopping process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for a killed process to be gone. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged local Sandbox directory and, once started, its daemon. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) directory: PathBuf, + pub(super) labels: Vec<(String, String)>, + pub(super) process: Option, + pub(super) port: u16, + /// The daemon this process started, kept so its exit is reaped here and + /// it does not stay a zombie of this process. `None` after a reopen. + pub(super) child: Mutex>, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + /// Its process groups were killed and found empty (by its stop). + pub(super) ended: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// A directory made by a start that has not spawned its daemon yet. + pub(super) fn unstarted( + id: ContainerId, + directory: PathBuf, + labels: Vec<(String, String)>, + ) -> Self { + Self { + id, + directory, + labels, + process: None, + port: 0, + child: Mutex::new(None), + stop_attempted: false, + stopped: false, + ended: false, + delete_attempted: false, + deleted: false, + } + } + /// Reopens a recorded Sandbox whose daemon still runs: the same process + /// (id and start time) and its loopback port. It starts nothing. + pub fn reopen( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Result> { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + if !owner.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved local Sandbox")); + } + owner.endpoint()?; + Ok(owner) + } + /// A recorded Sandbox that [`Self::reopen`] found stopped + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub fn stopped( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Self { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + /// Its private directory (setup, overlay, Workspace mounts, daemon log). + pub fn directory(&self) -> &Path { + &self.directory + } + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + self.directory.join("tmp") + } + /// Its daemon process, once started. + pub fn process(&self) -> Option { + self.process + } + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let process = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: process.pid, + start_time: process.start, + boot: process.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + /// The daemon's endpoint while that exact process runs. A daemon that + /// exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(process) = self.process else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !process.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then + /// KILL (a daemon this process started has ended once reaped); then + /// KILL to every process group of its commands and its daemon, as a + /// container's end kills what ran in it. A daemon already gone is the + /// same stopped state. Crash teardown, not a graceful filesystem drain. + pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original local Stop admission")); + } + self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = platform::mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } + if let Some(process) = self.process { + self.end( + process, + Signal::SIGTERM, + Duration::from_secs(u64::from(seconds)), + )?; + if !self.gone(process) { + self.end(process, Signal::SIGKILL, KILL_WAIT)?; + } + if process.alive() { + return Err(self.invalid("local daemon still runs after KILL")); + } + } + self.stopped = true; + groups::end(&self.directory, self.process, KILL_WAIT).map_err(|error| self.io(error))?; + self.ended = true; + Ok(()) + } + /// Deletes its directory once, after a known stop: any process left in + /// its groups is killed first (when its stop did not run here, as for a + /// daemon found stopped), and any mount its daemon left is detached, so + /// nothing writes into it or is removed through a mount. The shared + /// Store is outside it and never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.process.is_some() && !self.stopped) { + return Err(self.invalid("original local Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.process.is_some_and(LocalProcess::alive) { + return Err(self.invalid("local daemon runs again; nothing was deleted")); + } + if !self.ended { + groups::end(&self.directory, self.process, KILL_WAIT) + .map_err(|error| self.io(error))?; + self.ended = true; + } + platform::mounts::detach(&self.directory).map_err(|error| self.io(error))?; + match fs::remove_dir_all(&self.directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(self.directory.join(LOG))?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + /// Names the process group `pid` leads or is about to lead: a process + /// started at `start` by a command of this Sandbox in a session of its + /// own (the SDK's `setsid` background process), so it ends with the + /// Sandbox too. One already gone needs nothing. + pub fn track(&self, pid: u32, start: u64) -> Result<(), Box> { + match LocalProcess::observe(pid) { + Ok(process) if process.start == start && process.alive() => { + groups::name(&self.directory, process).map_err(|error| self.io(error)) + } + _ => Ok(()), + } + } + /// Reaps the daemon this process started, if it exited. True when no + /// daemon this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + /// The daemon has ended and, if this process started it, is reaped. Its + /// main thread can end (a zombie id, so not alive) while its other + /// threads still run and it cannot be reaped yet: not gone. + fn gone(&self, process: LocalProcess) -> bool { + self.reap() && !process.alive() + } + /// Sends `signal` once and waits up to `wait` for the process to be gone. + /// A KILL also goes to the daemon this process started and has not + /// reaped: its id cannot be reused until reaped, so it is still the one + /// started, though its main thread may have ended already. + fn end( + &self, + process: LocalProcess, + signal: Signal, + wait: Duration, + ) -> Result<(), Box> { + process.signal(signal).map_err(|error| self.io(error))?; + if signal == Signal::SIGKILL { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + let _ = started.kill(); + } + } + let deadline = Instant::now() + wait; + loop { + if self.gone(process) || Instant::now() >= deadline { + return Ok(()); + } + std::thread::sleep(PAUSE); + } + } + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// A daemon this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running daemon + /// is left alone: stopping it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/search.rs b/engine/crates/layerfs-sandbox/src/backend/native/search.rs new file mode 100644 index 0000000000..3db862f089 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/search.rs @@ -0,0 +1,26 @@ +//! The search path a local daemon and its commands start with: a container's +//! on Linux, this process's own on macOS (the native route). +use std::ffi::OsString; + +/// A container's search path: what a local daemon and its commands start with +/// on Linux, where the local runtime stands in for a container. +const CONTAINER_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on Linux [`CONTAINER_PATH`], whatever this +/// process has. +#[cfg(not(target_os = "macos"))] +pub(crate) fn search_path() -> OsString { + OsString::from(CONTAINER_PATH) +} + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on macOS this process's own `PATH`; without +/// one, or an empty one, [`CONTAINER_PATH`] and Homebrew's `/opt/homebrew/bin`. +#[cfg(target_os = "macos")] +pub(crate) fn search_path() -> OsString { + match std::env::var_os("PATH") { + Some(path) if !path.is_empty() => path, + _ => OsString::from(format!("{CONTAINER_PATH}:/opt/homebrew/bin")), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/runtime.rs b/engine/crates/layerfs-sandbox/src/backend/runtime.rs new file mode 100644 index 0000000000..43ed8208a0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/runtime.rs @@ -0,0 +1,307 @@ +//! The concrete runtimes a Sandbox can run on: one enum names the deployment's +//! choice, and [`Sandbox`] is the launched owner over it. There is no provider +//! trait; each backend is a concrete type and dispatch is a `match` here. +use super::docker::{self, DockerProvider}; +#[cfg(target_os = "linux")] +use super::namespaces::{self, NamespaceProvider}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use super::native::{self, NativeLinuxProvider, NativeMacosProvider}; +use crate::provider::{ + ExecExit, LaunchFailure, LaunchRequest, LocalRecord, ProviderCapabilities, SandboxInspection, + SandboxRecord, +}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +/// What Windows would need and does not have yet: no host runtime exists. +const WINDOWS: &str = "windows native Sandbox runtime"; + +/// One deployment's runtime choice, kept for every Sandbox it starts. +/// +/// A Sandbox's record names the variant that made it; a platform without a +/// runtime is refused before any effect, never replaced by another backend. +#[derive(Clone, Debug)] +pub enum Runtime { + /// A local Docker Engine container; the default. + Docker(DockerProvider), + /// A local daemon process serving loopback NFS as this user (macOS). + #[cfg(any(target_os = "linux", target_os = "macos"))] + MacosNative(NativeMacosProvider), + /// A root local daemon process serving FUSE, commands as a nonroot + /// identity (Linux). + #[cfg(any(target_os = "linux", target_os = "macos"))] + LinuxNative(NativeLinuxProvider), + /// A root daemon in its own mount, PID and IPC namespaces and cgroup, + /// commands entering them as a nonroot identity (Linux). + #[cfg(target_os = "linux")] + Namespaces(NamespaceProvider), + /// Placeholder: selecting Windows refuses before any effect. + Windows, +} + +impl Runtime { + pub fn capabilities(&self) -> Result { + match self { + Self::Docker(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => Ok(provider.capabilities()), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => Ok(provider.capabilities()), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// Local resource ownership, checked by API before encoding daemon topology. + /// Nonlocal providers return None. No resource is created by this observation. + pub fn local_directory(&self, name: &str) -> Result, RuntimeError> { + match self { + Self::Docker(_) => Ok(None), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// The named native Sandbox's paths, for its daemon setup. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn native_paths(&self, name: &str) -> Result { + match self { + Self::MacosNative(provider) => provider.paths(name), + Self::LinuxNative(provider) => provider.paths(name), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name), + Self::Docker(_) | Self::Windows => Err(RuntimeError::Unsupported( + "native Sandbox paths need a native runtime", + )), + } + } + + /// Starts one Sandbox once; a failed launch keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + match self { + Self::Docker(provider) => provider.launch(request).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .launch(request) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(Box::new(LaunchFailure { + phase: crate::provider::LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(WINDOWS), + })), + } + } + + /// Reopen starts nothing and checks the recorded deployment or process identity. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen_stopped(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen_stopped(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } +} + +/// The launched owner of one Sandbox, on the runtime that started it. +#[derive(Debug)] +pub enum Sandbox { + Docker(docker::Sandbox), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Native(native::Sandbox), + #[cfg(target_os = "linux")] + Namespaces(Box), +} + +impl Sandbox { + pub fn identity(&self) -> ContainerId { + match self { + Self::Docker(sandbox) => sandbox.identity(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.identity(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.identity(), + } + } + + pub fn endpoint(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.endpoint(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.endpoint(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.endpoint(), + } + } + + /// One command to its end, its streams copied to the sinks as they arrive. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + match self { + Self::Docker(sandbox) => docker::run_command(sandbox, request, stdout, stderr), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + let exit = sandbox.run(request, stdout, stderr)?; + Ok(ExecExit { + code: exit.code, + pid: Some(exit.pid), + stdout_bytes: exit.stdout, + stderr_bytes: exit.stderr, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.run(request, stdout, stderr), + } + } + + /// Names a background process a command started in a session of its own, + /// so it ends with the Sandbox as a container's descendants do. + pub fn track(&self, pid: u32, start_time: u64) -> Result<(), Box> { + match self { + // Container teardown owns all descendants; no host process is adopted. + Self::Docker(_) => Ok(()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.track(pid, start_time), + // its cgroup owns every process the Sandbox started, whatever + // session it left, so nothing needs naming + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.track(pid, start_time), + } + } + + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + match self { + Self::Docker(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.log_tail(lines, cap), + } + } + + pub fn inspect(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.inspect(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + } + } + + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.stop(timeout_seconds), + } + } + + pub fn delete(&mut self) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.delete(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.delete(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.delete(), + } + } + + /// Where its commands keep scratch files: `/tmp` in a container, the + /// Sandbox's own directory for a native one, deleted with it. + pub fn scratch_directory(&self) -> String { + match self { + Self::Docker(_) => "/tmp".into(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.tmp().display().to_string(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.tmp().display().to_string(), + } + } + + /// Physical facts for the binding's existing record format, when this is native. + pub fn local_record(&self) -> Option { + match self { + Self::Docker(_) => None, + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.local_record(), + #[cfg(target_os = "linux")] + Self::Namespaces(_) => None, + } + } + + /// Persisted deployment facts for reopening this runtime instance. + pub fn record(&self) -> Option { + match self { + Self::Docker(sandbox) => Some(SandboxRecord::Docker(sandbox.request().clone().into())), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(_) => self.local_record().map(SandboxRecord::Local), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.record(), + } + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/streams.rs b/engine/crates/layerfs-sandbox/src/backend/streams.rs new file mode 100644 index 0000000000..6c6e011a17 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/streams.rs @@ -0,0 +1,23 @@ +//! An ordinary command's streams, copied to the caller's sinks as they +//! arrive. Both local backends read the same two pipes and count only what +//! each sink accepted. +use std::io::{self, Read, Write}; + +/// Copies one stream to its sink until end of stream; the bytes delivered. +pub(crate) fn copy(source: Option, sink: &mut (dyn Write + Send)) -> io::Result { + let Some(mut source) = source else { + return Ok(0); + }; + let mut total = 0u64; + let mut window = [0u8; 8192]; + loop { + let n = match source.read(&mut window) { + Ok(0) => return Ok(total), + Ok(n) => n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + sink.write_all(&window[..n])?; + total = total.saturating_add(n as u64); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs b/engine/crates/layerfs-sandbox/src/backend/strings.rs similarity index 86% rename from engine/crates/layerfs-sandbox/src/backend/docker/strings.rs rename to engine/crates/layerfs-sandbox/src/backend/strings.rs index 1d2afc32e8..fb22cf7c79 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs +++ b/engine/crates/layerfs-sandbox/src/backend/strings.rs @@ -41,6 +41,27 @@ impl Json { self.string_each(|c| text.push(c, capture))?; Ok(text) } + /// Reads a string value into `out` as UTF-8, refusing one longer than + /// `max`: the Engine's selected fields are short, and the longest string + /// this crate reads is an image archive's path. Its only caller unpacks + /// images, which this crate does on Linux only. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn text_into(&mut self, out: &mut Vec, max: usize) -> Result<(), RuntimeError> { + let mut too_long = false; + self.string_each(|c| { + let mut encoded = [0; 4]; + let bytes = c.encode_utf8(&mut encoded).as_bytes(); + if out.len() + bytes.len() > max { + too_long = true; + } else { + out.extend_from_slice(bytes); + } + })?; + if too_long { + return Err(RuntimeError::Protocol("JSON string length")); + } + Ok(()) + } pub fn string_equals(&mut self, expected: &str) -> Result { let mut expected = expected.chars(); let mut equal = true; diff --git a/engine/crates/layerfs-sandbox/src/backend/windows/README.md b/engine/crates/layerfs-sandbox/src/backend/windows/README.md new file mode 100644 index 0000000000..b667620c81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/windows/README.md @@ -0,0 +1,49 @@ +# Windows：占位（尚未实现） + +状态：占位。选到 `Runtime::Windows` 时返回既有的“不支持此平台”错误 +（`RuntimeError::Unsupported`），不 panic、不新建公开错误码、不产生任何副作用。 + +## 现在有什么 + +- `layerfs_sandbox::Runtime` 的四个取值：`Docker`、`MacosNative`、 + `LinuxNative`、`Windows`。前三个已有后端，Windows 只有这个变体。 +- `Runtime::{capabilities, local_directory, launch, reopen, reopen_stopped}` + 在 Windows 上全部返回 `Unsupported`；`Sandbox` 句柄没有 Windows 分支， + 因为 Windows 上不可能启动出实例。 +- `layerfs-sandbox` 的 `backend` 只在 unix 下编译，`layerfs-node` 也只用 + unix 系统调用，所以今天在 Windows 主机上选不到这个变体：它存在的意义是 + 把“没有 Windows 宿主运行时”写进类型，而不是让调用方自己判平台。 + +## 为什么不做“Windows 本地进程”后端 + +macOS 原生的前提是：daemon 以本用户身份跑 NFS 回环服务，命令以同一身份 +运行，一切在同一个内核里。Windows 上没有等价物：LayerFS 的 FUSE 服务端是 +Linux 内核特性，Store 的 WAL 与目录锁只在一个内核内协调（见 #352 直接 +Store 访问的安全条件），命令身份、挂载可见性和进程组语义都要另建一套。 +因此 Windows 的路线不是“再写一个本地后端”，而是把 Linux 侧的能力搬到一个 +Linux 宿主里。 + +## 计划方向 + +两条路线，都由 #515 的 placement 设置表达，而不是在 Windows 本机上跑 daemon： + +1. **WSL2 分发（近期）**：Windows 上的集成适配器把 SDK 调用转发给 + 每个 LayerFS home 一个的、仅回环且带密钥的 worker；worker 在一个选定的 + WSL2 Linux 分发里直接用真实 SDK 打开共享 Store（ext4/xfs/btrfs）， + 并用该分发的 Docker Engine 起 Sandbox daemon。Windows 只拥有 UI、绑定和 + trace，Linux 拥有本地句柄和项目发现。这是仓库根 `AGENTS.md` 已记录的 + Windows 例外，规则不变：只连回环、每次调用只尝试一次、丢失的回复不重放、 + 结果不确定的 Init 保留其 Linux home 和卷。 +2. **Hyper-V / VM 方向（后续）**：每个 Store 一个 Linux VM，Sandbox 是 VM 内 + 的命名空间；宿主侧只通过一条私有流连到 VM 内的 worker。见 #515 §6 + （`vm:` placement 与各 provider 的 VM/磁盘/私有流对照表）和 + #512 的实验结论。 + +## 实现时要先定的事 + +- placement 与后端选择的设置名（#515 V9 提议 `LAYERFS_PLACEMENT` 与 + `LAYERFS_SANDBOX_BACKEND`，只读 `LAYERFS_*`；集成自己的前缀由集成映射）。 +- Windows 上 `deploy.sandbox`、`deploy.localDir` 等现有设置的语义：#515 + 要求 product SDK 与 daemon 不为 Windows 改动，Windows 差异留在适配器里。 +- 到那时 `Runtime` 是否新增一个持有 worker 端点的变体（那时才需要数据）， + 以及记录里如何标注该 Sandbox 由哪个运行时创建（无字段即 Docker）。 diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs new file mode 100644 index 0000000000..6135b177e7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs @@ -0,0 +1,168 @@ +//! The exec stage: one command inside a running Sandbox. The helper joins the +//! Sandbox's cgroup first (after entering the Sandbox's mount namespace this +//! host path no longer resolves), opens the three namespace handles, then +//! enters them. Its child is created inside the Sandbox's PID namespace, runs +//! as the requested uid:gid and inherits no-new-privs, so file capabilities +//! and setuid bits cannot raise it again. +use nix::{ + sched::{setns, CloneFlags}, + sys::prctl::set_no_new_privs, +}; +use std::{ + ffi::OsString, + fs::{self, File}, + io::Write, + os::unix::process::{CommandExt, ExitStatusExt}, + path::{Path, PathBuf}, + process::Command, +}; + +/// One parsed exec stage. +struct Run { + init: u32, + cgroup: PathBuf, + uid: u32, + gid: u32, + directory: PathBuf, + arguments: Vec, +} + +/// Runs one command and exits with its code (128 + signal for one a signal +/// ended). A setup failure ends this process with a signal, so the caller +/// never reads it as a command's exit code. +pub(super) fn run(arguments: &[OsString]) -> Result { + let run = parse(arguments)?; + join(&run.cgroup)?; + let namespaces = Namespaces::open(run.init)?; + namespaces.enter()?; + set_no_new_privs().map_err(|error| format!("setting no-new-privs: {error}"))?; + let (program, rest) = run.arguments.split_first().ok_or("no command")?; + let mut command = Command::new(program); + command + .args(rest) + .uid(run.uid) + .gid(run.gid) + .current_dir(&run.directory) + .process_group(0); + let mut child = match command.spawn() { + Ok(child) => child, + // the command never started: this stage's own failure, which must + // never look like the command's status + Err(error) => fail(&format!("starting {}: {error}", program.to_string_lossy())), + }; + let status = child + .wait() + .map_err(|error| format!("waiting for the command: {error}"))?; + Ok(status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(super::EXIT_FAILURE)) +} + +/// The three namespace handles of a Sandbox's init stage, all opened before +/// any is entered: after entering the Sandbox's mount namespace this host's +/// `/proc//ns` no longer resolves. +struct Namespaces { + mount: File, + pid: File, + ipc: File, +} + +impl Namespaces { + fn open(init: u32) -> Result { + let directory = PathBuf::from(format!("/proc/{init}/ns")); + let open = |name: &str| { + File::open(directory.join(name)) + .map_err(|error| format!("{}: {error}", directory.join(name).display())) + }; + Ok(Self { + mount: open("mnt")?, + pid: open("pid_for_children").or_else(|_| open("pid"))?, + ipc: open("ipc")?, + }) + } + + fn enter(self) -> Result<(), String> { + let Self { mount, pid, ipc } = self; + setns(mount, CloneFlags::CLONE_NEWNS) + .map_err(|error| format!("entering the Sandbox's mount namespace: {error}"))?; + setns(pid, CloneFlags::CLONE_NEWPID) + .map_err(|error| format!("entering the Sandbox's PID namespace: {error}"))?; + setns(ipc, CloneFlags::CLONE_NEWIPC) + .map_err(|error| format!("entering the Sandbox's IPC namespace: {error}")) + } +} + +/// The Sandbox's own cgroup, so `cgroup.kill` ends this command too. +fn join(cgroup: &Path) -> Result<(), String> { + let pid = nix::unistd::getpid().as_raw(); + let mut file = fs::OpenOptions::new() + .write(true) + .open(cgroup.join("cgroup.procs")) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}"))?; + file.write_all(pid.to_string().as_bytes()) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}")) +} + +/// Ends this stage the way only a failure can: with a signal, never an exit +/// code, so the caller cannot read it as the command's status. +fn fail(message: &str) -> ! { + eprintln!("layerfs-sandbox-helper: {message}"); + let _ = nix::sys::signal::raise(nix::sys::signal::Signal::SIGKILL); + std::process::abort() +} + +fn parse(arguments: &[OsString]) -> Result { + let (mut init, mut uid, mut gid) = (0, 0, 0); + let (mut cgroup, mut directory) = (PathBuf::new(), PathBuf::new()); + let mut command = Vec::new(); + let mut index = 0; + while index < arguments.len() { + let flag = arguments[index].to_str(); + if flag == Some("--") { + command.extend(arguments[index + 1..].iter().cloned()); + break; + } + let value = arguments + .get(index + 1) + .ok_or_else(|| format!("{} needs a value", arguments[index].to_string_lossy()))?; + match flag { + Some("--init") => init = number(value)?, + Some("--uid") => uid = number(value)?, + Some("--gid") => gid = number(value)?, + Some("--cgroup") => cgroup = PathBuf::from(value), + Some("--cwd") => directory = PathBuf::from(value), + _ => { + return Err(format!( + "unknown argument {}", + arguments[index].to_string_lossy() + )) + } + } + index += 2; + } + if init == 0 + || uid == 0 + || gid == 0 + || cgroup.as_os_str().is_empty() + || !directory.is_absolute() + || command.is_empty() + { + return Err("missing or invalid exec argument".to_owned()); + } + Ok(Run { + init, + cgroup, + uid, + gid, + directory, + arguments: command, + }) +} + +fn number(value: &OsString) -> Result { + value + .to_str() + .and_then(|text| text.parse().ok()) + .ok_or_else(|| format!("{} is not a number", value.to_string_lossy())) +} diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs new file mode 100644 index 0000000000..1576db1163 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs @@ -0,0 +1,109 @@ +//! The two init stages. The outer one creates the namespaces and joins the +//! cgroup; its first child, the inner one, is PID 1 of the new PID namespace, +//! so it builds the private root, starts the daemon and reaps. Neither stage +//! retries anything: a failure ends the stage and is written to the Sandbox's +//! daemon log, which the start's error keeps. +use super::mounts; +use layerfs_sandbox::backend::HelperSpec; +use nix::{ + sched::{unshare, CloneFlags}, + sys::wait::{waitpid, WaitStatus}, + unistd::Pid, +}; +use std::{ + fs, + io::Write, + path::Path, + process::{Command, Stdio}, +}; + +/// The outer init stage: joins the Sandbox's cgroup, becomes the first +/// process of its mount, PID and IPC namespaces, and starts the inner stage. +/// Its exit code is the daemon's, so the Sandbox ends when the daemon does. +pub(super) fn outer(spec_path: &Path) -> Result { + let spec = read(spec_path)?; + join(&spec.cgroup)?; + unshare(CloneFlags::CLONE_NEWNS | CloneFlags::CLONE_NEWPID | CloneFlags::CLONE_NEWIPC) + .map_err(|error| format!("creating the Sandbox's namespaces: {error}"))?; + // this mount namespace is the Sandbox's alone: nothing it mounts + // propagates back to the VM's own mount tree + nix::mount::mount( + None::<&Path>, + Path::new("/"), + None::<&str>, + nix::mount::MsFlags::MS_REC | nix::mount::MsFlags::MS_PRIVATE, + None::<&str>, + ) + .map_err(|error| format!("making the Sandbox's mount tree private: {error}"))?; + let executable = + std::env::current_exe().map_err(|error| format!("this helper's own path: {error}"))?; + let mut inner = Command::new(executable) + .arg("init-inner") + .arg("--spec") + .arg(spec_path) + .spawn() + .map_err(|error| format!("starting the Sandbox's init stage: {error}"))?; + let status = inner + .wait() + .map_err(|error| format!("waiting for the Sandbox's init stage: {error}"))?; + Ok(code(status)) +} + +/// The inner init stage, PID 1 of the Sandbox's PID namespace: builds and +/// pivots to the private root, starts the daemon as root and reaps, so no +/// process of the Sandbox is left a zombie. +pub(super) fn inner(spec_path: &Path) -> Result { + let spec = read(spec_path)?; + mounts::build(&spec)?; + mounts::pivot(&spec.root)?; + let daemon = Command::new(&spec.daemon) + .arg("--config") + .arg(&spec.config) + .current_dir("/") + .stdin(Stdio::null()) + .spawn() + .map_err(|error| format!("starting the daemon: {error}"))?; + let daemon_pid = Pid::from_raw( + i32::try_from(daemon.id()).map_err(|_| "daemon process id width".to_owned())?, + ); + loop { + match waitpid(None, None) { + Ok(WaitStatus::Exited(pid, code)) if pid == daemon_pid => return Ok(code), + Ok(WaitStatus::Signaled(pid, signal, _)) if pid == daemon_pid => { + return Ok(128 + signal as i32) + } + // an orphan of a command: reaped here, as PID 1 does + Ok(_) => continue, + Err(nix::errno::Errno::ECHILD) => { + return Err("the daemon ended without a waitable status".to_owned()) + } + Err(error) => return Err(format!("reaping the Sandbox's processes: {error}")), + } + } +} + +/// The Sandbox's own cgroup, written by the Sandbox's own init stage: every +/// process it creates afterwards inherits it, so one write ends them all. +fn join(cgroup: &Path) -> Result<(), String> { + let pid = nix::unistd::getpid().as_raw(); + let mut file = fs::OpenOptions::new() + .write(true) + .open(cgroup.join("cgroup.procs")) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}"))?; + file.write_all(pid.to_string().as_bytes()) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}")) +} + +fn read(spec_path: &Path) -> Result { + let bytes = + fs::read(spec_path).map_err(|error| format!("reading {}: {error}", spec_path.display()))?; + HelperSpec::parse(&bytes).map_err(|error| format!("reading {}: {error}", spec_path.display())) +} + +fn code(status: std::process::ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(super::EXIT_FAILURE) +} diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs new file mode 100644 index 0000000000..2e9b470c00 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs @@ -0,0 +1,65 @@ +//! The namespaces backend's helper: one single-threaded binary with two +//! stages, so the calls that act on the calling process (`unshare`, `setns`, +//! `pivot_root`, `PR_SET_NO_NEW_PRIVS`) run in a process of their own and the +//! product crate stays free of `unsafe`. +//! +//! - `init --spec `: joins the Sandbox's cgroup, creates its mount, PID +//! and IPC namespaces and starts the inner stage, which is PID 1 of them. +//! - `init-inner --spec `: builds the private root, pivots to it, starts +//! `layerfs-daemon` as root and reaps, as an init process does. +//! - `exec …`: enters a running Sandbox's namespaces, becomes the requested +//! nonroot identity with no-new-privs and runs one command. +#![deny(unsafe_code)] + +#[cfg(target_os = "linux")] +mod exec; +#[cfg(target_os = "linux")] +mod init; +#[cfg(target_os = "linux")] +mod mounts; + +#[cfg(target_os = "linux")] +fn main() { + match run() { + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("layerfs-sandbox-helper: {error}"); + std::process::exit(EXIT_FAILURE); + } + } +} + +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!("layerfs-sandbox-helper: the namespaces Sandbox runtime is Linux only"); + std::process::exit(125); +} + +/// A stage's own failure; a command's exit code is never this. +#[cfg(target_os = "linux")] +const EXIT_FAILURE: i32 = 125; + +#[cfg(target_os = "linux")] +fn run() -> Result { + let arguments: Vec = std::env::args_os().skip(1).collect(); + let (stage, rest) = arguments.split_first().ok_or(USAGE)?; + match stage.to_str() { + Some("init") => init::outer(spec(rest)?), + Some("init-inner") => init::inner(spec(rest)?), + Some("exec") => exec::run(rest), + _ => Err(USAGE.to_owned()), + } +} + +/// `--spec `, the only argument of either init stage. +#[cfg(target_os = "linux")] +fn spec(arguments: &[std::ffi::OsString]) -> Result<&std::path::Path, String> { + match arguments { + [flag, path] if flag == "--spec" => Ok(std::path::Path::new(path)), + _ => Err(USAGE.to_owned()), + } +} + +#[cfg(target_os = "linux")] +const USAGE: &str = "usage: layerfs-sandbox-helper init|init-inner --spec | exec --init \ + --cgroup --uid --gid --cwd -- …"; diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs new file mode 100644 index 0000000000..426faba656 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs @@ -0,0 +1,201 @@ +//! The Sandbox's root: the unpacked image (a read-only lower layer) with the +//! Sandbox's own empty writable layer on top, so every command sees the image +//! as `/` and everything it writes belongs to that Sandbox alone. The +//! Sandbox's own directory and the Store's directory are bound at their own +//! paths so every path keeps one meaning on both sides of the pivot, and a +//! private `/proc`, `/dev` and `/tmp` are mounted over the image's own. +//! Nothing the Sandbox mounts reaches the VM's mount tree: the init stage made +//! this mount namespace private first. +use layerfs_sandbox::backend::HelperSpec; +use nix::{ + mount::{mount, umount2, MntFlags, MsFlags}, + unistd::chdir, +}; +use std::{ + fs, io, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +/// The devices a daemon and ordinary commands need, bound from this host's +/// `/dev`; nothing else of `/dev` is visible. +const DEVICES: [&str; 7] = ["fuse", "null", "zero", "full", "random", "urandom", "tty"]; + +/// Builds the Sandbox's root at `spec.root`: the image as the overlay's lower +/// layer, the Sandbox's own `upper` and `work` above it, and the private mounts +/// that must not come from the image. +pub(super) fn build(spec: &HelperSpec) -> Result<(), String> { + fs::set_permissions(&spec.root, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("{}: {error}", spec.root.display()))?; + overlay(spec)?; + // an overlay's own root takes its attributes from the empty upper + // directory, which is root's 0700 on this host: the root a command sees + // must instead be one it can traverse, as a container's `/` is + fs::set_permissions(&spec.root, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("{}: {error}", spec.root.display()))?; + let proc_directory = inside(&spec.root, Path::new("/proc"))?; + mount( + None::<&Path>, + &proc_directory, + Some("proc"), + MsFlags::MS_NOSUID | MsFlags::MS_NODEV | MsFlags::MS_NOEXEC, + None::<&str>, + ) + .map_err(|error| format!("mounting {}: {error}", proc_directory.display()))?; + devices(&inside(&spec.root, Path::new("/dev"))?)?; + let scratch = inside(&spec.root, Path::new("/tmp"))?; + tmpfs( + &scratch, + MsFlags::MS_NOSUID | MsFlags::MS_NODEV, + "mode=1777", + )?; + // the daemon's setup, overlay, Workspace mounts and `tmp` + bind(&spec.sandbox, &inside(&spec.root, &spec.sandbox)?, false)?; + // the shared Store's directory: SQLite's WAL and the writers' directory + // lock see the same inode as every other opener on this kernel + bind(&spec.store, &inside(&spec.root, &spec.store)?, false)?; + Ok(()) +} + +/// The image as the root's lower layer and the Sandbox's own upper and work +/// directories above it. The result is one mount, so a command's writes stay +/// in the upper directory and the image is never modified. +fn overlay(spec: &HelperSpec) -> Result<(), String> { + let data = format!( + "lowerdir={},upperdir={},workdir={}", + spec.lower.display(), + spec.upper.display(), + spec.work.display() + ); + mount( + Some("overlay"), + &spec.root, + Some("overlay"), + MsFlags::MS_NOSUID, + Some(data.as_str()), + ) + .map_err(|error| { + format!( + "mounting the image {} over {} with {}: {error}", + spec.lower.display(), + spec.root.display(), + data + ) + }) +} + +/// Pivots to `root`, so the VM's own tree is no longer reachable from the +/// Sandbox, and drops the old root. +pub(super) fn pivot(root: &Path) -> Result<(), String> { + // the private root must be a mount point of its own, which the overlay is + chdir(root).map_err(|error| format!("{}: {error}", root.display()))?; + nix::unistd::pivot_root(Path::new("."), Path::new(".")) + .map_err(|error| format!("pivoting to the Sandbox's root: {error}"))?; + umount2(Path::new("."), MntFlags::MNT_DETACH) + .map_err(|error| format!("dropping the VM's root: {error}"))?; + chdir("/").map_err(|error| format!("entering the Sandbox's root: {error}")) +} + +/// A private `/dev`: the basic devices, a `devpts` of its own for pseudo +/// terminals, and a `shm` directory for what expects one. +fn devices(dev: &Path) -> Result<(), String> { + tmpfs(dev, MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC, "mode=0755")?; + let host = Path::new("/dev"); + for name in DEVICES { + let source = host.join(name); + if fs::symlink_metadata(&source).is_err() { + continue; + } + // a bind needs its target to exist; the device then replaces this + // placeholder in the private tmpfs + let target = dev.join(name); + fs::write(&target, b"").map_err(|error| format!("{}: {error}", target.display()))?; + bind(&source, &target, false)?; + } + let pts = dev.join("pts"); + directory(&pts, 0o755)?; + mount( + None::<&Path>, + &pts, + Some("devpts"), + MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC, + Some("newinstance,ptmxmode=0666,mode=0620"), + ) + .map_err(|error| format!("mounting {}: {error}", pts.display()))?; + let shm = dev.join("shm"); + directory(&shm, 0o755)?; + tmpfs(&shm, MsFlags::MS_NOSUID | MsFlags::MS_NODEV, "mode=1777")?; + for (name, target) in [ + ("ptmx", "pts/ptmx"), + ("fd", "/proc/self/fd"), + ("stdin", "/proc/self/fd/0"), + ("stdout", "/proc/self/fd/1"), + ("stderr", "/proc/self/fd/2"), + ] { + let link = dev.join(name); + let _ = fs::remove_file(&link); + std::os::unix::fs::symlink(target, &link) + .map_err(|error| format!("{}: {error}", link.display()))?; + } + Ok(()) +} + +/// `path`'s place inside `root`, made: `/a/b` becomes `/a/b`. +fn inside(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix("/") + .map_err(|_| format!("{} is not an absolute runtime path", path.display()))?; + let target = root.join(relative); + fs::create_dir_all(&target).map_err(|error| format!("{}: {error}", target.display()))?; + Ok(target) +} + +/// A tmpfs at `target`, so nothing it holds outlives the namespaces. +fn tmpfs(target: &Path, flags: MsFlags, data: &str) -> Result<(), String> { + mount( + Some("tmpfs"), + target, + Some("tmpfs"), + flags | MsFlags::MS_NOSUID, + Some(data), + ) + .map_err(|error| format!("mounting a tmpfs at {}: {error}", target.display())) +} + +/// `source` at `target`, read-only when `read_only`. +fn bind(source: &Path, target: &Path, read_only: bool) -> Result<(), String> { + mount( + Some(source), + target, + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REC, + None::<&str>, + ) + .map_err(|error| { + format!( + "binding {} at {}: {error}", + source.display(), + target.display() + ) + })?; + if !read_only { + return Ok(()); + } + mount( + None::<&Path>, + target, + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REMOUNT | MsFlags::MS_RDONLY, + None::<&str>, + ) + .map_err(|error| format!("making {} read-only: {error}", target.display())) +} + +fn directory(path: &Path, mode: u32) -> Result<(), String> { + if let Err(error) = fs::DirBuilder::new().mode(mode).create(path) { + if error.kind() != io::ErrorKind::AlreadyExists { + return Err(format!("{}: {error}", path.display())); + } + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..497fa2024b 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,9 +1,14 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. -#![forbid(unsafe_code)] +// unsafe only in backend/macos/darwin.rs: macOS process facts come from +// proc_pidinfo, a system call (`ps` is setuid, which Seatbelt refuses) +#![deny(unsafe_code)] #[cfg(unix)] pub mod backend; +pub mod provider; mod types; +#[cfg(unix)] +pub use backend::{Runtime, Sandbox}; pub use types::{ - CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, PendingRequest, - RuntimeError, WireFailure, + CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, ExecRequest, Image, + PendingRequest, RuntimeError, WireFailure, }; diff --git a/engine/crates/layerfs-sandbox/src/provider/capabilities.rs b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs new file mode 100644 index 0000000000..a11548c3eb --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs @@ -0,0 +1,30 @@ +//! Explicit runtime capabilities; these describe support, not qualification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProviderCapabilities { + pub name: &'static str, + pub isolation: IsolationModel, + pub mount_driver: MountDriver, + pub requires_image: bool, + pub search_path_policy: SearchPathPolicy, + pub supports_resource_limits: bool, + pub direct_store_access: bool, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IsolationModel { + Container, + HostProcess, + MicroVm, + RemoteCloud, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MountDriver { + Fuse, + LoopbackNfs, + SharedFs, + None, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchPathPolicy { + FixedContainer, + InheritHost, +} diff --git a/engine/crates/layerfs-sandbox/src/provider/launch.rs b/engine/crates/layerfs-sandbox/src/provider/launch.rs new file mode 100644 index 0000000000..0c654ad481 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/launch.rs @@ -0,0 +1,26 @@ +//! One-attempt launch failure: acknowledged partial ownership and the last phase reached. +use crate::backend::Sandbox; +use crate::RuntimeError; +use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LaunchPhase { + Validate, + Create, + Start, + Listener, +} +/// The caller retains the exact resource even when launch fails after creating it. +pub struct LaunchFailure { + pub phase: LaunchPhase, + pub instance: Option>, + pub cause: RuntimeError, +} +impl fmt::Debug for LaunchFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchFailure") + .field("phase", &self.phase) + .field("identity", &self.instance.as_ref().map(|i| i.identity())) + .field("cause", &self.cause) + .finish() + } +} diff --git a/engine/crates/layerfs-sandbox/src/provider/mod.rs b/engine/crates/layerfs-sandbox/src/provider/mod.rs new file mode 100644 index 0000000000..b9eee1e304 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/mod.rs @@ -0,0 +1,12 @@ +//! Runtime inputs and observations; concrete backends own their own lifecycle. +mod capabilities; +mod launch; +mod options; +pub use capabilities::{IsolationModel, MountDriver, ProviderCapabilities, SearchPathPolicy}; +pub use launch::{LaunchFailure, LaunchPhase}; +pub use options::{ + DockerRecord, ExecExit, LaunchRequest, LocalRecord, NamespaceRecord, ProviderLaunchOptions, + SandboxInspection, SandboxRecord, +}; +/// Compatible name for the existing exact 64-hex runtime identity. +pub type SandboxId = crate::ContainerId; diff --git a/engine/crates/layerfs-sandbox/src/provider/options.rs b/engine/crates/layerfs-sandbox/src/provider/options.rs new file mode 100644 index 0000000000..0ea23bd979 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/options.rs @@ -0,0 +1,193 @@ +//! Launch inputs and observations independent of the daemon control protocol. +use crate::{CommandIdentity, ContainerId}; +use std::{fmt, path::PathBuf, time::Duration}; +/// Configuration bytes may contain secrets; Debug only reports their length. +#[derive(Clone)] +pub struct LaunchRequest { + pub name: String, + pub daemon_executable: PathBuf, + pub daemon_config: Vec, + pub labels: Vec<(String, String)>, + pub startup_timeout: Duration, + pub options: ProviderLaunchOptions, +} +impl fmt::Debug for LaunchRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchRequest") + .field("name", &self.name) + .field("daemon_executable", &self.daemon_executable) + .field("config_bytes", &self.daemon_config.len()) + .field("startup_timeout", &self.startup_timeout) + .finish_non_exhaustive() + } +} +#[derive(Clone, Debug)] +pub enum ProviderLaunchOptions { + Local { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + }, + /// The Linux namespace backend: a namespaced daemon plus the Store file it + /// opens. `store` is the exact file; its directory is what the Sandbox's + /// private root binds, so SQLite's WAL and the writers' directory lock see + /// the same inode as every other opener on this kernel. `image` is the + /// digest its root is an overlay of, and `archive` the docker-archive in + /// the bundle that digest names, which is unpacked once per digest. + Namespaces { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + store: PathBuf, + image: crate::Image, + archive: PathBuf, + }, + Docker { + image: String, + store_volume: String, + port: u16, + connect_host: Option, + }, +} +/// Persisted physical facts supplied by the record owner; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalRecord { + pub name: String, + pub pid: u32, + pub start_time: u64, + pub boot: Option, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// A namespaced Linux Sandbox's record: the local facts (its directory name, +/// init process identity and listener port) and the image digest its root is +/// an overlay of, so a reopen finds the same unpacked image. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamespaceRecord { + pub local: LocalRecord, + pub image: crate::Image, +} +impl NamespaceRecord { + /// The physical local facts every record of this backend keeps. + pub fn local(&self) -> &LocalRecord { + &self.local + } + /// The image digest the Sandbox's root is an overlay of. + pub fn image(&self) -> &crate::Image { + &self.image + } +} +/// Persisted deployment facts for a Docker container; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DockerRecord { + pub image: String, + pub store_volume: String, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// Recorded runtime identity and deployment facts for reopen. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SandboxRecord { + Local(LocalRecord), + /// A namespaced Linux Sandbox; the physical facts kept are the same as a + /// local one (its directory name, init process identity and listener port), + /// plus the image digest its root is an overlay of. The variant itself + /// names the backend, so a record is never reopened by another one. + Namespaces(NamespaceRecord), + Docker(DockerRecord), +} +impl SandboxRecord { + pub fn local(&self) -> Option<&LocalRecord> { + match self { + Self::Local(record) => Some(record), + _ => None, + } + } + pub fn namespaces(&self) -> Option<&NamespaceRecord> { + match self { + Self::Namespaces(record) => Some(record), + _ => None, + } + } + pub fn docker(&self) -> Option<&DockerRecord> { + match self { + Self::Docker(record) => Some(record), + _ => None, + } + } +} +impl From for SandboxRecord { + fn from(record: LocalRecord) -> Self { + Self::Local(record) + } +} +impl From<&LocalRecord> for SandboxRecord { + fn from(record: &LocalRecord) -> Self { + Self::Local(record.clone()) + } +} +impl From for SandboxRecord { + fn from(record: DockerRecord) -> Self { + Self::Docker(record) + } +} +impl From<&DockerRecord> for SandboxRecord { + fn from(record: &DockerRecord) -> Self { + Self::Docker(record.clone()) + } +} +impl From for DockerRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image, + store_volume: request.store_volume, + port: request.port, + labels: request.labels, + } + } +} +impl From<&crate::backend::docker::SandboxRequest> for DockerRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image.clone(), + store_volume: request.store_volume.clone(), + port: request.port, + labels: request.labels.clone(), + } + } +} +impl From for crate::backend::docker::SandboxRequest { + fn from(record: DockerRecord) -> Self { + Self { + image: record.image, + store_volume: record.store_volume, + port: record.port, + labels: record.labels, + } + } +} +impl From for SandboxRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +impl From<&crate::backend::docker::SandboxRequest> for SandboxRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecExit { + pub code: i32, + /// Docker may not expose a process id after exit. + pub pid: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, +} +/// Unknown resource/timestamp fields stay None rather than invented zeroes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SandboxInspection { + pub identity: ContainerId, + pub running: bool, + pub exit_code: Option, + pub process_id: Option, + pub scratch_bytes: Option, +} diff --git a/engine/crates/layerfs-sandbox/src/types/failure.rs b/engine/crates/layerfs-sandbox/src/types/failure.rs index de4ddfe27d..26c3b3d915 100644 --- a/engine/crates/layerfs-sandbox/src/types/failure.rs +++ b/engine/crates/layerfs-sandbox/src/types/failure.rs @@ -3,6 +3,9 @@ use std::{fmt, io}; /// Original ordinary-runtime refusal or I/O cause. #[derive(Debug)] pub enum RuntimeError { + /// Original Docker provider stage failure, including retained transfer/stream owners. + #[cfg(unix)] + Docker(Box), Io(io::Error), Protocol(&'static str), /// Original HTTP status, with a separately retained bounded error-body diagnostic. @@ -18,6 +21,8 @@ impl From for RuntimeError { impl fmt::Display for RuntimeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + #[cfg(unix)] + Self::Docker(failure) => write!(f, "Docker provider: {failure:?}"), Self::Io(e) => write!(f, "runtime I/O: {e}"), Self::Protocol(e) => write!(f, "runtime protocol: {e}"), Self::Http(s) => write!(f, "runtime HTTP status {s}"), @@ -53,6 +58,24 @@ pub struct WireFailure { /// The cause [`WireFailure::container_stopped`] reports. pub(crate) const STOPPED: &str = "exact stopped container observation"; impl WireFailure { + /// A non-HTTP cause before an attempt, with no invented transport facts. + pub fn local(cause: RuntimeError) -> Box { + Box::new(Self { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: None, + requested_exec: None, + error_body: None, + }) + } /// The Engine still has the exact requested container, intact, but it is /// not running: its daemon exited, so nothing it held survives. pub fn container_stopped(&self) -> bool { diff --git a/engine/crates/layerfs-sandbox/src/types/image.rs b/engine/crates/layerfs-sandbox/src/types/image.rs new file mode 100644 index 0000000000..107ce35bae --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/image.rs @@ -0,0 +1,43 @@ +//! A Sandbox image's identity: the digest a deployment names, which is also +//! the name its unpacked root is cached under. It is a value type on every +//! platform, so a record written where the image backend runs can still be +//! read where it cannot. +use super::RuntimeError; + +/// The prefix every image digest carries. +const ALGORITHM: &str = "sha256:"; + +/// A Sandbox image's identity: `sha256:<64 lowercase hex>`, the same text the +/// Docker backend's `deploy.image` holds and a docker-archive names its config +/// blob by. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Image(String); + +impl Image { + /// Parses `sha256:<64 lowercase hex>`, the only form an archive's config + /// digest is compared with. + pub fn parse(text: &str) -> Result { + let hex = text.strip_prefix(ALGORITHM).unwrap_or(""); + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(RuntimeError::Protocol( + "Sandbox image must be an exact digest, sha256:<64 lowercase hex>", + )); + } + Ok(Self(text.to_owned())) + } + + /// The whole digest text, `sha256:<64 hex>`. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The 64 hex digits without the algorithm: the name this image's + /// unpacked root is cached under. + pub fn hex(&self) -> &str { + &self.0[ALGORITHM.len()..] + } +} diff --git a/engine/crates/layerfs-sandbox/src/types/mod.rs b/engine/crates/layerfs-sandbox/src/types/mod.rs index 69827343cb..2d8c1768ae 100644 --- a/engine/crates/layerfs-sandbox/src/types/mod.rs +++ b/engine/crates/layerfs-sandbox/src/types/mod.rs @@ -2,9 +2,13 @@ mod execution; mod failure; mod identity; +mod image; +mod request; mod transfer; pub use execution::{CommandIdentity, ExecInspection}; pub(crate) use failure::STOPPED; pub use failure::{ErrorDiagnostic, RuntimeError, WireFailure}; pub use identity::{ContainerId, ExecId}; +pub use image::Image; +pub use request::ExecRequest; pub use transfer::PendingRequest; diff --git a/engine/crates/layerfs-sandbox/src/types/request.rs b/engine/crates/layerfs-sandbox/src/types/request.rs new file mode 100644 index 0000000000..e14e795a9b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/request.rs @@ -0,0 +1,48 @@ +//! Provider-independent ordinary command metadata and validation. +use crate::{CommandIdentity, RuntimeError}; +use std::fmt; +/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. +#[derive(Clone)] +pub struct ExecRequest { + pub identity: CommandIdentity, + pub arguments: Vec, + pub environment: Vec, + pub directory: String, + pub stdin: bool, +} +impl fmt::Debug for ExecRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecRequest") + .field("identity", &self.identity) + .field("arguments_count", &self.arguments.len()) + .field("environment_count", &self.environment.len()) + .field("directory", &self.directory) + .field("stdin", &self.stdin) + .finish() + } +} +impl ExecRequest { + pub(crate) fn check(&self) -> Result<(), RuntimeError> { + self.identity.check()?; + if self.arguments.is_empty() + || self.arguments[0].is_empty() + || !std::path::Path::new(&self.directory).is_absolute() + || self + .arguments + .iter() + .chain(&self.environment) + .chain(std::iter::once(&self.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if self + .environment + .iter() + .any(|v| !v.contains('=') || v.starts_with('=')) + { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + Ok(()) + } +} diff --git a/engine/crates/layerfs-sandbox/tests/docker_provider.rs b/engine/crates/layerfs-sandbox/tests/docker_provider.rs new file mode 100644 index 0000000000..8022f295d6 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/docker_provider.rs @@ -0,0 +1,394 @@ +//! Docker provider contracts exercised through bounded Unix HTTP peers, without an Engine. +#![cfg(unix)] +#[allow(dead_code)] +#[path = "support/engine.rs"] +mod peer; +use layerfs_sandbox::{ + backend::docker::{DockerProvider, ProviderFailure, Sandbox, SandboxRequest}, + provider::{ + DockerRecord, LaunchPhase, LaunchRequest, LocalRecord, ProviderLaunchOptions, SandboxRecord, + }, + CommandIdentity, ContainerId, ExecRequest, Runtime, RuntimeError, +}; +use peer::{fixed, frame, upgrade, Peer}; +use std::{ + fs, io, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; +const CID: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const IMAGE: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +static NEXT: AtomicUsize = AtomicUsize::new(0); +fn deployment() -> SandboxRequest { + SandboxRequest { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + } +} +struct Binary(PathBuf); +impl Binary { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "h22-provider-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&path, b"sealed executable").unwrap(); + Self(path) + } + fn launch(&self) -> LaunchRequest { + LaunchRequest { + name: "h22-sandbox-1".into(), + daemon_executable: self.0.clone(), + daemon_config: b"private config".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: None, + }, + } + } +} +impl Drop for Binary { + fn drop(&mut self) { + fs::remove_file(&self.0).unwrap(); + } +} +fn logs(payload: &[u8]) -> Vec { + let mut bytes = format!("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.multiplexed-stream\r\nContent-Length: {}\r\n\r\n", payload.len()).into_bytes(); + bytes.extend(payload); + bytes +} +fn launch_responses() -> Vec> { + vec![ + fixed( + 200, + br#"{"Name":"provider-store","Driver":"local","Scope":"local","Options":null}"#, + ), + fixed(201, format!(r#"{{"Id":"{CID}"}}"#).as_bytes()), + fixed(200, b""), + fixed(204, b""), + logs(&frame(1, b"LAYERFS_DAEMON_LISTEN 0.0.0.0:30421\n")), + ] +} +fn exec_request() -> ExecRequest { + ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), "printf ordinary".into()], + environment: vec![], + directory: "/workspaces/1".into(), + stdin: false, + } +} +fn exec_inspection(running: bool, exit: &str) -> Vec { + fixed(200, format!(r#"{{"ID":"{EID}","ContainerID":"{CID}","Running":{running},"ExitCode":{exit},"Pid":42}}"#).as_bytes()) +} +#[test] +fn launch_stream_log_stop_delete_and_negotiated_versions_use_the_original_runtime() { + let binary = Binary::new(); + for minor in [48, 54] { + let mut responses = vec![fixed( + 200, + format!(r#"{{"ApiVersion":"1.{minor}"}}"#).as_bytes(), + )]; + responses.extend(launch_responses()); + let mut payload = frame(1, &vec![b'x'; 1024 * 1024 + 3]); + payload.extend(frame(2, b"err")); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&payload), + exec_inspection(false, "7"), + logs(&frame(1, b"daemon log\n")), + fixed(204, b""), + fixed(204, b""), + ]); + let (docker, peer) = Peer::scripted(responses, 8192); + let provider = Runtime::Docker(DockerProvider::new(docker)); + assert_eq!(provider.capabilities().unwrap().name, "docker"); + let mut instance = provider.launch(binary.launch()).unwrap(); + assert_eq!(instance.identity().to_string(), CID); + assert_eq!(instance.scratch_directory(), "/tmp"); + assert!(instance.local_record().is_none()); + assert_eq!( + instance.record(), + Some(SandboxRecord::Docker(DockerRecord { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + })) + ); + instance.track(42, 1).unwrap(); + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let exit = instance + .run(&exec_request(), &mut stdout, &mut stderr) + .unwrap(); + assert_eq!((exit.code, exit.pid), (7, Some(42))); + assert_eq!(exit.stdout_bytes, 1024 * 1024 + 3); + assert_eq!(stdout.len() as u64, exit.stdout_bytes); + assert_eq!(stderr, b"err"); + assert_eq!(instance.log_tail(20, 2048).unwrap(), b"daemon log\n"); + instance.stop(1).unwrap(); + instance.delete().unwrap(); + assert!(instance.stop(1).is_err()); + assert!(instance.delete().is_err()); + let calls = peer.finish(); + assert_eq!(calls.len(), 12); + assert!(calls[0].starts_with(b"GET /version ")); + for call in &calls[1..] { + assert!(String::from_utf8_lossy(call) + .split(' ') + .nth(1) + .unwrap() + .starts_with(&format!("/v1.{minor}/"))); + } + assert!(String::from_utf8_lossy(&calls[2]).contains("create?name=h22-sandbox-1")); + assert!(String::from_utf8_lossy(calls.last().unwrap()).contains("force=false&v=false")); + } +} +#[test] +fn uncertain_create_retains_partial_id_and_never_adopts_or_cleans_it() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.truncate(2); + responses[1] = fixed(201, format!(r#"{{"Id":"{CID}","Id":"{CID}"}}"#).as_bytes()); + let (docker, peer) = Peer::new(responses, 1); + let failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, LaunchPhase::Create); + assert!(failure.instance.is_none()); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed original cause") + }; + let ProviderFailure::Create(cause) = *cause else { + panic!("create custody") + }; + assert_eq!(cause.transfer.observed_container.unwrap().to_string(), CID); + assert!(cause.transfer.attempted); + assert_eq!(peer.finish().len(), 2); +} +#[test] +fn upload_start_and_listener_failures_keep_acknowledged_instance_and_original_receipts() { + let binary = Binary::new(); + for (stage, expected) in [ + (2, LaunchPhase::Create), + (3, LaunchPhase::Start), + (4, LaunchPhase::Listener), + ] { + let mut responses = launch_responses(); + responses.truncate(stage + 1); + responses[stage] = fixed(409, b"refused"); + let (docker, peer) = Peer::new(responses, 8192); + let mut failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, expected); + assert_eq!(failure.instance.take().unwrap().identity().to_string(), CID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed receipt") + }; + match (*cause, stage) { + (ProviderFailure::Upload(f), 2) => { + assert!(f.work.tar_bytes > 0); + assert_eq!(f.transfer.status, Some(409)); + } + (ProviderFailure::Start(f), 3) => assert_eq!(f.status, Some(409)), + (ProviderFailure::Listener(f), 4) => assert_eq!(f.transfer.status, Some(409)), + _ => panic!("original stage lost"), + } + assert_eq!(peer.finish().len(), stage + 1); + } +} +#[test] +fn validation_and_stdin_refuse_without_an_engine_attempt() { + let binary = Binary::new(); + let (docker, peer) = Peer::new(vec![], 1); + let provider = Runtime::Docker(DockerProvider::new(docker.clone())); + let mut request = binary.launch(); + request.name = "../escape".into(); + assert_eq!( + provider.launch(request).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut bad_host = binary.launch(); + bad_host.options = ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: Some("bad host\0".into()), + }; + assert_eq!( + provider.launch(bad_host).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = binary.launch(); + wrong_options.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + let foreign = SandboxRecord::Local(LocalRecord { + name: "test".into(), + pid: 1, + start_time: 1, + boot: None, + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(CID).unwrap(); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + assert!( + !provider + .reopen_stopped(id, &foreign) + .err() + .unwrap() + .attempted + ); + let stopped = Sandbox::stopped(docker, ContainerId::parse(CID).unwrap(), deployment()); + let mut request = exec_request(); + request.stdin = true; + assert!( + !layerfs_sandbox::Sandbox::Docker(stopped) + .run(&request, &mut Vec::new(), &mut Vec::new()) + .unwrap_err() + .attempted + ); + assert!(peer.finish().is_empty()); +} +#[test] +fn eof_with_unknown_exit_preserves_the_handle_without_polling_or_replay() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"ok")), + exec_inspection(true, "null"), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecExitUnknown { + handle, + inspection, + progress, + } = *cause + else { + panic!("EOF must not invent exit") + }; + assert_eq!(progress.stdout, 2); + assert_eq!(progress.stderr, 0); + assert!(inspection.running); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(peer.finish().len(), 8); +} +#[test] +fn exec_stage_failures_keep_deciding_wire_facts_and_delivered_progress() { + let binary = Binary::new(); + for stage in 0..3 { + let mut responses = launch_responses(); + let mut exec = vec![ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"delivered")), + exec_inspection(false, "0"), + ]; + exec.truncate(stage + 1); + exec[stage] = fixed(409, b"original refusal"); + responses.extend(exec); + let (docker, peer) = Peer::new(responses, 8192); + let owner = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = owner + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.status, Some(409)); + assert!(failure.sent_bytes > 0); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("original custody") + }; + match (*cause, stage) { + (ProviderFailure::ExecCreate(f), 0) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes) + } + (ProviderFailure::ExecStart(f), 1) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes); + assert_eq!(f.created.identity().1.to_string(), EID); + } + ( + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + 2, + ) => { + assert_eq!(failure.sent_bytes, f.sent_bytes); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(progress.stdout, 9); + assert_eq!(progress.stderr, 0); + } + _ => panic!("wrong stage"), + } + assert_eq!(peer.finish().len(), 6 + stage); + } +} +struct BrokenSink; +impl io::Write for BrokenSink { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn failed_sink_retains_pending_output_and_exec_identity_without_inspection_or_cleanup() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"undelivered")), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut BrokenSink, &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecOutput { handle, failure } = *cause else { + panic!("stream owner retained") + }; + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(failure.output.progress().stdout, 0); + assert_eq!(failure.output.progress().wire_payload, 11); + assert_eq!(peer.finish().len(), 7); +} diff --git a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs index 9c59dc4c25..e3d18209d7 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs @@ -405,6 +405,9 @@ fn reopened_from(body: &str) -> Box { fn only_the_exact_intact_container_not_running_is_a_stopped_container() { let stopped = reopened_from(&exited()); assert!(stopped.container_stopped()); + assert!(stopped.attempted); + assert_eq!(stopped.status, Some(200)); + assert!(stopped.sent_bytes >= stopped.request_header_bytes && stopped.request_header_bytes > 0); assert!(!stopped.container_gone()); let other = "c".repeat(64); assert!(!reopened_from(&exited().replace(CID, &other)).container_stopped()); @@ -414,6 +417,47 @@ fn only_the_exact_intact_container_not_running_is_a_stopped_container() { assert!(!reopened_from(&image).container_stopped()); } +#[test] +fn provider_inspects_and_reopens_exact_running_and_stopped_deployments() { + use layerfs_sandbox::backend::docker::DockerProvider; + use layerfs_sandbox::provider::SandboxRecord; + let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); + let running = inspect().replace( + r#""Running":true"#, + r#""Running":true,"Pid":42,"ExitCode":0"#, + ); + let stopped = exited().replace( + r#""Running":false"#, + r#""Running":false,"Pid":0,"ExitCode":7"#, + ); + let (docker, peer) = Peer::new( + vec![ + fixed(200, running.as_bytes()), + fixed(200, running.as_bytes()), + fixed(200, stopped.as_bytes()), + fixed(204, b""), + ], + 64, + ); + let provider = DockerProvider::new(docker); + let record = SandboxRecord::from(request()); + let mut owner = provider.reopen(id, &record).unwrap(); + let facts = owner.inspect().unwrap(); + assert!(facts.running); + assert_eq!(facts.process_id, Some(42)); + assert_eq!(facts.exit_code, None); + assert_eq!(facts.scratch_bytes, None); + let facts = owner.inspect().unwrap(); + assert!(!facts.running); + assert_eq!(facts.process_id, None); + assert_eq!(facts.exit_code, Some(7)); + // A read-only observation does not authorize Delete on the running owner. + assert!(!owner.delete().unwrap_err().attempted); + let mut stopped = provider.reopen_stopped(id, &record).unwrap(); + stopped.delete().unwrap(); + assert_eq!(peer.finish().len(), 4); +} + #[test] fn a_stopped_container_is_deleted_once_and_never_forced() { let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs new file mode 100644 index 0000000000..85987cbfe0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs @@ -0,0 +1,706 @@ +//! The namespace backend's Linux lifecycle and the #512 parity checks: a +//! Sandbox's daemon runs in its own mount, PID and IPC namespaces and its own +//! cgroup, on a root that is the Sandbox image with its own writable layer, +//! commands enter them as a nonroot identity with no capabilities and +//! no-new-privs, another Sandbox's processes, paths and writes are invisible, +//! and a close leaves no process, mount, writable layer or cgroup behind. +//! +//! It needs root, `/dev/fuse`, cgroup v2 and `user.max_user_namespaces=0`; it +//! starts a stub daemon (a shell script that prints the listener line), so no +//! Store or real daemon is involved. The image is this host's own userland in +//! a docker-archive, so the stub and the tools the checks run exist in it and +//! nowhere else in the Sandbox. Without those conditions it reports why and +//! returns, and the run is not proof. +#![cfg(target_os = "linux")] +#[allow(dead_code)] +mod support { + pub mod image; +} +use layerfs_sandbox::{ + backend::namespaces::{NamespaceProvider, Sandbox}, + provider::{LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ExecRequest, Image, RuntimeError, +}; +use std::{ + collections::BTreeSet, + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use support::image::{archive, hex, Tar, DIGEST}; + +/// The port the stub daemon claims; nothing listens on it. +const PORT: u16 = 40404; + +/// The programs the checks below run: each is copied into the image from this +/// host, with the libraries `ldd` names for it. +const PROGRAMS: [&str; 12] = [ + "/bin/sh", + "/bin/ls", + "/bin/cat", + "/bin/grep", + "/bin/test", + "/bin/echo", + "/bin/sleep", + "/bin/setsid", + "/bin/mkdir", + "/bin/mount", + "/bin/unshare", + "/bin/id", +]; + +/// One test's private directories: the runtime root, the Store's directory, +/// the stub daemon, a copy of the helper beside it, and the image archive. +struct Fixture { + base: PathBuf, + root: PathBuf, + store: PathBuf, + daemon: PathBuf, + archive: PathBuf, +} + +impl Fixture { + fn new(name: &str) -> Self { + let base = + std::env::temp_dir().join(format!("layerfs-namespaces-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let root = base.join("local"); + let store = base.join("store"); + for directory in [&root, &store] { + fs::create_dir(directory).unwrap(); + fs::set_permissions(directory, fs::Permissions::from_mode(0o755)).unwrap(); + } + fs::write(store.join("sealed.sqlite"), b"stub").unwrap(); + let daemon = base.join("stub-daemon"); + fs::write( + &daemon, + format!( + "#!/bin/sh\necho LAYERFS_DAEMON_LISTEN 127.0.0.1:{PORT}\nexec /bin/sleep 300\n" + ), + ) + .unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + let helper = helper(); + fs::copy(&helper, base.join("layerfs-sandbox-helper")).unwrap(); + let archive = image(&base); + Self { + base, + root, + store, + daemon, + archive, + } + } + + fn provider(&self) -> Result { + NamespaceProvider::new(&self.root) + } + + fn image(&self) -> Image { + Image::parse(DIGEST).unwrap() + } + + fn request(&self, name: &str) -> LaunchRequest { + LaunchRequest { + name: name.to_owned(), + daemon_executable: self.daemon.clone(), + daemon_config: b"stub setup".to_vec(), + labels: vec![("layerfs.role".to_owned(), "sandbox".to_owned())], + startup_timeout: Duration::from_secs(10), + options: ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())], + store: self.store.join("sealed.sqlite"), + image: self.image(), + archive: self.archive.clone(), + }, + } + } + + /// The Sandbox directory a launch names. + fn directory(&self, name: &str) -> PathBuf { + self.root.join(name) + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + if std::env::var_os("LAYERFS_H38_KEEP").is_some() { + eprintln!("kept {}", self.base.display()); + return; + } + let _ = fs::remove_dir_all(&self.base); + } +} + +/// The helper this build produced, beside the test binary's own directory. +fn helper() -> PathBuf { + let exe = std::env::current_exe().unwrap(); + let target = exe.parent().unwrap().parent().unwrap(); + let helper = target.join("layerfs-sandbox-helper"); + assert!( + helper.is_file(), + "build the helper first: cargo build -p layerfs-sandbox --bin layerfs-sandbox-helper" + ); + helper +} + +/// One docker-archive holding this host's own userland for the programs above, +/// plus a marker file a command can read and `/var/tmp`, the one directory a +/// command may write to. +fn image(base: &Path) -> PathBuf { + let mut tar = Tar::new() + .directory("etc/", 0o755) + .file("etc/layerfs-image", 0o644, b"layerfs image\n") + .directory("var/", 0o755) + .directory("var/tmp/", 0o1777); + let mut copied = BTreeSet::new(); + for program in PROGRAMS { + let Ok(source) = std::fs::canonicalize(program) else { + continue; + }; + tar = copy(tar, &mut copied, &source, Path::new(program)); + for library in libraries(&source) { + tar = copy(tar, &mut copied, &library, &library); + } + } + assert!( + copied.contains(Path::new("/bin/sh")) && copied.contains(Path::new("/bin/ls")), + "the host has no userland to build an image from" + ); + archive(base, "image", DIGEST, vec![tar.finish()]) +} + +/// The same bytes at `path` in the image, with the source's mode. +fn copy(mut tar: Tar, copied: &mut BTreeSet, source: &Path, path: &Path) -> Tar { + if !copied.insert(path.to_path_buf()) { + return tar; + } + let Ok(body) = fs::read(source) else { + return tar; + }; + let mode = fs::metadata(source).map_or(0o755, |metadata| metadata.permissions().mode() & 0o777); + // inside the image the path is relative: the tar carries `/bin/sh` as + // `bin/sh`, and the root's own `/` is the image itself + let text = path.to_str().expect("a UTF-8 host path"); + tar = tar.file(text.trim_start_matches('/'), mode, &body); + tar +} + +/// The libraries `ldd` names for `program`: every absolute path it prints. +fn libraries(program: &Path) -> Vec { + let Ok(output) = std::process::Command::new("ldd").arg(program).output() else { + return Vec::new(); + }; + let text = String::from_utf8_lossy(&output.stdout); + let mut libraries = Vec::new(); + for line in text.lines() { + if let Some(path) = (line.split_whitespace()).find(|token| token.starts_with('/')) { + libraries.push(PathBuf::from(path)); + } + } + libraries +} + +/// One command in `sandbox`, its streams captured. +fn run(sandbox: &Sandbox, arguments: &[&str], environment: &[&str]) -> (i32, String, String) { + let directory = sandbox.tmp().display().to_string(); + let request = ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: arguments.iter().map(|value| (*value).to_owned()).collect(), + environment: environment + .iter() + .map(|value| (*value).to_owned()) + .collect(), + directory: directory.clone(), + stdin: false, + }; + let mut out = Vec::new(); + let mut err = Vec::new(); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + let text = |bytes: Vec| String::from_utf8_lossy(&bytes).into_owned(); + (exit.code, text(out), text(err)) +} + +/// Why this host cannot run the lifecycle test, if it cannot. +fn unavailable() -> Option { + if !nix::unistd::Uid::effective().is_root() { + return Some("not root".into()); + } + match fs::read_to_string("/proc/sys/user/max_user_namespaces") { + Ok(value) if value.trim() == "0" => {} + Ok(value) => return Some(format!("user.max_user_namespaces is {}", value.trim())), + Err(error) => return Some(format!("user.max_user_namespaces: {error}")), + } + if !Path::new("/dev/fuse").exists() { + return Some("no /dev/fuse".into()); + } + if !Path::new("/sys/fs/cgroup/cgroup.controllers").exists() { + return Some("no cgroup v2".into()); + } + None +} + +#[test] +fn a_sandbox_is_its_own_namespaces_and_nothing_of_it_is_left_behind() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces lifecycle: {reason}"); + return; + } + let fixture = Fixture::new("lifecycle"); + let provider = fixture.provider().unwrap(); + let mut sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + assert_eq!(sandbox.port(), PORT); + assert!(sandbox.endpoint().is_ok()); + assert!(sandbox.process().unwrap().alive()); + assert!(matches!( + sandbox.record(), + Some(SandboxRecord::Namespaces(_)) + )); + let directory = fixture.directory("sandbox-1"); + assert!(directory.is_dir()); + // its root is the image, with this Sandbox's own empty writable layer + assert!(directory.join("overlay/upper").is_dir()); + assert_eq!( + fs::read_to_string( + fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + ) + .unwrap(), + "layerfs image\n" + ); + + // identity, capabilities and no-new-privs: a command is uid 501, gid 20, + // with no effective capability and no way to gain one + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "id -u; id -g"], &[]); + assert_eq!((code, out.as_str()), (0, "501\n20\n")); + let (code, out, _err) = run( + &sandbox, + &[ + "/bin/grep", + "-E", + "^(CapEff|NoNewPrivs|CapPrm):", + "/proc/self/status", + ], + &[], + ); + assert_eq!(code, 0); + assert!(out.contains("CapEff:\t0000000000000000"), "{out}"); + assert!(out.contains("CapPrm:\t0000000000000000"), "{out}"); + assert!(out.contains("NoNewPrivs:\t1"), "{out}"); + + // user namespaces are blocked, so a command cannot copy the mount + let (code, _, _) = run(&sandbox, &["/bin/sh", "-c", "unshare -U /bin/true"], &[]); + assert_ne!(code, 0, "unshare -U succeeded"); + // and without capabilities it cannot mount either + let (code, _, _) = run( + &sandbox, + &["/bin/sh", "-c", "mount -t tmpfs tmpfs /tmp/escape"], + &[], + ); + assert_ne!(code, 0, "a command mounted a filesystem"); + + // the root is the image: its own /proc, /dev/fuse, userland and marker + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "pwd"], &[]); + assert_eq!( + (code, out.trim_end().to_owned()), + (0, sandbox.tmp().display().to_string()) + ); + let (code, out, _) = run(&sandbox, &["/bin/ls", "/dev"], &[]); + assert_eq!(code, 0); + assert!(out.lines().any(|line| line == "fuse"), "{out}"); + let (code, out, _) = run(&sandbox, &["/bin/cat", "/proc/1/comm"], &[]); + assert_eq!(code, 0); + assert!(!out.trim().is_empty()); + let (code, out, _) = run(&sandbox, &["/bin/cat", "/etc/layerfs-image"], &[]); + assert_eq!((code, out.as_str()), (0, "layerfs image\n")); + // and it is read-only: a command cannot write the image, only its own + // writable layer above it + let (code, _, _) = run( + &sandbox, + &["/bin/sh", "-c", "echo changed > /etc/layerfs-image"], + &[], + ); + assert_ne!(code, 0, "a command wrote the image"); + let (code, _, _) = run(&sandbox, &["/bin/test", "-w", "/etc"], &[]); + assert_ne!(code, 0, "a command may write the image's directory"); + let (code, _, _) = run(&sandbox, &["/bin/test", "-w", "/var/tmp"], &[]); + assert_eq!(code, 0, "a command may not write its own writable layer"); + let (code, _, _) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "/bin/mkdir /var/tmp/written && echo x > /var/tmp/written/x", + ], + &[], + ); + assert_eq!(code, 0, "a command could not write its own layer"); + + // streams, environment and exit status + let (code, out, err) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "printf out; printf err >&2; printf %s \"$MARK\"; exit 7", + ], + &["MARK=value"], + ); + assert_eq!((code, out.as_str(), err.as_str()), (7, "outvalue", "err")); + + // the Store's directory is bound, the VM's own tree is not + let (code, _, _) = run( + &sandbox, + &["/bin/test", "-d", fixture.store.to_str().unwrap()], + &[], + ); + assert_eq!(code, 0); + let (code, _, _) = run(&sandbox, &["/bin/sh", "-c", "test -e /var/lib"], &[]); + assert_ne!(code, 0, "the VM's own tree is visible"); + + // close: one write ends every process, and nothing is left behind + sandbox.stop(5).unwrap(); + sandbox.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); + assert!(!fixture.root.join("sandbox-1.root").exists()); + assert!(!Path::new("/sys/fs/cgroup/layerfs") + .join(sandbox.identity().as_str()) + .exists()); + let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap(); + assert!( + !mounts.contains("sandbox-1"), + "a mount of the Sandbox remains" + ); + // the image stays: it is the cache every other Sandbox of it reuses + assert!(fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + .is_file()); +} + +#[test] +fn two_sandboxes_on_one_store_cannot_see_each_other() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces isolation: {reason}"); + return; + } + let fixture = Fixture::new("isolation"); + let provider = fixture.provider().unwrap(); + let first = Arc::new(provider.launch(fixture.request("sandbox-1")).unwrap()); + let second = Arc::new(provider.launch(fixture.request("sandbox-2")).unwrap()); + let other = fixture.directory("sandbox-2").display().to_string(); + + // the other Sandbox's directory is not in this mount namespace + let (code, _, _) = run(&first, &["/bin/test", "-e", &other], &[]); + assert_ne!(code, 0, "the other Sandbox's directory is visible"); + // and none of its processes is in this PID namespace: the other Sandbox's + // host PID does not exist here, and neither PID 1 nor PID 2 names it + let second_pid = second.process().unwrap().pid; + let (code, _, _) = run( + &first, + &["/bin/test", "-e", &format!("/proc/{second_pid}")], + &[], + ); + assert_ne!( + code, 0, + "the other Sandbox's PID is visible in this PID namespace" + ); + let (code, out, _) = run( + &first, + &[ + "/bin/sh", + "-c", + "grep -F -- \"$OTHER\" /proc/1/cmdline /proc/2/cmdline 2>/dev/null || true", + ], + &[&format!("OTHER={other}")], + ); + assert_eq!(code, 0); + assert!( + out.is_empty(), + "the other Sandbox's process is visible: {out}" + ); + // each cgroup holds only its own Sandbox + let (code, out, _) = run(&first, &["/bin/cat", "/proc/self/cgroup"], &[]); + assert_eq!(code, 0); + assert!(out.contains(first.identity().as_str()), "{out}"); + + // one image, two writable layers: what one Sandbox writes the other never + // sees, and each keeps its own + let (code, _, _) = run( + &first, + &["/bin/sh", "-c", "echo first > /var/tmp/mine"], + &[], + ); + assert_eq!(code, 0); + let (code, _, _) = run( + &second, + &["/bin/sh", "-c", "echo second > /var/tmp/mine"], + &[], + ); + assert_eq!(code, 0); + let (code, out, _) = run(&first, &["/bin/cat", "/var/tmp/mine"], &[]); + assert_eq!((code, out.as_str()), (0, "first\n")); + let (code, out, _) = run(&second, &["/bin/cat", "/var/tmp/mine"], &[]); + assert_eq!((code, out.as_str()), (0, "second\n")); + let (code, _, _) = run(&first, &["/bin/test", "-e", "/var/tmp/only-second"], &[]); + assert_ne!(code, 0); + // both still see the image itself, unchanged + let (code, out, _) = run(&first, &["/bin/cat", "/etc/layerfs-image"], &[]); + assert_eq!((code, out.as_str()), (0, "layerfs image\n")); + // and the writes live in each Sandbox's own upper layer, never in the + // shared image + let image_root = fixture.root.join("images").join(hex(DIGEST)); + assert!( + fs::read_to_string(image_root.join("var/tmp/mine")).is_err(), + "a Sandbox's write reached the shared image" + ); + assert_eq!( + fs::read_to_string( + fixture + .directory("sandbox-1") + .join("overlay/upper/var/tmp/mine") + ) + .unwrap(), + "first\n" + ); + assert_eq!( + fs::read_to_string( + fixture + .directory("sandbox-2") + .join("overlay/upper/var/tmp/mine") + ) + .unwrap(), + "second\n" + ); + + // both run at the same time, neither disturbed by the other + let handles = [first.clone(), second.clone()]; + let codes = std::thread::scope(|scope| { + let started = handles + .iter() + .map(|sandbox| { + let sandbox = sandbox.clone(); + scope.spawn(move || { + let (code, out, _) = run( + &sandbox, + &["/bin/sh", "-c", "/bin/sleep 0.2; printf done"], + &[], + ); + (code, out) + }) + }) + .collect::>(); + started + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + assert_eq!(codes, vec![(0, "done".to_owned()), (0, "done".to_owned())]); + drop(handles); + + for (mut sandbox, name) in [ + (Arc::try_unwrap(first).ok().unwrap(), "sandbox-1"), + (Arc::try_unwrap(second).ok().unwrap(), "sandbox-2"), + ] { + sandbox.stop(5).unwrap(); + sandbox.delete().unwrap(); + assert!(!fixture.directory(name).exists()); + } +} + +#[test] +fn a_record_reopens_the_exact_sandbox_and_nothing_else() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces reopen: {reason}"); + return; + } + let fixture = Fixture::new("reopen"); + let provider = fixture.provider().unwrap(); + let sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + let id = sandbox.identity(); + let record = sandbox.record().unwrap(); + // the record names the image its root is an overlay of + assert_eq!( + record.namespaces().map(|record| record.image().as_str()), + Some(DIGEST) + ); + assert_eq!( + record + .namespaces() + .map(|record| record.local().name.as_str()), + Some("sandbox-1") + ); + // what a new process reads: the record alone, with the provider's root, + // and it finds the same image under the digest the record keeps + let reopened_provider = fixture.provider().unwrap(); + let mut reopened = reopened_provider.reopen(id, &record).unwrap(); + assert_eq!(reopened.identity(), id); + assert_eq!(reopened.port(), PORT); + assert_eq!(reopened.record().unwrap(), record); + assert_eq!(reopened.image().image().as_str(), DIGEST); + let (code, out, _) = run(&reopened, &["/bin/echo", "again"], &[]); + assert_eq!((code, out.as_str()), (0, "again\n")); + assert!(!reopened.log_tail(10, 4096).unwrap().is_empty()); + + // a record whose image is gone is refused, not given another root + let mut without_image = record.clone(); + if let SandboxRecord::Namespaces(record) = &mut without_image { + record.image = Image::parse(&format!("sha256:{}", "e".repeat(64))).unwrap(); + } + let refused = reopened_provider + .reopen(id, &without_image) + .expect_err("a record without its image was reopened"); + assert!( + format!("{:?}", refused.cause).contains("cannot be found"), + "{:?}", + refused.cause + ); + + // a Docker record is refused rather than reinterpreting it + let foreign = SandboxRecord::Docker(layerfs_sandbox::provider::DockerRecord { + image: "sha256:0".to_owned(), + store_volume: "volume".to_owned(), + port: 30421, + labels: Vec::new(), + }); + assert!(reopened_provider.reopen(id, &foreign).is_err()); + + // a stopped Sandbox admits only its one Delete + reopened.stop(5).unwrap(); + drop(sandbox); + let stopped = reopened_provider.reopen_stopped(id, &record).unwrap(); + assert!(stopped.endpoint().is_err()); + let mut stopped = stopped; + stopped.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); +} + +#[test] +fn a_request_is_refused_before_any_effect() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces refusals: {reason}"); + return; + } + let fixture = Fixture::new("refusals"); + let provider = fixture.provider().unwrap(); + // a root command identity + let mut request = fixture.request("sandbox-1"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 0, gid: 0 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(matches!( + failure.cause, + RuntimeError::Protocol("nonroot command uid/gid") + )); + assert!(failure.instance.is_none()); + assert!(!fixture.directory("sandbox-1").exists()); + // the other runtime's options + let mut request = fixture.request("sandbox-2"); + request.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none()); + // a Store directory that would expose every Sandbox + let mut request = fixture.request("sandbox-3"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.root.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(matches!(failure.cause, RuntimeError::Protocol(_))); + assert!(!fixture.directory("sandbox-3").exists()); + // a bundle without the image archive + let mut request = fixture.request("sandbox-4"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.base.join("no-such-image.tar"), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none(), "{:?}", failure.cause); + assert!(format!("{:?}", failure.cause).contains("no-such-image.tar")); + assert!(!fixture.directory("sandbox-4").exists()); + // an image archive that holds another image than the deployment named + let mut request = fixture.request("sandbox-5"); + let other = Image::parse(&format!("sha256:{}", "d".repeat(64))).unwrap(); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: other.clone(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none(), "{:?}", failure.cause); + let message = format!("{:?}", failure.cause); + assert!(message.contains(other.hex()), "{message}"); + assert!(message.contains(hex(DIGEST)), "{message}"); + assert!(!fixture.directory("sandbox-5").exists()); +} + +#[test] +fn a_running_command_ends_with_its_sandbox() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces teardown: {reason}"); + return; + } + let fixture = Fixture::new("teardown"); + let provider = fixture.provider().unwrap(); + let mut sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + // a command that outlives its own process group and keeps running + let (code, out, _) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "setsid /bin/sleep 300 >/dev/null 2>&1 & echo started", + ], + &[], + ); + assert_eq!((code, out.as_str()), (0, "started\n")); + sandbox.stop(5).unwrap(); + // cgroup.kill ended it: nothing of the Sandbox is left in its group + let events = fs::read_to_string( + Path::new("/sys/fs/cgroup/layerfs") + .join(sandbox.identity().as_str()) + .join("cgroup.events"), + ) + .unwrap(); + assert!(events.contains("populated 0"), "{events}"); + // closing it removes the writable layer and the directory, and the shared + // image stays for the next Sandbox of it + assert!(fixture + .directory("sandbox-1") + .join("overlay/upper") + .is_dir()); + sandbox.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); + assert!(fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + .is_file()); +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs b/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs new file mode 100644 index 0000000000..6a1af22a34 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs @@ -0,0 +1,145 @@ +//! The namespace backend's formats and records: the init stage's spec, the +//! recorded facts a reopen reads, and the refusals that keep one runtime's +//! records from another. Nothing here starts a process. +use layerfs_sandbox::{ + backend::HelperSpec, + provider::{DockerRecord, LocalRecord, NamespaceRecord, SandboxRecord}, + Image, +}; +use std::path::PathBuf; + +/// The image digest these records name. +fn image() -> Image { + Image::parse(&format!("sha256:{}", "a".repeat(64))).unwrap() +} + +fn spec() -> HelperSpec { + HelperSpec { + root: PathBuf::from("/var/lib/layerfs/local/sandbox-1.root"), + sandbox: PathBuf::from("/var/lib/layerfs/local/sandbox-1"), + store: PathBuf::from("/layerfs-store/project"), + daemon: PathBuf::from("/var/lib/layerfs/local/sandbox-1/bin/layerfs-daemon"), + config: PathBuf::from("/var/lib/layerfs/local/sandbox-1/config/daemon.setup"), + cgroup: PathBuf::from("/sys/fs/cgroup/layerfs/0000000000000000000000000000000000000000000000000000000000000001"), + lower: PathBuf::from( + "/var/lib/layerfs/local/images/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), + upper: PathBuf::from("/var/lib/layerfs/local/sandbox-1/overlay/upper"), + work: PathBuf::from("/var/lib/layerfs/local/sandbox-1/overlay/work"), + } +} + +#[test] +fn the_init_spec_roundtrips() { + let spec = spec(); + let encoded = spec.encode().unwrap(); + assert_eq!(HelperSpec::parse(&encoded).unwrap(), spec); + // one line per field, key first + let text = String::from_utf8(encoded).unwrap(); + assert_eq!(text.lines().count(), 9); + assert!(text.starts_with("root=/var/lib/layerfs/local/sandbox-1.root\n")); +} + +#[test] +fn a_value_may_hold_an_equals_sign() { + let mut spec = spec(); + spec.store = PathBuf::from("/layerfs-store/a=b"); + let encoded = spec.encode().unwrap(); + assert_eq!(HelperSpec::parse(&encoded).unwrap().store, spec.store); +} + +#[test] +fn a_path_the_init_stage_cannot_mount_is_refused() { + let relative = HelperSpec { + root: PathBuf::from("relative/root"), + ..spec() + }; + assert!(relative.encode().is_err()); + let parent = HelperSpec { + sandbox: PathBuf::from("/var/lib/../sandbox"), + ..spec() + }; + assert!(parent.encode().is_err()); + let newline = HelperSpec { + config: PathBuf::from("/var/lib/a\nb"), + ..spec() + }; + assert!(newline.encode().is_err()); + let bytes = newline.encode().unwrap_err(); + assert_eq!(format!("{bytes:?}"), r#"Protocol("helper spec path")"#); + // the three overlay paths are one mount option string: neither a comma nor + // a colon may appear in them + let comma = HelperSpec { + upper: PathBuf::from("/var/lib/a,b/upper"), + ..spec() + }; + assert!(comma.encode().is_err()); + let colon = HelperSpec { + lower: PathBuf::from("/var/lib/a:b"), + ..spec() + }; + assert_eq!( + format!("{:?}", colon.encode().unwrap_err()), + r#"Protocol("an overlay path holds a `,` or `:`")"# + ); +} + +#[test] +fn a_spec_is_refused_unless_every_field_appears_once() { + let encoded = spec().encode().unwrap(); + let text = String::from_utf8(encoded).unwrap(); + // one field missing + let missing = text + .lines() + .filter(|line| !line.starts_with("cgroup=")) + .collect::>(); + assert!(HelperSpec::parse(missing.join("\n").as_bytes()).is_err()); + // one field twice + let twice = format!("{text}cgroup=/sys/fs/cgroup/layerfs/2\n"); + assert!(HelperSpec::parse(twice.as_bytes()).is_err()); + // an unknown field + let unknown = format!("{text}kernel=nfs\n"); + assert!(HelperSpec::parse(unknown.as_bytes()).is_err()); + // a line without a key + assert!(HelperSpec::parse(format!("{text}/\n").as_bytes()).is_err()); + // the whole file is bounded + let huge = vec![b'a'; 8193]; + assert!(HelperSpec::parse(&huge).is_err()); +} + +#[test] +fn a_namespaces_record_is_its_own_variant_and_keeps_the_local_facts() { + let local = LocalRecord { + name: "layerfs-project-1".to_owned(), + pid: 4242, + start_time: 7, + boot: Some(1), + port: 41000, + labels: vec![("layerfs.role".to_owned(), "sandbox".to_owned())], + }; + let namespace = NamespaceRecord { + local: local.clone(), + image: image(), + }; + let record = SandboxRecord::Namespaces(namespace.clone()); + assert_eq!(record.namespaces(), Some(&namespace)); + // the image digest the record keeps is the one its root is an overlay of + assert_eq!( + record.namespaces().map(|record| record.image().as_str()), + Some(image().as_str()) + ); + assert!(record.local().is_none()); + assert!(record.docker().is_none()); + // the native record of the same facts stays a native record + let native = SandboxRecord::Local(local); + assert!(native.namespaces().is_none()); + // a Docker record written before this backend existed still reads as Docker + let docker = SandboxRecord::Docker(DockerRecord { + image: "sha256:0".to_owned(), + store_volume: "volume".to_owned(), + port: 30421, + labels: Vec::new(), + }); + assert!(docker.namespaces().is_none()); + assert!(docker.local().is_none()); +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs b/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs new file mode 100644 index 0000000000..a7cb03e528 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs @@ -0,0 +1,344 @@ +//! The Sandbox image: one docker-archive in the bundle, unpacked once per +//! config digest. These tests need no root, no Engine and no namespaces: they +//! unpack archives into a scratch directory and read what a command would see. +//! What they prove is the contract the lifecycle tests build on — the digest +//! check, the archive's own whiteouts, one unpack per digest, and the entries +//! a Sandbox root refuses. +#![cfg(target_os = "linux")] +#[allow(dead_code)] +mod support { + pub mod image; +} +use layerfs_sandbox::{ + backend::namespaces::rootfs::{self, ImageRoot}, + Image, +}; +use std::{ + fs, io, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; +use support::image::{archive, hex, legacy_archive, Scratch, Tar, DIGEST, OTHER}; + +/// Two layers: the first holds the image, the second hides part of it and adds +/// to it, which is what a whiteout and an opaque directory mean. +fn two_layers() -> Vec> { + let first = Tar::new() + .directory("etc/", 0o755) + .file("etc/marker", 0o644, b"layer one\n") + .file("etc/gone", 0o644, b"replaced\n") + .file("etc/kept", 0o644, b"kept\n") + .directory("etc/opaque/", 0o755) + .file("etc/opaque/old", 0o644, b"below\n") + .finish(); + let second = Tar::new() + // `.wh.` hides the entry the layer below placed at the same level + .file("etc/.wh.gone", 0o644, b"") + // `.wh..wh..opq` empties its directory, then this layer refills it + .file("etc/opaque/.wh..wh..opq", 0o644, b"") + .file("etc/opaque/new", 0o644, b"above\n") + // and an ordinary file the first layer never had + .file("etc/added", 0o644, b"added\n") + .finish(); + vec![first, second] +} + +fn image(digest: &str) -> Image { + Image::parse(digest).unwrap() +} + +/// The unpacked root of a digest, which must not exist yet. +fn unpacked(scratch: &Scratch, archive: &Path, digest: &str) -> ImageRoot { + rootfs::ensure(archive, &image(digest), &scratch.cache).unwrap() +} + +fn read(path: &Path) -> String { + fs::read_to_string(path).unwrap_or_else(|error| panic!("{}: {error}", path.display())) +} + +#[test] +fn the_layers_unpack_in_order_and_their_whiteouts_apply() { + let scratch = Scratch::new("layers"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(root.image().as_str(), DIGEST); + assert_eq!(root.directory(), scratch.cache.join(hex(DIGEST)).as_path()); + let directory = root.directory(); + // the first layer's files, and the second layer's + assert_eq!(read(&directory.join("etc/marker")), "layer one\n"); + assert_eq!(read(&directory.join("etc/kept")), "kept\n"); + assert_eq!(read(&directory.join("etc/added")), "added\n"); + // `.wh.gone` hid what the layer below put there + assert!(!directory.join("etc/gone").exists()); + // `.wh..wh..opq` emptied the directory: only the second layer's entry is left + assert_eq!(read(&directory.join("etc/opaque/new")), "above\n"); + assert!(!directory.join("etc/opaque/old").exists()); + // the cache directory is not writable by the command identity + let mode = fs::symlink_metadata(directory) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o555, "the image cache is writable"); +} + +#[test] +fn a_whiteout_with_nothing_below_it_creates_nothing() { + let scratch = Scratch::new("whiteout"); + let layers = vec![ + Tar::new().file("etc/kept", 0o644, b"kept\n").finish(), + // a whiteout under a directory no layer made: it hides nothing and + // must not become the empty directory it names + Tar::new() + .file("etc/.wh.hidden", 0o644, b"") + .file("missing/.wh..wh..opq", 0o644, b"") + .finish(), + ]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(read(&root.directory().join("etc/kept")), "kept\n"); + assert!(!root.directory().join("etc/hidden").exists()); + assert!(!root.directory().join("missing").exists()); +} + +#[test] +fn a_digest_the_archive_is_not_refuses_the_start_and_unpacks_nothing() { + let scratch = Scratch::new("digest"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let error = rootfs::ensure(&path, &image(OTHER), &scratch.cache).unwrap_err(); + let message = format!("{error}"); + assert!(message.contains(hex(DIGEST)), "{message}"); + assert!(message.contains(hex(OTHER)), "{message}"); + assert!(message.contains("config digest"), "{message}"); + // nothing was published, and no half unpack was left behind + assert!(!scratch.cache.join(hex(OTHER)).exists()); + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn an_older_writers_config_name_is_the_same_digest() { + let scratch = Scratch::new("legacy"); + let path = legacy_archive(&scratch.base, "image", DIGEST, two_layers()); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(read(&root.directory().join("etc/marker")), "layer one\n"); +} + +#[test] +fn one_unpack_per_digest_however_many_sandboxes_ask_for_it() { + let scratch = Scratch::new("once"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let first = unpacked(&scratch, &path, DIGEST); + let published = fs::metadata(first.directory()).unwrap(); + // a second Sandbox of the same image finds the finished directory: the + // same inode, and no new unpack + let second = unpacked(&scratch, &path, DIGEST); + assert_eq!(second.directory(), first.directory()); + let observed = fs::metadata(second.directory()).unwrap(); + assert_eq!( + (observed.ino(), observed.mtime()), + (published.ino(), published.mtime()), + "the image was unpacked again" + ); + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn a_second_sandbox_waits_for_the_unpack_that_is_running() { + let scratch = Scratch::new("lock"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + // holding the digest's lock is what an unpack in progress looks like: a + // second one neither starts nor publishes anything until it is released. + // The test takes the lock itself rather than publishing an unpack and + // deleting it: a published image is 0555, and unlinking inside a read-only + // directory is root's privilege, while these tests must also run as the + // nonroot CI user. Publishing and deleting are not what this proves. + let cache = scratch.cache.clone(); + fs::create_dir_all(&cache).unwrap(); + let held = nix::fcntl::Flock::lock( + fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(cache.join(format!("{}.lock", hex(DIGEST)))) + .unwrap(), + nix::fcntl::FlockArg::LockExclusive, + ) + .unwrap(); + let directory = cache.join(hex(DIGEST)); + let waiting = { + let (archive, cache) = (path.clone(), scratch.cache.clone()); + std::thread::spawn(move || { + rootfs::ensure(&archive, &image(DIGEST), &cache).map(|root| root.directory().to_owned()) + }) + }; + let deadline = Instant::now() + Duration::from_millis(300); + while Instant::now() < deadline { + assert!( + !directory.exists(), + "a second unpack published while the first held the lock" + ); + std::thread::sleep(Duration::from_millis(20)); + } + drop(held); + let waited = waiting.join().expect("the waiting unpack").unwrap(); + assert_eq!(waited, directory); + assert_eq!(read(&directory.join("etc/added")), "added\n"); +} + +#[test] +fn a_path_that_is_not_plain_and_relative_is_refused() { + let scratch = Scratch::new("escape"); + for (name, layer) in [ + ( + "parent", + Tar::new().file("../outside", 0o644, b"x").finish(), + ), + ( + "absolute", + Tar::new().file("/etc/outside", 0o644, b"x").finish(), + ), + ( + "nested", + Tar::new() + .directory("a/", 0o755) + .file("a/../../outside", 0o644, b"x") + .finish(), + ), + ] { + let path = archive(&scratch.base, name, DIGEST, vec![layer]); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("absolute or holds a `..` component"), + "{name}: {error}" + ); + assert!(!scratch.cache.join(hex(DIGEST)).exists(), "{name}"); + assert!( + !scratch.base.join("outside").exists(), + "{name} wrote outside" + ); + assert!(!Path::new("/etc/outside").exists(), "{name} wrote outside"); + } + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn a_symlink_cannot_redirect_a_later_layers_write() { + let scratch = Scratch::new("symlink"); + let layers = vec![ + Tar::new() + .directory("etc/", 0o755) + .symlink("etc/escape", "/tmp") + .finish(), + // the parent of this entry is a symbolic link, so writing it would + // leave the root entirely + Tar::new().file("etc/escape/outside", 0o644, b"x").finish(), + ]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("passes through a symbolic link"), + "{error}" + ); + assert!(!Path::new("/tmp/outside").exists(), "the link was followed"); + assert!(!scratch.cache.join(hex(DIGEST)).exists()); +} + +#[test] +fn a_device_node_is_refused() { + let scratch = Scratch::new("device"); + let layers = vec![Tar::new() + .directory("dev/", 0o755) + .raw("dev/sda", b'3', 0o660, &[]) + .finish()]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("device node"), + "a device node was extracted: {error}" + ); + assert!(!scratch.cache.join(hex(DIGEST)).exists()); +} + +#[test] +fn a_compressed_layer_is_refused_with_the_reason() { + let scratch = Scratch::new("gzip"); + let mut gzip = vec![0x1f, 0x8b]; + gzip.extend_from_slice(&[0u8; 64]); + let path = archive(&scratch.base, "image", DIGEST, vec![gzip]); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + let message = format!("{error}"); + assert!(message.contains("gzip-compressed"), "{message}"); + assert!(message.contains("docker save"), "{message}"); +} + +#[test] +fn an_archive_without_its_manifest_or_a_layer_is_refused() { + let scratch = Scratch::new("missing"); + let without_manifest = Tar::new() + .file("1/layer.tar", 0o644, &Tar::new().finish()) + .finish(); + let path = scratch.base.join("no-manifest.tar"); + fs::write(&path, without_manifest).unwrap(); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("manifest.json"), "{error}"); + // a manifest naming a layer the archive does not hold + let mut tar = Tar::new().file( + "manifest.json", + 0o644, + format!( + "[{{\"Config\":\"blobs/sha256/{}\",\"Layers\":[\"7/layer.tar\"]}}]", + hex(DIGEST) + ) + .as_bytes(), + ); + tar = tar.file(&format!("blobs/sha256/{}", hex(DIGEST)), 0o644, b"{}"); + let path = scratch.base.join("no-layer.tar"); + fs::write(&path, tar.finish()).unwrap(); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("7/layer.tar"), "{error}"); +} + +#[test] +fn two_digests_share_one_cache_directory() { + let scratch = Scratch::new("digests"); + let first = archive(&scratch.base, "first", DIGEST, two_layers()); + let second = archive( + &scratch.base, + "second", + OTHER, + vec![Tar::new().file("other", 0o644, b"other\n").finish()], + ); + let one = unpacked(&scratch, &first, DIGEST); + let two = unpacked(&scratch, &second, OTHER); + assert_ne!(one.directory(), two.directory()); + assert_eq!(read(&one.directory().join("etc/marker")), "layer one\n"); + assert_eq!(read(&two.directory().join("other")), "other\n"); + // a record's image is found without unpacking anything again + assert_eq!( + rootfs::lookup(&image(OTHER), &scratch.cache) + .unwrap() + .directory(), + two.directory() + ); + // and one that is not there is refused rather than invented + let missing = Image::parse(&format!("sha256:{}", "d".repeat(64))).unwrap(); + let error = rootfs::lookup(&missing, &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("cannot be found"), "{error}"); +} + +/// The temporary directories an unpack in progress leaves, if any. +fn temporaries(cache: &Path) -> Vec { + match fs::read_dir(cache) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Vec::new(), + Err(error) => panic!("{}: {error}", cache.display()), + Ok(entries) => (entries.filter_map(Result::ok)) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(".tmp-")) + }) + .collect(), + } +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs b/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs new file mode 100644 index 0000000000..fcfa2b0347 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs @@ -0,0 +1,59 @@ +//! The user-namespace block is a precondition, not a setting: the namespace +//! runtime refuses to start while a command could create a user namespace and +//! hold its own copy of the Workspace mount. +//! +//! This test opens the host limit for its own duration and restores it, so it +//! runs in its own binary (no Sandbox runs beside it) and only as root in the +//! VM or CI host that owns this kernel. +#![cfg(target_os = "linux")] +use layerfs_sandbox::{backend::namespaces::NamespaceProvider, RuntimeError}; +use std::{fs, os::unix::fs::PermissionsExt, path::Path, path::PathBuf}; + +/// The kernel's limit on user namespaces. +const LIMIT: &str = "/proc/sys/user/max_user_namespaces"; + +/// Restores the limit however the test ends. +struct Restore(String); + +impl Drop for Restore { + fn drop(&mut self) { + let _ = fs::write(LIMIT, &self.0); + } +} + +fn root(name: &str) -> PathBuf { + let root = + std::env::temp_dir().join(format!("layerfs-namespaces-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + root +} + +#[test] +fn an_open_user_namespace_limit_is_refused_before_any_effect() { + if !nix::unistd::Uid::effective().is_root() { + eprintln!("SKIPPED namespaces user-namespace refusal: not root"); + return; + } + let previous = fs::read_to_string(LIMIT).unwrap().trim().to_owned(); + let restore = Restore(previous.clone()); + let root = root("userns"); + // open the limit: a Sandbox must not start, and nothing may be made + fs::write(LIMIT, "1\n").unwrap(); + let error = NamespaceProvider::new(&root).unwrap_err(); + assert!( + matches!(&error, RuntimeError::Unsupported(message) + if message.contains("user namespaces are not blocked")), + "{error}" + ); + assert_eq!(fs::read_dir(&root).unwrap().count(), 0); + // block it again: the same root is accepted, given the other host facts + fs::write(LIMIT, &previous).unwrap(); + if previous == "0" && Path::new("/dev/fuse").exists() { + assert!(NamespaceProvider::new(&root).is_ok()); + } + drop(restore); + assert_eq!(fs::read_to_string(LIMIT).unwrap().trim(), previous); + let _ = fs::remove_dir_all(&root); +} diff --git a/engine/crates/layerfs-sandbox/tests/native_platform.rs b/engine/crates/layerfs-sandbox/tests/native_platform.rs new file mode 100644 index 0000000000..46ffae1258 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_platform.rs @@ -0,0 +1,71 @@ +//! Platform refusal precedes filesystem access; a runtime never selects another backend. +#![cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ContainerId, Runtime, RuntimeError, +}; +use std::time::Duration; + +#[test] +fn wrong_platform_is_refused_before_root_access() { + #[cfg(target_os = "macos")] + let result = layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider", + ); + #[cfg(target_os = "linux")] + let result = layerfs_sandbox::backend::native::NativeMacosProvider::new( + "/does-not-exist/native-provider", + ); + assert!(matches!(result, Err(RuntimeError::Unsupported(_)))); +} + +#[test] +fn native_linux_requires_root_before_root_access() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(matches!( + layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider" + ), + Err(RuntimeError::Unsupported(_)) + )); + } +} + +/// Windows has no host runtime: the placeholder variant refuses every +/// operation before any effect, and never falls back to Docker or native. +#[test] +fn windows_is_refused_before_any_effect() { + let runtime = Runtime::Windows; + assert!(matches!( + runtime.capabilities(), + Err(RuntimeError::Unsupported(_)) + )); + assert!(matches!( + runtime.local_directory("one"), + Err(RuntimeError::Unsupported(_)) + )); + let request = LaunchRequest { + name: "one".into(), + daemon_executable: "/does-not-exist/daemon".into(), + daemon_config: b"setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }, + }; + let failure = runtime.launch(request).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Validate); + assert!(failure.instance.is_none()); + assert!(matches!(failure.cause, RuntimeError::Unsupported(_))); + let record = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-store".into(), + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(&"b".repeat(64)).unwrap(); + assert!(runtime.reopen(id, &record).is_err()); + assert!(runtime.reopen_stopped(id, &record).is_err()); +} diff --git a/engine/crates/layerfs-sandbox/tests/runtime_enum.rs b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs new file mode 100644 index 0000000000..1f7c1ba78f --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs @@ -0,0 +1,15 @@ +//! The Runtime enum's Docker selection: no local directory, container capabilities. +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::Runtime; +use std::time::Duration; + +/// Selecting Docker names an Engine endpoint only; nothing is dialed here. +#[test] +fn docker_reports_container_capabilities_and_owns_no_local_directory() { + let docker = Docker::new("/does-not-exist/docker.sock", Duration::from_secs(1)).unwrap(); + let runtime = Runtime::Docker(DockerProvider::new(docker)); + let capabilities = runtime.capabilities().unwrap(); + assert_eq!(capabilities.name, "docker"); + assert!(capabilities.requires_image); + assert!(runtime.local_directory("anything").unwrap().is_none()); +} diff --git a/engine/crates/layerfs-sandbox/tests/support/image.rs b/engine/crates/layerfs-sandbox/tests/support/image.rs new file mode 100644 index 0000000000..48dba21eea --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/support/image.rs @@ -0,0 +1,150 @@ +//! A docker-archive as a bundle would carry it, built in the test: the format +//! `docker save` writes (`manifest.json`, a config blob named by the image's +//! digest, uncompressed layer tars). Written by hand so a test needs no +//! Engine, no Docker and no root. +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// The digest the fixtures' images use unless a test needs another. +pub const DIGEST: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +/// A second digest, for one cache that holds two images. +pub const OTHER: &str = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + +/// The 64 hex digits of a digest. +pub fn hex(digest: &str) -> &str { + digest.strip_prefix("sha256:").expect("a sha256 digest") +} + +/// One tar archive, built entry by entry. +pub struct Tar { + bytes: Vec, +} + +impl Tar { + pub fn new() -> Self { + Self { bytes: Vec::new() } + } + + /// A regular file with `mode` and `body`. + pub fn file(self, path: &str, mode: u32, body: &[u8]) -> Self { + self.entry(path, mode, b'0', body, "") + } + + /// A directory entry; `path` ends with `/` as tar writes it. + pub fn directory(self, path: &str, mode: u32) -> Self { + self.entry(path, mode, b'5', &[], "") + } + + pub fn symlink(self, path: &str, target: &str) -> Self { + self.entry(path, 0o777, b'2', &[], target) + } + + pub fn hard_link(self, path: &str, target: &str) -> Self { + self.entry(path, 0o644, b'1', &[], target) + } + + /// Any entry type the writer must be able to produce, including the ones + /// a Sandbox root refuses (a device node is type `3`). + pub fn raw(self, path: &str, kind: u8, mode: u32, body: &[u8]) -> Self { + self.entry(path, mode, kind, body, "") + } + + fn entry(mut self, path: &str, mode: u32, kind: u8, body: &[u8], link: &str) -> Self { + assert!( + path.len() < 100 && link.len() < 100, + "{path} or {link} needs a pax header" + ); + let mut header = [0u8; 512]; + header[..path.len()].copy_from_slice(path.as_bytes()); + octal(&mut header[100..108], u64::from(mode)); + octal(&mut header[108..116], 0); + octal(&mut header[116..124], 0); + octal(&mut header[124..136], body.len() as u64); + octal(&mut header[136..148], 0); + header[156] = kind; + header[157..157 + link.len()].copy_from_slice(link.as_bytes()); + header[257..263].copy_from_slice(b"ustar\0"); + header[263..265].copy_from_slice(b"00"); + header[148..156].fill(b' '); + let sum: u64 = header.iter().map(|byte| u64::from(*byte)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + self.bytes.extend_from_slice(&header); + self.bytes.extend_from_slice(body); + let padding = (512 - body.len() % 512) % 512; + self.bytes.resize(self.bytes.len() + padding, 0); + self + } + + /// The archive's bytes, ending with the two zero blocks that end a tar. + pub fn finish(mut self) -> Vec { + self.bytes.resize(self.bytes.len() + 1024, 0); + self.bytes + } +} + +fn octal(field: &mut [u8], value: u64) { + let text = format!("{value:0width$o}\0", width = field.len() - 1); + field.copy_from_slice(text.as_bytes()); +} + +/// A docker-archive at `/.tar` holding one image whose config +/// digest is `digest`, with `layers` applied in the order given. +pub fn archive(directory: &Path, name: &str, digest: &str, layers: Vec>) -> PathBuf { + write( + directory, + name, + &format!("blobs/sha256/{}", hex(digest)), + layers, + ) +} + +/// The same image as older writers name it: the config blob is `.json`. +pub fn legacy_archive(directory: &Path, name: &str, digest: &str, layers: Vec>) -> PathBuf { + write(directory, name, &format!("{}.json", hex(digest)), layers) +} + +fn write(directory: &Path, name: &str, config: &str, layers: Vec>) -> PathBuf { + let mut manifest = + format!("[{{\"Config\":\"{config}\",\"RepoTags\":[\"layerfs-test:latest\"],\"Layers\":["); + for index in 0..layers.len() { + if index > 0 { + manifest.push(','); + } + manifest.push_str(&format!("\"{index}/layer.tar\"")); + } + manifest.push_str("]}]"); + let mut tar = Tar::new() + .file("manifest.json", 0o644, manifest.as_bytes()) + .file(config, 0o644, br#"{"architecture":"amd64","os":"linux"}"#); + for (index, layer) in layers.iter().enumerate() { + tar = tar.file(&format!("{index}/layer.tar"), 0o644, layer); + } + let path = directory.join(format!("{name}.tar")); + fs::write(&path, tar.finish()).unwrap(); + path +} + +/// A scratch directory one test owns and removes when it ends. +pub struct Scratch { + pub base: PathBuf, + pub cache: PathBuf, +} + +impl Scratch { + pub fn new(name: &str) -> Self { + let base = + std::env::temp_dir().join(format!("layerfs-rootfs-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let cache = base.join("images"); + Self { base, cache } + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.base); + } +} diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index c4159c3c12..97201c5102 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -17,12 +17,14 @@ # Crates whose `unsafe` surface is bounded. layerfs-storage keeps one audited # FFI module (encoding/codec.rs); Persistence has the owner-authorized macOS -# seal file-control module. Their siblings stay unsafe-free. The +# seal file-control module; Sandbox reads macOS process facts through +# proc_pidinfo in backend/macos/darwin.rs. Their siblings stay unsafe-free. The # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-sandbox": "src/backend/macos/darwin.rs", } UNSAFE_FREE_CRATES = ( "layerfs-content", @@ -32,10 +34,10 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", - "layerfs-sandbox", ) UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", @@ -47,10 +49,11 @@ "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", + "layerfs-nfs": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", - "layerfs-sandbox": "#![forbid(unsafe_code)]", + "layerfs-sandbox": "#![deny(unsafe_code)]", } @@ -84,6 +87,8 @@ "layerfs-overlay": set(), "layerfs-workspace": {"layerfs-content", "layerfs-overlay", "layerfs-telemetry"}, "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, + # The NFS export reuses the kernel ports and operations; never daemon. + "layerfs-nfs": {"layerfs-content", "layerfs-overlay", "layerfs-workspace", "layerfs-fuse"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. "layerfs-store-service": { @@ -103,6 +108,7 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-bridge", "layerfs-content", "layerfs-storage", @@ -135,6 +141,7 @@ "layerfs-bridge": ENGINE, "layerfs-daemon": ENGINE, "layerfs-fuse": ENGINE, + "layerfs-nfs": ENGINE, "layerfs-overlay": ENGINE, "layerfs-persistence": ENGINE, "layerfs-project": ENGINE, @@ -196,6 +203,7 @@ def test_path_violations(path, source, packages): "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", diff --git a/tools/test_check_product_boundary.py b/tools/test_check_product_boundary.py index ec4196fbff..55f600b5f4 100644 --- a/tools/test_check_product_boundary.py +++ b/tools/test_check_product_boundary.py @@ -126,7 +126,15 @@ def test_sdk_source_edges_and_unsafe_boundary(self): def test_sandbox_active_edges_and_unsafe_boundary(self): path = Path("engine/crates/layerfs-sandbox/src/backend/docker/http.rs") self.assertTrue(unsafe_violations(path, "unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![deny(unsafe_code)]")) + self.assertTrue(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + darwin = Path("engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs") + self.assertFalse(unsafe_violations(darwin, "fn f() { unsafe { libc::proc_listallpids(); } }")) + self.assertTrue( + unsafe_violations( + Path("engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs"), "fn f() { unsafe { call(); } }" + ) + ) prefix = '[package]\nname="layerfs-sandbox"\n[dependencies]\n' for dependency in ( "layerfs-api-core", diff --git a/tools/worktree_build.py b/tools/worktree_build.py index 3af7c4bd28..cdabf8d1e8 100644 --- a/tools/worktree_build.py +++ b/tools/worktree_build.py @@ -219,6 +219,9 @@ def build(args): "layerfs-daemon", "-p", "layerfs-node", + # the namespaces Sandbox backend runs this beside the daemon (#515 §4.5) + "-p", + "layerfs-sandbox", "-j", str(args.jobs), ] @@ -230,8 +233,9 @@ def build(args): execute(docker_build(record, command), record["worktree"], log=receipt / "output.log") script = ( 'set -eu; cp "/work/target/$1/layerfs-daemon" /export/layerfs-daemon; ' + 'cp "/work/target/$1/layerfs-sandbox-helper" /export/layerfs-sandbox-helper; ' 'cp "/work/target/$1/liblayerfs_node.so" /export/layerfs.node; ' - 'chown "$2:$3" /export/layerfs-daemon /export/layerfs.node' + 'chown "$2:$3" /export/layerfs-daemon /export/layerfs-sandbox-helper /export/layerfs.node' ) export = docker_build( record,