diff --git a/AGENTS.md b/AGENTS.md index 7ec8a92359..80c78a7412 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,6 +164,26 @@ remains a separate MEMORY/OFF/EXCLUSIVE database file. prefixed names, such as dsh's `DSH_*`, maps them itself. Linux still makes Store calls in the caller's process, and the worker's rules above (loopback, secret, one attempt, no replay) are unchanged. +- **VM placement (`@layerfs/runtime`, owner 2026-10-31, #515).** A LayerFS + home may be placed in Linux VMs on this machine instead of in the host's + Docker: `LAYERFS_PLACEMENT=vm:` (or the runtime's `placement` + setting), with `local` the default and unchanged. **One Linux VM per Store** + (V1): a project's VM holds that project's Store on its own ext4 disk, its + Sandboxes are Linux namespaces inside it + (`LAYERFS_SANDBOX_BACKEND=namespaces`, documented in + [the agent guide](api/docs/agent.md#namespace-sandboxes-linux)), and its + worker runs the real SDK as root with its own secret and its own + loopback-published port; the host reaches only each VM's worker, with one + attempt per call and no replay, the same rules as the macOS and Windows + routes. Store calls therefore leave the caller's process on Linux too, under + `vm:` placement only, and a process routes them per project + (`useWorker((project) => endpoint)`, or that project's + `LAYERFS_WORKER_ENDPOINT`). `project.init` makes the project's VM and + `project.remove` deletes it with its Store disk; a bare VM delete is refused + while the project exists, because that Store is on that disk only and no + remote Store push exists (#506). Implemented and verified providers: + `microsandbox` (`msb`) and `apple-container` (`container` 1.6.1). + `@layerfs/runtime`'s guide and `api/runtime/src/platform/vm/` hold the details. - **One global database, one writer, short holds (owner decision 2026-10-18).** All committed state lives in one SQLite Store file; no sharding, second database, concurrent-writer engine or objects outside it. Writes go one at a diff --git a/Cargo.lock b/Cargo.lock index ca39be0e98..a9221a273c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -590,6 +590,7 @@ dependencies = [ "layerfs-content", "layerfs-fuse", "layerfs-history", + "layerfs-nfs", "layerfs-overlay", "layerfs-persistence", "layerfs-project", @@ -620,6 +621,23 @@ dependencies = [ "layerfs-content", ] +[[package]] +name = "layerfs-nfs" +version = "1.1.0" +dependencies = [ + "layerfs-bridge", + "layerfs-content", + "layerfs-daemon", + "layerfs-fuse", + "layerfs-history", + "layerfs-overlay", + "layerfs-persistence", + "layerfs-project", + "layerfs-storage", + "layerfs-telemetry", + "layerfs-workspace", +] + [[package]] name = "layerfs-node" version = "1.1.0" @@ -633,6 +651,7 @@ dependencies = [ "layerfs-storage", "layerfs-store-service", "layerfs-telemetry", + "libc", "napi", "napi-build", "napi-derive", diff --git a/api/agent/src/client.js b/api/agent/src/client.js index 7e1ae74a98..8455d1cb28 100644 --- a/api/agent/src/client.js +++ b/api/agent/src/client.js @@ -17,8 +17,9 @@ export const BACKEND = /^(backend|runtime)\./; /** Runs one operation once and returns its value or throws its refusal; * `options` are the client's settings, whose `trace` selects its trace file; - * `transport` is the client's binding (default: the one a new client would use). */ -export async function invoke(op, args, options, transport = binding()) { + * `transport` is the client's binding (default: the one a new client of + * `args.project` would use). */ +export async function invoke(op, args, options, transport = binding(args.project)) { return outcome(await traced(options, op, args, (scope) => { if (BACKEND.test(op)) throw new LayerFSError("UnknownMethod", "backend and runtime methods are @layerfs/backend's and @layerfs/runtime's", null); return transport.call(op, observed(args, scope)); @@ -26,7 +27,7 @@ export async function invoke(op, args, options, transport = binding()) { } /** One call on the binding's backend table (`backend.*`, `runtime.*`). */ -export async function invokeBackend(op, args, options, transport = binding()) { +export async function invokeBackend(op, args, options, transport = binding(args?.project)) { return outcome(await traced(options, op, args, (scope) => transport.backendCall(op, observed(args, scope)))); } @@ -60,9 +61,19 @@ export function sandboxArg(sandbox) { /** Deployment settings from the client options, else the environment * (LAYERFS_). A Workspace's Sandbox needs - * every one but the container prefix and daemon host. */ + * every one but the container prefix and daemon host. With `sandbox` + * (LAYERFS_SANDBOX) `native` (Linux or macOS), a Sandbox is a host daemon: + * Linux uses FUSE with a root daemon and nonroot commands (501:20); + * macOS uses loopback NFS and this user. It needs the daemon and + * `localDir` (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is + * Docker; any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || undefined; + const sandbox = options.sandbox ?? env("SANDBOX"); + if (sandbox === "native") return nativeDeployment(options, env, sandbox); + if (sandbox !== undefined && sandbox !== "docker") { + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not ${sandbox}`, null); + } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", image: options.image ?? env("IMAGE"), @@ -80,6 +91,35 @@ export function deployment(options = {}) { return deploy; } +/** A native Sandbox's settings: the daemon, the directory its Sandboxes + * live in, the optional name prefix, and the backend inside that directory + * (`sandboxBackend`, LAYERFS_SANDBOX_BACKEND: `native` when unset, or + * `namespaces` on Linux). The namespaces backend gives each Sandbox a root + * filesystem from an image, so it also needs `image` (LAYERFS_IMAGE, the + * same digest Docker deploys) and `imageArchive` (LAYERFS_IMAGE_ARCHIVE, the + * docker-archive the bundle carries). */ +function nativeDeployment(options, env, sandbox) { + const deploy = { + sandbox, + daemon: options.daemon ?? env("DAEMON"), + localDir: options.localDir ?? env("LOCAL_DIR"), + containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), + }; + const backend = options.sandboxBackend ?? env("SANDBOX_BACKEND"); + if (backend !== undefined) deploy.sandboxBackend = backend; + const required = [["daemon", "DAEMON"], ["localDir", "LOCAL_DIR"]]; + if (backend === "namespaces") { + deploy.image = options.image ?? env("IMAGE"); + deploy.imageArchive = options.imageArchive ?? env("IMAGE_ARCHIVE"); + required.push(["image", "IMAGE"], ["imageArchive", "IMAGE_ARCHIVE"]); + } + for (const [name, setting] of required) { + if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); + } + if (!deploy.containerPrefix) delete deploy.containerPrefix; + return deploy; +} + /** Refuses every call on a closed client (or one no factory opened). */ export function live(client) { if (client?.[STATE]?.closed !== false) { diff --git a/api/agent/src/index.js b/api/agent/src/index.js index 446b59a534..1dc96b3c7e 100644 --- a/api/agent/src/index.js +++ b/api/agent/src/index.js @@ -100,7 +100,15 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON; a * Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. + * container it starts `-`. On Linux or macOS, + * LAYERFS_SANDBOX=native starts each Sandbox as a host daemon in a directory + * under LAYERFS_LOCAL_DIR. Linux requires root for FUSE and runs commands + * as 501:20; macOS uses loopback NFS and the caller's identity. No Docker + * setting is read; native provides no isolation from the host. + * LAYERFS_SANDBOX_BACKEND picks the backend inside that directory: `native` + * (unset) or, on Linux, `namespaces` (namespaces, cgroups, nonroot commands). + * It is read for every call, so a Sandbox made with one backend is never + * reopened as another. * LAYERFS_DAEMON_HOST is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on @@ -129,13 +137,18 @@ export class LayerFS { volume: env("VOLUME"), daemon: env("DAEMON"), ...(env("CONTAINER_PREFIX") ? { containerPrefix: env("CONTAINER_PREFIX") } : {}), + ...(env("SANDBOX") ? { sandbox: env("SANDBOX") } : {}), + ...(env("SANDBOX_BACKEND") ? { sandboxBackend: env("SANDBOX_BACKEND") } : {}), + ...(env("LOCAL_DIR") ? { localDir: env("LOCAL_DIR") } : {}), + ...(env("IMAGE_ARCHIVE") ? { imageArchive: env("IMAGE_ARCHIVE") } : {}), ...(env("DAEMON_HOST") ? { daemonHost: env("DAEMON_HOST") } : {}), }, storePolicy: env("STORE_POLICY"), storePath: env("STORE_PATH"), }; - // not open until a factory has opened its Store; the binding is chosen now, once - this[STATE] = { closed: null, transport: binding() }; + // not open until a factory has opened its Store; the binding is chosen now, once, + // for this client's project (a VM placement has one worker per project) + this[STATE] = { closed: null, transport: binding(this.projectName) }; this.fs = bind(this, fsMethods); this.project = bind(this, { list_branches, list_sandboxes }); this.branch = bind(this, { list_commits, diff_commits, diff_branches, fork, merge, publish }); diff --git a/api/agent/src/transport/index.js b/api/agent/src/transport/index.js index 68d9e32760..5301914202 100644 --- a/api/agent/src/transport/index.js +++ b/api/agent/src/transport/index.js @@ -8,13 +8,14 @@ // // The worker route is open to any integration that names an endpoint // (AGENTS.md, owner 2026-10-24): @layerfs/runtime starts the worker on macOS -// and Windows, where a process cannot open the Store itself. LayerFS names no -// integration here. +// and Windows, where a process cannot open the Store itself, and in a +// `vm:` home, where each project's Store is in that project's VM +// (one endpoint per project). LayerFS names no integration here. import { LayerFSError } from "../errors.js"; import { inprocess } from "./inprocess.js"; import { workerClient } from "./worker_client.js"; -/** The worker `useWorker` handed over: { endpoint }, else null. */ +/** The worker `useWorker` handed over: { endpoint }, { resolve } or null. */ let handed = null; let local = null; /** Worker clients by endpoint file. */ @@ -24,18 +25,28 @@ const workers = new Map(); export const setting = (name) => process.env[`LAYERFS_${name}`] || undefined; /** Internal, for @layerfs/runtime: clients made from now on in this process - * use the worker at `endpoint` (an endpoint file). Takes precedence over the - * environment. One worker per process: another endpoint is refused. */ + * use the worker at `endpoint` (an endpoint file), or, when `endpoint` is a + * function, the worker that function names for a client's project — one VM + * per Store has one endpoint each. Takes precedence over the environment. + * One worker per process: another endpoint is refused, and so is a second + * resolver. */ export function useWorker(endpoint) { - if (typeof endpoint !== "string" || !endpoint) throw new LayerFSError("InvalidInput", "a worker endpoint file is required", null); - if (handed && handed.endpoint !== endpoint) throw new LayerFSError("InvalidInput", `this process already uses the worker at ${handed.endpoint}`, null); - handed = { endpoint }; + const next = typeof endpoint === "function" ? { resolve: endpoint } + : typeof endpoint === "string" && endpoint ? { endpoint } + : null; + if (!next) throw new LayerFSError("InvalidInput", "a worker endpoint file is required", null); + if (handed && (handed.endpoint !== next.endpoint || handed.resolve !== next.resolve)) { + throw new LayerFSError("InvalidInput", `this process already uses the worker at ${handed.endpoint ?? "one endpoint per project"}`, null); + } + handed = next; } -/** The binding a client made now uses: the handed-over worker, else - * LAYERFS_WORKER_ENDPOINT's, else the addon in this process. */ -export function binding() { - const endpoint = handed?.endpoint ?? setting("WORKER_ENDPOINT"); +/** The binding a client made now uses: the handed-over worker (by project, or + * the one endpoint), else LAYERFS_WORKER_ENDPOINT's, else the addon in this + * process. A call without a project (an in-process backend call) uses the + * environment's endpoint, never a project's resolver. */ +export function binding(project) { + const endpoint = handed?.resolve && typeof project === "string" ? handed.resolve(project) : handed?.endpoint ?? setting("WORKER_ENDPOINT"); if (!endpoint) return (local ??= inprocess()); let client = workers.get(endpoint); if (!client) workers.set(endpoint, (client = workerClient(endpoint))); diff --git a/api/agent/test/native_deployment.mjs b/api/agent/test/native_deployment.mjs new file mode 100644 index 0000000000..4b0d951e30 --- /dev/null +++ b/api/agent/test/native_deployment.mjs @@ -0,0 +1,143 @@ +// Which runtime a Sandbox uses. No real addon, Store or Docker (the +// stand-in binding answers the client's calls): +// node api/agent/test/native_deployment.mjs +// LAYERFS_SANDBOX unset or `docker` gives exactly the Docker deployment it +// always gave; `native` (macOS) needs only the daemon and LAYERFS_LOCAL_DIR +// (no Docker setting); any other value, the former `local` included, is +// refused rather than taken as either. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { deployment } from "../src/client.js"; + +const FAKE = fileURLToPath(new URL("./fixture/fake-native.cjs", import.meta.url)); +const native = createRequire(import.meta.url)(FAKE); + +const clear = () => { + for (const name of Object.keys(process.env)) { + if (name.startsWith("LAYERFS_") || name.startsWith("DSH_LAYERFS_")) delete process.env[name]; + } + process.env.LAYERFS_NATIVE = FAKE; +}; +const docker = { docker: "/run/docker.sock", image: `sha256:${"a".repeat(64)}`, volume: "lfs-v", daemon: "/opt/layerfs-daemon" }; +const setDocker = () => { + process.env.LAYERFS_DOCKER = docker.docker; + process.env.LAYERFS_IMAGE = docker.image; + process.env.LAYERFS_VOLUME = docker.volume; + process.env.LAYERFS_DAEMON = docker.daemon; +}; +const store = () => { + process.env.LAYERFS_HOME = "/private/local-home"; + process.env.LAYERFS_STORE_PATH = "/fake/store.sqlite"; +}; + +// unset: the Docker deployment, field for field +clear(); +setDocker(); +assert.deepEqual(deployment(), docker); +// `docker` is the same deployment +process.env.LAYERFS_SANDBOX = "docker"; +assert.deepEqual(deployment(), docker); +// the Docker settings are still required there +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_IMAGE/.test(e.message)); + +// native: daemon + localDir, no Docker setting read or required +clear(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +assert.deepEqual(deployment(), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// an image (sandbox.create's option) does not change a native deployment +assert.deepEqual(deployment({ image: docker.image }), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// the backend inside the directory is native unless named; namespaces is +// passed through (the binding refuses it off Linux, and an unknown value) +assert.equal(deployment().sandboxBackend, undefined); +const archive = `/bundle/images/${"a".repeat(64)}.tar`; +process.env.LAYERFS_SANDBOX_BACKEND = "namespaces"; +process.env.LAYERFS_IMAGE = docker.image; +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// a namespaces Sandbox's root is an image: the same digest Docker deploys +// and the docker-archive the bundle carries +assert.deepEqual(deployment(), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "namespaces", + image: docker.image, + imageArchive: archive, +}); +// each of those is named when it is missing +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE\)/.test(e.message)); +process.env.LAYERFS_IMAGE = docker.image; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE_ARCHIVE\)/.test(e.message)); +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// the native backend inside the same directory reads neither of them +assert.deepEqual(deployment({ sandboxBackend: "native" }), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "native", +}); +delete process.env.LAYERFS_SANDBOX_BACKEND; +delete process.env.LAYERFS_IMAGE; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +// the container prefix names native Sandbox directories too +process.env.LAYERFS_CONTAINER_PREFIX = "ci-"; +assert.equal(deployment().containerPrefix, "ci-"); +delete process.env.LAYERFS_CONTAINER_PREFIX; +// each required setting is named when missing +delete process.env.LAYERFS_LOCAL_DIR; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_LOCAL_DIR/.test(e.message)); +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +delete process.env.LAYERFS_DAEMON; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_DAEMON/.test(e.message)); +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +// a DSH_LAYERFS_NFS_ name is not mapped (an integration maps its own names) +process.env.DSH_LAYERFS_NFS_MOUNT = "/bin/true"; +deployment(); +assert.equal(process.env.LAYERFS_NFS_MOUNT, undefined); +// since #428 the SDK reads LAYERFS_ names only: a DSH_LAYERFS_ twin selects nothing +clear(); +setDocker(); +process.env.DSH_LAYERFS_SANDBOX = "native"; +process.env.DSH_LAYERFS_LOCAL_DIR = "/l"; +assert.deepEqual(deployment(), docker); + +// anything else, the former `local` included, is refused, never silently Docker +for (const typo of ["local", "Native", "nativ", "NATIVE", "podman"]) { + clear(); + setDocker(); + process.env.LAYERFS_SANDBOX = typo; + assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && e.message.includes(`not ${typo}`)); +} + +// a client reads LAYERFS_SANDBOX and LAYERFS_LOCAL_DIR once, and +// sandbox.create sends the native deployment (no Docker setting) +clear(); +store(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +const { LayerFS } = await import("../src/index.js"); +const client = await LayerFS.open("website"); +assert.equal(client.options.deploy.sandbox, "native"); +assert.equal(client.options.deploy.localDir, "/Users/me/lfs"); +const before = native.deploys.length; +await client.sandbox.create(); +assert.deepEqual(native.deploys.slice(before), [{ sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }]); +// an image given to sandbox.create does not turn it into a Docker deployment +await client.sandbox.create({ image: docker.image }); +assert.deepEqual(native.deploys.at(-1), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +await client.close(); +// unset: the client carries no new key +clear(); +store(); +setDocker(); +const plain = await LayerFS.open("website"); +assert.equal("sandbox" in plain.options.deploy, false); +assert.equal("localDir" in plain.options.deploy, false); +await plain.close(); +console.log("native_deployment: ok"); diff --git a/api/agent/test/packaging.mjs b/api/agent/test/packaging.mjs index 496a616b95..114ed0ad3d 100644 --- a/api/agent/test/packaging.mjs +++ b/api/agent/test/packaging.mjs @@ -1,7 +1,7 @@ // @layerfs/agent's surface and client lifecycle (#428). Each case is one // command; every case but `dispatch` uses the stand-in binding and creates no // Store: -// node api/agent/test/packaging.mjs surface|lifecycle|copy|sandbox|worker-route|transport-order|dispatch +// node api/agent/test/packaging.mjs surface|lifecycle|copy|sandbox|worker-route|worker-per-project|transport-order|dispatch // `dispatch` loads the real addon (LAYERFS_NATIVE or engine/crates/layerfs-node/layerfs.node). import assert from "node:assert/strict"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; @@ -173,6 +173,32 @@ if (selected === "surface") { const calls = native.calls.length; await assert.rejects(LayerFS.open("website"), { message: /nonexistent/ }); assert.equal(native.calls.length, calls, "the worker route never reaches the addon"); +} else if (selected === "worker-per-project") { + // one process, one worker per project (`vm:`: one VM per Store) + const { fakeWorker } = await import("./fixture/fake-worker.mjs"); + const { useWorker } = await import("@layerfs/agent/internal"); + const shopWorker = await fakeWorker(), blogWorker = await fakeWorker(); + const dir = mkdtempSync(join(tmpdir(), "layerfs-per-project-")); + try { + const files = { shop: join(dir, "shop.json"), blog: join(dir, "blog.json") }; + shopWorker.endpoint(files.shop, "/private/shop"); + blogWorker.endpoint(files.blog, "/private/blog"); + // a resolver instead of one endpoint: this process's route is one VM per project + useWorker((project) => files[project]); + assert.throws(() => useWorker((project) => `${project}.json`), { code: "InvalidInput", message: /already uses/ }); + assert.throws(() => useWorker(files.shop), { code: "InvalidInput", message: /already uses/ }); + const shop = await LayerFS.open("shop"), blog = await LayerFS.open("blog"); + await shop.fs.read("main", "/README.md"); + await blog.fs.read("main", "/README.md"); + await shop.close(); + await blog.close(); + assert.deepEqual(shopWorker.calls.map((c) => c.op), ["project.list_branches", "fs.read"], "shop's calls reach shop's VM"); + assert.deepEqual(blogWorker.calls.map((c) => c.op), ["project.list_branches", "fs.read"], "blog's calls reach blog's VM"); + } finally { + shopWorker.close(); + blogWorker.close(); + rmSync(dir, { recursive: true, force: true }); + } } else if (selected === "transport-order") { // each client keeps the binding it was made with, whatever the call order const { fakeWorker } = await import("./fixture/fake-worker.mjs"); diff --git a/api/agent/test/verify/native_unmount_real.mjs b/api/agent/test/verify/native_unmount_real.mjs new file mode 100644 index 0000000000..d3c6b71b6a --- /dev/null +++ b/api/agent/test/verify/native_unmount_real.mjs @@ -0,0 +1,138 @@ +// Real native SDK regression. Explicit binaries + isolated root are required. +// node api/agent/test/verify/native_unmount_real.mjs /abs/root /abs/addon.node /abs/daemon +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, writeFile, readFile, access, readdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +const [base, addon, daemon, childMode, serialized] = process.argv.slice(2); +assert.ok(base?.startsWith('/') && addon?.startsWith('/') && daemon?.startsWith('/'), 'explicit absolute fixture paths'); +const repo = fileURLToPath(new URL('../../../..', import.meta.url)); +Object.assign(process.env, { + LAYERFS_SANDBOX: 'native', LAYERFS_NATIVE: addon, LAYERFS_DAEMON: daemon, + LAYERFS_HOME: base+'/home', LAYERFS_LOCAL_DIR: base+'/local', LAYERFS_STORE_PATH: base+'/store/store.sqlite', + LAYERFS_NFS_UNMOUNT: `sh ${base}/helper.sh {dir}`, +}); +const { LayerFS } = await import(repo+'/api/agent/src/index.js'); +const { LayerFSBackend } = await import(repo+'/api/backend/src/index.js'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function deadline(promise, ms) { + let timer; + try { return await Promise.race([promise, new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error('deadline '+ms)),ms);})]); } + finally { clearTimeout(timer); } +} +async function waitFile(path) { + const end=Date.now()+5000; + while(Date.now()=0);return error.detail;} +} +function child(mode, ws) { + const p=spawn(process.execPath,[fileURLToPath(import.meta.url),base,addon,daemon,mode,JSON.stringify(ws)],{env:process.env,stdio:['ignore','pipe','pipe']}); + let output='';p.stdout.on('data',d=>output+=d);p.stderr.on('data',d=>output+=d); + const ended=new Promise(resolve=>p.once('exit',(code,signal)=>resolve({code,signal,output}))); + return {p,ended}; +} +if(childMode){ + const lf=await LayerFS.open('unmount');const ws=JSON.parse(serialized); + if(childMode==='probe')await fast('cross-process-same-workspace',()=>refused(()=>lf.workspace.unmount(ws,true))); + if(childMode==='owner')await lf.workspace.unmount(ws,true); + await lf.close(); +}else{ + assert.equal(process.platform,'darwin','this fixture uses host mount_nfs'); + await mkdir(base+'/local',{recursive:true});await mkdir(base+'/seed',{recursive:true});await mkdir(base+'/store',{recursive:true,mode:0o700}); + await writeFile(base+'/seed/file','base\n'); + await writeFile(base+'/helper.sh',`#!/bin/sh\nprintf '%s' "$1" > '${base}/started'\ni=0\nwhile [ ! -f '${base}/release' ] && [ "$i" -lt 200 ]; do sleep 0.1; i=$((i+1)); done\nif [ -f '${base}/fail' ]; then echo injected-helper-failure >&2; exit 1; fi\nexec /sbin/umount "$1"\n`); + const lf=await LayerFS.init('unmount',base+'/seed');let owner;let active; + try{ + await lf.branch.fork('a');await lf.branch.fork('b'); + const sb=await lf.sandbox.create();const a=await lf.workspace.mount(sb.sandbox,'a');const b=await lf.workspace.mount(sb.sandbox,'b'); + active=lf.workspace.unmount(a,true);active.catch(()=>{}); + await waitFile(base+'/started'); + await fast('sandbox-log-needs-discovery-lock',()=>LayerFSBackend.of(lf).daemon.sandbox.get_log(sb.sandbox,{lines:2})); + await fast('other-workspace-status',()=>LayerFSBackend.of(lf).daemon.workspace.status(b)); + await fast('other-workspace-exec',()=>lf.workspace.exec(b,'printf ready'),r=>assert.equal(r.stdout,'ready')); + await fast('same-workspace-status',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.status(a))); + await fast('same-workspace-exec',()=>refused(()=>lf.workspace.exec(a,'true'))); + await fast('same-workspace-commit',()=>refused(()=>lf.workspace.commit(a,'must not run'))); + await fast('same-workspace-list-changes',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.list_changes(a))); + await fast('repeat-unmount',()=>refused(()=>lf.workspace.unmount(a,true))); + await fast('sandbox-close',()=>refused(()=>lf.sandbox.close(sb.sandbox))); + const probe=child('probe',a);assert.equal((await deadline(probe.ended,5000)).code,0); + const another=await fast('new-workspace-mount',()=>lf.workspace.mount(sb.sandbox,'b')); + const locker=spawn('python3',['-c','import fcntl,sys; f=open(sys.argv[1],"r+"); fcntl.flock(f,fcntl.LOCK_EX); print("LOCKED",flush=True); sys.stdin.read(1)',base+'/home/unmount/sandboxes/'+sb.sandbox+'/ownership.lock'],{stdio:['pipe','pipe','pipe']}); + try { + await deadline(new Promise((resolve,reject)=>{locker.stdout.once('data',resolve);locker.once('error',reject);}),2000); + await writeFile(base+'/release','go');await sleep(200); + // The terminal result is visible before cleanup can reacquire this lock. + await fast('published-success-before-cleanup-lock',()=>lf.workspace.unmount(a,true)); + } finally { locker.stdin.end('x');await deadline(new Promise(r=>locker.once('exit',r)),2000); } + await deadline(active,5000);active=null; + await fast('repeat-completed-unmount',()=>lf.workspace.unmount(a,true)); + // Busy and Dirty are reversible: release only this attempt's intent. + await lf.workspace.exec(b,'printf dirty > dirty'); + await assert.rejects(lf.workspace.unmount(b),e=>e.code==='Dirty'); + await lf.workspace.exec(b,'printf still-usable'); + const holder=spawn('/bin/sleep',['5'],{cwd:b.path,stdio:'ignore'}); + try { await assert.rejects(lf.workspace.unmount(b,true),e=>e.code==='Busy'); } + finally { holder.kill('SIGTERM');await deadline(new Promise(r=>holder.once('exit',r)),2000); } + await lf.workspace.unmount(b,true);await lf.workspace.unmount(another,true);await lf.sandbox.close(sb.sandbox); + // A killed binding loses its lease, not its intent. No subsequent caller + // may send a second unmount when the daemon may still own the first. + const {unlink}=await import('node:fs/promises');await unlink(base+'/release');await unlink(base+'/started'); + const sb2=await lf.sandbox.create();const lost=await lf.workspace.mount(sb2.sandbox,'a'); + owner=child('owner',lost);await waitFile(base+'/started');owner.p.kill('SIGKILL');await deadline(owner.ended,5000); + await fast('lost-owner-no-replay',()=>refused(()=>lf.workspace.unmount(lost,true),'Retained')); + await writeFile(base+'/release','go'); + await sleep(300); + // Explicit close takes over the orphan while its daemon is healthy; + // it never resends the unknown Workspace unmount. + await fast('lost-owner-close-takeover',()=>lf.sandbox.close(sb2.sandbox)); + // A failed helper leaves daemon custody and the original SDK error. + // Repeated calls read that terminal result rather than launching again. + const sb3=await lf.sandbox.create();const kept=await lf.workspace.mount(sb3.sandbox,'a'); + await writeFile(base+'/fail','fail'); + const error=await lf.workspace.unmount(kept,true).then(()=>null,e=>e);assert.ok(error); + await fast('repeat-retained-result',async()=>{ + const repeated=await lf.workspace.unmount(kept,true).then(()=>null,e=>e); + assert.equal(repeated.code,error.code);assert.equal(repeated.message,error.message);assert.equal(repeated.detail.state,'Retained'); + }); + await fast('retained-close-takeover',()=>lf.sandbox.close(sb3.sandbox)); + console.log('UNMOUNT_CONCURRENCY PASS'); + }finally{ + await writeFile(base+'/release','cleanup'); + if(owner?.p.exitCode===null&&owner?.p.signalCode===null)owner.p.kill('SIGKILL'); + if(active)await deadline(active.catch(()=>{}),5000); + // Always give the fixture helpers permission to finish. A failed test + // must not leave a stopped helper or a held client request behind. + await lf.close(); + // Failure cleanup is scoped by the explicit fixture root. Detach while + // its daemon is still alive; never remove directories through mounts. + const {spawnSync}=await import('node:child_process'); + const table=spawnSync('/sbin/mount',[],{encoding:'utf8',timeout:5000}); + for(const line of (table.stdout||'').split('\n')){ + const point=line.match(/ on (.*) \(/)?.[1]; + if(point?.startsWith(base+'/local/')){ + const off=spawnSync('/sbin/umount',['-f',point],{encoding:'utf8',timeout:15000,killSignal:'SIGKILL'}); + assert.equal(off.status,0,'fixture detach: '+off.stderr); + } + } + const processes=spawnSync('/bin/ps',['-axo','pid=,command='],{encoding:'utf8',timeout:5000}); + for(const line of (processes.stdout||'').split('\n')){ + if(line.includes(daemon+' --config '+base+'/local/')){ + const pid=Number(line.trim().split(/\s+/)[0]); + try{process.kill(pid,'SIGTERM');}catch(e){if(e.code!=='ESRCH')throw e;} + } + } + } +} diff --git a/api/docs/agent.md b/api/docs/agent.md index 330fcab914..7dc9c7b565 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -45,7 +45,7 @@ async and return a ready client, and `new LayerFS()` is refused `InvalidInput`. **There is no options argument:** every setting comes from the environment, read once when the client is made, by its `LAYERFS_` name only: `LAYERFS_HOME`, `_STORE_PATH`, `_DOCKER`, `_IMAGE`, -`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_TRACE`, +`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_SANDBOX`, `_LOCAL_DIR`, `_TRACE`, `_STORE_POLICY` (and `_TRACE_STEP`, read at each call). An object passed where the options went (`LayerFS.open(name, {…})`, or an object as `LayerFS.init`'s third argument) is refused `InvalidInput`, never ignored. `open` opens the project's Store and makes one project lookup @@ -105,7 +105,9 @@ the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere else opening fails `Unsupported`, and without a `storePath` it fails `InvalidInput`. On macOS and Windows, any process that names its LayerFS home's worker (`LAYERFS_WORKER_ENDPOINT`, or the endpoint `@layerfs/runtime` -hands over with `useWorker`) sends the calls there instead. The route is open to any integration (owner, +hands over with `useWorker` — or that project's endpoint, on a +`vm:` home where each project's Store is in its own VM) sends the +calls there instead. The route is open to any integration (owner, 2026-10-24); LayerFS names none. A client keeps the binding it was made with: the in-process addon, or the worker if one was set when it was made, so a client made before `@layerfs/runtime` hands over its worker stays in-process and @@ -141,10 +143,13 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional path of the real addon; defaults to `engine/crates/layerfs-node/layerfs.node`. | -| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows: the endpoint file of the LayerFS home's worker, which then makes every binding call. Any integration may set it. | +| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows, and a `vm:` project VM: the endpoint file of the worker that makes this project's binding calls. Any integration may set it. | | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | +| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `native` (macOS only): a Sandbox is a daemon process of this user, with no container, no root and no Docker setting read; Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Elsewhere `sandbox.create` refuses `native` as `Unsupported`. Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=native`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each native Sandbox gets its own directory under it. | +| `LAYERFS_SANDBOX_BACKEND` | With `LAYERFS_SANDBOX=native` on Linux, optional: `native` (unset) or `namespaces`. `namespaces` puts each Sandbox's daemon in its own mount, PID and IPC namespaces and its own cgroup v2 directory, and runs commands as the sandbox's uid:gid with no capabilities and no-new-privs; see [Namespace Sandboxes (Linux)](#namespace-sandboxes-linux). It needs root, cgroup v2, `/dev/fuse` and `user.max_user_namespaces=0`, and is refused `Unsupported` elsewhere or without them. Any other value is refused `InvalidInput` before anything is made. It is read when the client is made and sent with every call, so a Sandbox made with one backend is never reopened as the other. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | @@ -244,6 +249,125 @@ Daemon/container restart recovery and replay of uncertain operations are exclude Background process ownership is in the native executor, including persisted process identity; the filesystem daemon launches and supervises no commands. +### Native Sandboxes (macOS) + +`LAYERFS_SANDBOX=native` is the native runtime for macOS, with no root, no +Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon +runs as a process of this user and serves its Workspaces over NFSv3 on +127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them +(`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands +run as this user, in this machine's namespaces: **it isolates nothing** +beyond what the caller's own sandbox (for instance Seatbelt) imposes. A +command starts with this process's `PATH` (so `node`, Homebrew's tools and +the like are found as in the caller's own shell) and its own environment, +nothing else of this process's. Docker stays the default, and Linux keeps +the Docker route only. + +It is the in-process route: the client loads the macOS addon and opens the +Store itself, so no worker endpoint (`LAYERFS_WORKER_ENDPOINT`) is set. It +needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned by +this user, 0755; the directories above it need not be traversable by others) +and `LAYERFS_STORE_PATH` on a local apfs or hfs volume; a Store on any other +filesystem is refused `Unsupported`. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` +and `_DAEMON_HOST` are not read, and an `image` given to `sandbox.create` is +ignored. Everything else above holds: numbered Sandboxes of up to 16 +Workspaces, `sandbox.create` as the only start, `sandbox.close`, idle close +after 2 minutes, reclaim, records under `LAYERFS_HOME//sandboxes//` +and start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME`) and +`daemon.log` (the daemon's output, what `backend.daemon.sandbox.get_log` +reads; a failed start's report carries its last 20 lines). The daemon listens +on 127.0.0.1 only. Its records name the daemon by process id and start time, +so a reused id is never taken for it. `sandbox.create`, the mount result and +`project.list_sandboxes()` give a native Sandbox's `image` as `local`. A +record's `runtime` is `native`; it is used only by a client whose deployment +is `native` with the same `LAYERFS_LOCAL_DIR` and prefix, so it puts that +Sandbox exactly where the record says; any other is refused `InvalidInput` +and nothing is deleted. A record of the former `local` runtime (an earlier, +unmerged build) is never used: it is refused the same way and nothing it +names is deleted. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next `sandbox.create` +or `runtime.sandbox.reclaim` reclaims it, forcing off any mount the dead +daemon left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. `@layerfs/runtime`'s Docker-side +`runtime.sandbox.list` and `sweep` do not see native Sandboxes. Before a Commit the daemon +writes back what the client buffered; after a Commit or merge installs +another root, it makes the client drop the replaced files' cached attributes +at once. + +`LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's +helper commands (`{port}` and `{dir}` are filled in). The default mount is +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client +then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are +read; an integration that can set only its own prefixed names maps them +itself. + +Limits: background processes still read `/proc` and are not available on +macOS; many small writes cost about six times APFS's time; checked on Apple +silicon with macOS 27.2. + +### Namespace Sandboxes (Linux) + +`LAYERFS_SANDBOX_BACKEND=namespaces` keeps the native deployment (no Docker, no +image) and adds isolation inside the project's VM: each Sandbox's daemon runs +in its own mount, PID and IPC namespaces and its own cgroup v2 directory +(`/sys/fs/cgroup/layerfs/`), so two Sandboxes on one Store cannot +see each other's mounts, processes or files, and closing one ends everything +it started. + +It needs root (mount and PID namespaces, `/dev/fuse`), the unified cgroup v2 +hierarchy with `cgroup.kill`, and `user.max_user_namespaces=0`: without the +last, a command could unshare a user namespace and keep its own copy of the +Workspace mount, outside the daemon's unmount. The runtime never sets that +limit; the VM's owner does (`sysctl -w user.max_user_namespaces=0`). A host +that fails any of these is refused `Unsupported` before anything is made, and +the message names the missing condition. + +Sandbox `n` lives where a native Sandbox does — `LAYERFS_LOCAL_DIR/-/` +with `config/daemon.setup`, `overlay/`, `workspaces/`, `tmp/` and +`daemon.log` — plus `bin/layerfs-daemon` (the daemon executable, copied in +0755) and its sibling `-.root/`, the private root its init +stage mounts a tmpfs on and pivots to. The private root holds: the VM's own +userland (`/usr`, `/bin`, `/sbin`, `/lib`, `/lib64`, `/etc`) bound read-only, +so there is no image and no package restoration; a private `/proc` (only the +Sandbox's processes), `/dev` (the basic devices, `/dev/fuse`, a private +`devpts` and `shm`), `/tmp`; the Sandbox's own directory bound at its own path, +so `config/`, `overlay/` and `workspaces/` keep the same names on both +sides; and the Store's directory bound read-write, so the daemon's SQLite WAL +and the writers' directory lock see the same inode as every other opener on +the kernel. The daemon listens on the VM's `127.0.0.1`, as a native one does, +and `127.0.0.1:0` in its setup means the same thing. + +Commands run through `layerfs-sandbox-helper`, which is looked up at +`LAYERFS_SANDBOX_HELPER` when set and otherwise beside the daemon executable. +Each command enters the Sandbox's namespaces and runs as its uid:gid with no +effective or permitted capabilities and no-new-privs; it cannot mount, cannot +unmount the Workspace and cannot create a user namespace. A background process +it leaves behind (even after `setsid`) is ended by the Sandbox's close, which +is one `cgroup.kill` write followed by waiting for the group to be empty; +Delete then removes the cgroup and the directory, so a closed Sandbox leaves no +process, mount or cgroup behind. No graceful filesystem drain exists. + +A record's `runtime` is `namespaces`, with the same fields as a native record; +it is used only by a client whose `LAYERFS_SANDBOX_BACKEND` is `namespaces` +with the same `LAYERFS_LOCAL_DIR` and prefix. Another backend's record — and a +`namespaces` record seen by a native deployment — is refused `InvalidInput` +and nothing is deleted. Reopen checks the recorded init process (id, start +time, boot) and its cgroup membership, so a reused id, or a process outside +the Sandbox's cgroup, is never taken for it. Everything else above holds: +numbered Sandboxes, up to 16 Workspaces each, one daemon per Sandbox, idle +close after 2 minutes, reclaim, and `image` reported as `local`. + +Limits: the network namespace is shared with the VM, so a command can reach +another Sandbox's daemon port on `127.0.0.1`; `/sys` is not mounted; the +userland is the VM's, not a pinned image; and the private root's bind mounts +are read-only at their top only, as this kernel's remount is not recursive. + ### Exec output `exec` returns `{ kind, exitCode, stdout, stderr, message, truncated }` when the diff --git a/api/docs/runtime.md b/api/docs/runtime.md index 6ce3e1e341..418096d6a3 100644 --- a/api/docs/runtime.md +++ b/api/docs/runtime.md @@ -15,7 +15,7 @@ Agents don't import it; a host that runs projects for its users does. import { LayerFSRuntime } from "@layerfs/runtime"; const runtime = await LayerFSRuntime.open({ home, image, daemon, bundle, docker, containerPrefix }); try { - await runtime.status(); // { platform, missing, docker, worker, disk, sweep } + await runtime.status(); // { platform, placement, missing, docker, worker, vm, disk, sweep } const job = await runtime.project.start_init("shop", "/src/shop", { timeout }); // timeout optional: LayerFS.init's 300 s default await runtime.project.get_init(job.id); // { step, scanned, totals, imported, project, error, ended } await runtime.project.list(); // [{ name, volume, image, source, created, initialized }] @@ -34,6 +34,19 @@ try { } ``` +A `vm:` home has one call group more (`vm`), names a project in +`worker.start`/`worker.restart` (one VM per Store, see +[Placement](#placement-one-linux-vm-per-store)), and reports each project's VM: + +```js + await runtime.worker.start("shop"); // this project's VM: create/start, bundle, worker, endpoint + await runtime.worker.restart("shop", { confirm: "restart" }); + await runtime.vm.status(); // { provider, placement, vms: [{ project, name, state, worker, disk }] } + await runtime.vm.status("shop"); // that project's VM record + await runtime.vm.stop("shop"); // stops it; its Store disk and its Store stay + await runtime.vm.delete("shop", { confirm: "delete", discardStore: false }); // the VM record only, unless discardStore +``` + ## Opening and settings The runtime opens on a LayerFS home, not a project, so it works before the @@ -44,7 +57,8 @@ on macOS Docker Desktop's `~/.docker/run/docker.sock` when only that one exists, checked once at `open`), `volumes` (Linux), `traceDir` (default `/traces`), `containerPrefix`, `stateDir` (default `/.runtime`, the runtime's project records), `sweep` -(default true) and on Windows `wsl: { distribution, user, node, home }`. A +(default true), `placement` (default `local`; `vm:` places the home +in one Linux VM, below) and on Windows `wsl: { distribution, user, node, home }`. A missing setting does not refuse `open`: `status().missing` lists it. ## Init @@ -109,7 +123,10 @@ lines from now on, or from `since`, each with `actor`, its trace file's name. ## Platforms and the worker Each platform folder implements one interface (`endpoint`, `status`, -`start`, `restart`, `init`, `path` and the Docker, home and disk helpers): +`start`, `restart`, `init`, `path`, the Store hooks `makeStore`, +`storeHolders`, `releaseStoreHolders` and `removeStore`, and the Docker, home +and disk helpers); `platform/vm/` implements the same interface over a VM +(see [Placement](#placement-one-linux-vm-per-home)). - **Linux** runs Store calls in this process. There is no worker: `worker.start` and `worker.restart` are refused `InvalidInput`. Init @@ -128,27 +145,206 @@ Each platform folder implements one interface (`endpoint`, `status`, `wslpath`. This process owns the worker through its stdin: `close()` drains and ends it. -The runtime starts a worker only on macOS and Windows. The worker route is -open to any integration (owner, 2026-10-24): on those platforms the runtime -hands its endpoint to `@layerfs/agent` (`useWorker(endpoint)`, without -changing the environment), so every client made after `open` in its process -uses the worker; a client made before keeps its in-process binding. One -worker per process: a second runtime of another home is refused. Another -process, such as an agent's command, uses the worker when its -`LAYERFS_WORKER_ENDPOINT` names the endpoint file; the integration that runs -it sets that. Such a process also needs the project's settings: make the -project with `project.start_init`, which creates and records its volume, then -set `LAYERFS_VOLUME` to the volume `project.list` gives, -`LAYERFS_STORE_PATH` to `/var/lib/docker/volumes//_data/global/store.sqlite`, -and `LAYERFS_IMAGE` and `LAYERFS_DAEMON` for its Sandboxes (on macOS, the -bundle's `layerfs-daemon`). A volume made outside the runtime works for Store -calls, but `project.list` shows the project with `volume: null` and -`volume.list` marks the volume `unknown`. The endpoint is `/worker/endpoint.json`, `{ home, port, -secret, build, daemonHost?, docker?, init? }`, readable by its user only. On macOS -`docker` is the engine's socket inside its VM, `/var/run/docker.sock`, which the -worker and Init's container mount; it replaces every call's `deploy.docker`, so the -Mac's own socket (`LAYERFS_DOCKER`) never reaches Linux. The worker -checks every file of the bundle's `manifest.json` before it listens. +The runtime starts a worker only on macOS and Windows, and in a +`vm:` home. The worker route is open to any integration (owner, +2026-10-24): on macOS and Windows the runtime hands its endpoint to +`@layerfs/agent` (`useWorker(endpoint)`, without changing the environment), +and on `vm:` it hands over a resolver (`useWorker((project) => endpoint)`), +so every client made after `open` in its process uses the worker; a client +made before keeps its in-process binding. One worker route per process: a +second runtime of another home is refused. Another process, such as an +agent's command, uses the worker when its `LAYERFS_WORKER_ENDPOINT` names +the endpoint file; the integration that runs it sets that (on a `vm:` home +`deploymentEnv` sets it to that project's VM). Such a process also needs +the project's settings: make the project with `project.start_init`, which +creates and records its volume, then set `LAYERFS_VOLUME` to the volume +`project.list` gives, +`LAYERFS_STORE_PATH` to `/var/lib/docker/volumes//_data/global/store.sqlite` +(or on `vm:` `/layerfs/store/global/store.sqlite`), and `LAYERFS_IMAGE` and +`LAYERFS_DAEMON` for its Sandboxes (on macOS, the bundle's `layerfs-daemon`). +A volume made outside the runtime works for Store calls, but `project.list` +shows the project with `volume: null` and `volume.list` marks the volume +`unknown`. On macOS and Windows the endpoint is +`/worker/endpoint.json`, on `vm:` `/worker/.json`, `{ home, +port, secret, build, daemonHost?, docker?, init?, vm? }`, readable by its user +only. On macOS `docker` is the engine's socket inside its VM, +`/var/run/docker.sock`, which the worker and Init's container mount; it +replaces every call's `deploy.docker`, so the Mac's own socket +(`LAYERFS_DOCKER`) never reaches Linux. The worker checks every file of the +bundle's `manifest.json` before it listens. + +## Placement: one Linux VM per Store + +`placement` (also read from `LAYERFS_PLACEMENT` when the setting is absent) is +`local` (the default: this machine's Docker, macOS worker container or WSL2 +worker, unchanged) or `vm:`: **each of this home's projects — its +Stores — lives in one Linux VM of its own** on this machine (#515 §6, V9, V1). +A project's VM holds that project's Store on its own ext4 disk, its Sandboxes +are Linux namespaces (`LAYERFS_SANDBOX_BACKEND=namespaces`, H36's backend) and +its worker runs the real SDK as root; the host reaches only each VM's worker, +with one attempt per call and no replay. Providers, both verified against the +real CLI by `test/verify/vm_real.mjs` and by H40's own run: `microsandbox` +(`msb` 0.7.8), `apple-container` (`container` 1.6.1) and `ssh` (any Linux host +the user provides, reached with the system `ssh` client; see +[the SSH provider](#the-ssh-provider-any-linux-host) below). + +```sh +LAYERFS_PLACEMENT=vm:microsandbox layerfs-home # any integration that sets it +``` + +Every per-VM thing is that project's alone: the VM `-vm`, its +Store disk `-vm-store`, its host port, its bundle copy and its +worker secret. Nothing is shared between two projects' VMs except the read-only +host home bind mount (§ Guest layout). + +Settings: `vm: { image, cpus, memory, storeSize }` (`LAYERFS_VM_IMAGE`, +`LAYERFS_VM_CPUS`, `LAYERFS_VM_MEMORY`, `LAYERFS_VM_STORE_SIZE`; defaults +`node:22-trixie-slim`, 2, `2G`, `20G`). The image is the VM image: it must hold +`node`, and it is not the Sandbox image. A VM home needs `bundle` and no +`image`; its `daemon` setting is the guest's, `/layerfs-daemon`, with +`layerfs-sandbox-helper` beside it. `storeSize` is the disk's size where the +provider has one: Microsandbox takes it when it makes the volume, Apple +Container at `volume create -s` (the CLI's own default is a sparse 504 GiB +image), and an existing volume keeps the size it was made with. + +Guest layout (`settings.js`'s `GUEST`), the same paths in every project's VM: +the home is a bind mount of the host's ``, so project entries and traces +are the same files on both sides; **the Store disk is mounted at the Store +directory**, so the Store is `/layerfs/store/global/store.sqlite`; the sealed +bundle is copied in at `/layerfs/bundle`; an Init source folder is copied to +`/layerfs/import/`; the worker's settings, pid and log live in +`/layerfs/state`; **Sandbox directories are on the Store disk, at +`/layerfs/store/local`**, not on the VM's root: a namespace Sandbox's root is +an `overlay` whose writable layer must sit on a local filesystem, and a +provider's own root can be an overlay itself (Microsandbox's is), while the +Store disk is an ext4 disk on both providers. The home mount is the one thing +two VMs share, and it holds records, not Stores: what a VM cannot see is +another project's Store, which is on another disk in another VM. + +A bundle may also carry the Sandbox image as `/images/<64 hex>.tar` +(#515 §4.7 V7, the namespaces backend's convention): it comes in with the rest +of the bundle — its own copy in each VM — and the worker is started with +`LAYERFS_IMAGE=sha256:<64 hex>` and `LAYERFS_IMAGE_ARCHIVE=/layerfs/bundle/images/<64 hex>.tar`, +while `deploymentEnv` gives clients that digest and that archive path as +`LAYERFS_IMAGE` and `LAYERFS_IMAGE_ARCHIVE`. The namespaces backend requires an +image (`capabilities.requires_image`), so a bundle without one starts no +Sandbox: the request is refused, naming the missing setting. + +The adapter's five steps, each attempted once, all for one named project: + +1. the project's VM exists and runs, from `settings.vm`, with its named disk + volume and the home bind mount — `create` once, else `start`; a record of + another home, or one whose mounts or port differ, is refused rather than + reused or repaired; +2. the sealed bundle is copied into that VM once per bundle hash (the hash of + its `manifest.json`), with a marker at `/layerfs/bundle/.sealed`; +3. `user.max_user_namespaces=0` is written and read back (V4; Apple Container's + guest mounts `/proc/sys` read-only, so its adapter remounts it first), then + the worker starts as root with its secret in a 0600 guest file — never in a + command line — and the pid it wrote; +4. the endpoint is written to `/worker/.json` once the worker + answers on `127.0.0.1:`, the VM's `-p 127.0.0.1::7070` + (V5: the Sandboxes share the VM's network, so a daemon's loopback port + stays inside it); +5. `runtime.vm.stop(project)` stops that VM and keeps its disk; `runtime.vm.delete( + project, { confirm: "delete" })` removes the VM record and keeps the Store + disk unless `discardStore`. **A delete is refused while that project + exists**: its Store lies on that disk and nowhere else, and this build has + no remote Store push (#506), so V8's "never delete a Store that was not + pushed" makes the refusal the only safe answer. `project.remove` is the + explicit removal: it stops the project's Sandboxes, closes its Store, then + deletes the VM and its disk together. + +Init runs as one more call to that project's worker: `platform.init` starts its +VM and copies the source folder into `/layerfs/import`, and `LayerFS.init` +reads it there. `deploymentEnv` gives clients and agents the VM's settings: +`LAYERFS_SANDBOX=native`, `LAYERFS_SANDBOX_BACKEND=namespaces` (an explicit +`LAYERFS_SANDBOX_BACKEND` in the caller's environment wins), `LAYERFS_DAEMON`, +`LAYERFS_LOCAL_DIR`, `LAYERFS_HOME=/layerfs/home`, the Store's +`LAYERFS_STORE_PATH`, this project's `LAYERFS_WORKER_ENDPOINT`, and the bundle +image's `LAYERFS_IMAGE` when it has one — no `LAYERFS_DOCKER` or `_VOLUME`. + +A VM home's `status()` reports `placement`, `docker: { level: "not-used" }` +(host Docker is not in its data path), `worker: null` (no single worker serves +the home) and `vm: { provider, placement, vms: [{ project, name, state, image, +home, store, created, worker, disk }] }`, one entry per project of +`project.list()`. The Docker-shaped helpers read host Docker and refuse +`Unsupported` with the reason: `volume.list`, `volume.remove`, `sandbox.list`, +`sandbox.list_leftovers` and `sandbox.clean_leftovers`; `sandbox.sweep` answers +`{ reclaimed: {}, failed: [], host: false }`. The SDK's +`project.list_sandboxes` reads the VM's Sandboxes. + +The real proof is `node api/runtime/test/verify/vm_real.mjs [microsandbox|apple-container]`: two projects, each in its own VM — Init +through each VM's worker, a fork, a write, a Sandbox, an exec as the nonroot +identity with user namespaces blocked, a Commit, a stop/start of one VM and a +worker restart, then a check that neither Store holds the other's Branch, and +that removing one project leaves the other's VM running. The unit tests +(`node api/runtime/test/vm_placement.mjs settings|flow|two|bundle|delete|hosts|apple-container|ssh`) +use a stand-in provider CLI. + +### The SSH provider (any Linux host) + +`vm:ssh` places the home on a Linux host the user already has — an EC2 +instance, a machine on the network, or a VM they made — instead of one this +runtime starts (#515 §6). The host and its disk are the user's: the adapter +makes no disk and formats nothing, it proves that the path given is an ext4 +filesystem on that host's own kernel (`uname -s` is `Linux`, `findmnt` or +`blkid` says `ext4`) and attaches it at `/layerfs/store` (a bind mount for a +directory, `mount -t ext4` for a block device) when it is somewhere else. + +```sh +LAYERFS_PLACEMENT=vm:ssh LAYERFS_SSH_HOST=ec2-…amazonaws.com \ +LAYERFS_SSH_IDENTITY=~/.ssh/deploy LAYERFS_SSH_STORE=/mnt/layerfs \ + layerfs-home +``` + +Settings (each also read from the environment named): + +| Setting | Environment | Default | +| --- | --- | --- | +| `vm.host` | `LAYERFS_SSH_HOST` | required | +| `vm.user` | `LAYERFS_SSH_USER` | `root` | +| `vm.port` | `LAYERFS_SSH_PORT` | `22` | +| `vm.identityFile` | `LAYERFS_SSH_IDENTITY` | none: the agent's keys | +| `vm.storePath` | `LAYERFS_SSH_STORE` | `/layerfs/store` | +| `vm.strictHostKeyChecking` | — | `yes` | +| `vm.knownHostsFile` | `LAYERFS_SSH_KNOWN_HOSTS` | the agent's | + +Key login only: a password is refused by name, the key is a path that goes to +`ssh -i` (its bytes never enter an argument list or a log), and +`StrictHostKeyChecking` is `yes` unless the setting says otherwise — the +loosest it accepts is `accept-new`. + +What differs from a microVM provider, and why: + +- **One host serves one Store.** The Store path is one directory on the user's +disk, so a second project naming the same host and path is refused `Busy`, +naming the VM that holds it, instead of writing one Store file twice. +- **The private stream is `ssh -L 127.0.0.1::127.0.0.1:7070`**, started +with `-o ExitOnForwardFailure=yes`: a busy local port fails at once, before +anything is uploaded or started. Its stderr goes to +`/-forward.log`, not a pipe, so the host process (CLI, test, an +integration) exits while the user's VM keeps running. `stop` ends the forward +and the remote worker (SIGTERM, then SIGKILL, bounded). +- **Init sources are copied, not shared.** A microVM bind-mounts the host home; +an SSH host shares no path with this machine, so `platform.init` copies the +source to `/layerfs/import/`. The host's `/layerfs/home` is therefore its +own directory, not this machine's home. +- **A delete never touches the disk.** `runtime.vm.delete` removes the session +record, the forward, `/layerfs/bundle` and `/layerfs/state`, and keeps the +Store; the refusal while the project exists is the same V8 rule as above. The +host's `/layerfs/home` and `/layerfs/import` also stay (they are LayerFS's own +copies, but removing the home entry would let a second Init duplicate a project +that the live Store already holds). + +Its session records live in `LAYERFS_SSH_STATE_DIR` (default +`~/.layerfs/ssh`), one JSON file per VM, and a forward that no longer runs reads +as a stopped VM, so the next `worker.start` re-opens it. The failure matrix is +`node api/runtime/test/verify/ssh_faults.mjs [port|store|ext4|host|drop|disconnect|remote|delete|shared]`: +a busy local port, a missing or non-ext4 Store path, a non-Linux host, an +unreachable network and a reconnected one, a dropped forward, a rebooted host, +a refused and a permitted delete, and a second project on the same host. ## Docker diff --git a/api/runtime/src/clients.js b/api/runtime/src/clients.js index dd52c0e64a..6cbb348fe2 100644 --- a/api/runtime/src/clients.js +++ b/api/runtime/src/clients.js @@ -8,8 +8,12 @@ import { callHome } from "./settings.js"; const IDLE_MS = 60_000; -/** The environment one project's client (and its agents' bash calls) use. */ +/** The environment one project's client (and its agents' bash calls) use. A + * platform may replace it whole (`deploymentEnv`), as a `vm:` home + * does: its deployment is native Sandboxes in the VM's namespace backend, not + * host Docker. */ export function deploymentEnv(settings, platform, project) { + if (platform.deploymentEnv) return platform.deploymentEnv(settings, project); return { HOME: callHome(settings), DOCKER: settings.docker, diff --git a/api/runtime/src/index.js b/api/runtime/src/index.js index f33f05084b..5a390526be 100644 --- a/api/runtime/src/index.js +++ b/api/runtime/src/index.js @@ -20,6 +20,7 @@ import { resolveSettings } from "./settings.js"; import { status } from "./status.js"; import linux from "./platform/linux/index.js"; import macos from "./platform/macos/index.js"; +import { vmPlatform } from "./platform/vm/index.js"; import windows from "./platform/windows/index.js"; import { list } from "./project/list.js"; import { InitJobs } from "./project/jobs.js"; @@ -34,6 +35,8 @@ import { list as listVolumes, remove as removeVolumes } from "./volume.js"; import { subscribe } from "./events/subscribe.js"; const PLATFORMS = { linux, macos, windows }; +/** The platform object `settings` selects: this host's, or one VM's (§6). */ +const platformFor = (settings) => (settings.placement.kind === "vm" ? vmPlatform(settings.placement.provider) : PLATFORMS[settings.platform]); /** Held only by `open`: the constructor refuses without it. */ const FACTORY = Symbol("LayerFSRuntime.open"); @@ -50,10 +53,15 @@ export class LayerFSRuntime { constructor(settings, token = undefined) { if (token !== FACTORY) throw new LayerFSError("InvalidInput", "use `await LayerFSRuntime.open(settings)`; there is no public constructor", null); this.settings = settings; - this.platform = PLATFORMS[settings.platform]; + this.platform = platformFor(settings); this.closed = false; - const endpoint = this.platform.endpoint(settings); - if (endpoint) useWorker(endpoint); + // one worker for the home, or (a `vm:` placement) one endpoint per project + const endpoints = this.platform.endpoints?.(settings); + if (endpoints) useWorker(endpoints); + else { + const endpoint = this.platform.endpoint?.(settings); + if (endpoint) useWorker(endpoint); + } this.clients = new Clients(this); const jobs = new InitJobs((prepared, report) => runInit(this, prepared, report)); const live = (method) => async (...args) => { @@ -69,10 +77,13 @@ export class LayerFSRuntime { }; this.status = live(async () => ({ ...(await status(this)), sweep: this.swept })); this.worker = { - start: live(() => this.platform.start(settings)), - restart: live((options) => { + start: live((project) => this.platform.start(settings, project)), + restart: live((first, second) => { + // `restart(project, { confirm })` on a `vm:` placement, `restart({ confirm })` elsewhere + const project = typeof first === "string" ? first : undefined; + const options = project ? second : first; if (options?.confirm !== "restart") throw new LayerFSError("InvalidInput", "a restart cuts off every project's Store calls; confirm it with { confirm: \"restart\" }", null); - return this.platform.restart(settings, options); + return this.platform.restart(settings, project, options); }), }; this.project = { @@ -93,6 +104,18 @@ export class LayerFSRuntime { list: live(() => listVolumes(this)), remove: live((names) => removeVolumes(this, names)), }; + /** `vm:` only: one project's VM (`status(project)`), else every project's; absent on a local placement. */ + if (this.platform.vm) { + this.vm = { + status: live(async (project) => { + const names = project ? [project] : (await this.project.list()).map((one) => one.name); + const reported = await this.platform.vm.status(settings, names); + return project ? reported.vm.vms[0] : reported.vm; + }), + stop: live((project) => this.platform.vm.stop(settings, project)), + delete: live((project, options) => this.platform.vm.delete(settings, project, options)), + }; + } /** The open-time sweep: null when off, else { state: running | done | failed, … }. */ this.swept = null; this.sweeping = null; diff --git a/api/runtime/src/platform/linux/index.js b/api/runtime/src/platform/linux/index.js index 58adc9577c..3770bd969a 100644 --- a/api/runtime/src/platform/linux/index.js +++ b/api/runtime/src/platform/linux/index.js @@ -4,7 +4,7 @@ // keeps root-only: `init` checks this process can write there first. import { accessSync, constants, existsSync, rmSync, statSync, statfsSync } from "node:fs"; import { join } from "node:path"; -import { docker, homeNames, refuse } from "../shared.js"; +import { docker, homeNames, makeStore, refuse, releaseStoreHolders, removeStore, storeHolders } from "../shared.js"; const NO_WORKER = "Linux runs Store calls in-process; there is no worker"; @@ -27,6 +27,10 @@ export default { }, path: (settings, hostPath) => hostPath, docker, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `${settings.volumes}/${volume}/_data/global/store.sqlite`, homeNames: (settings) => homeNames(settings.home), async removeHome(settings, name) { rmSync(join(settings.home, name), { recursive: true, force: true }); }, diff --git a/api/runtime/src/platform/macos/index.js b/api/runtime/src/platform/macos/index.js index 3a815089ca..5702bd8dcc 100644 --- a/api/runtime/src/platform/macos/index.js +++ b/api/runtime/src/platform/macos/index.js @@ -14,7 +14,7 @@ import { randomBytes } from "node:crypto"; import { mkdirSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; -import { docker as cli, endpointFile, homeNames, ping, ready, refuse, writeEndpoint } from "../shared.js"; +import { docker as cli, endpointFile, homeNames, makeStore, ping, ready, refuse, releaseStoreHolders, removeStore, storeHolders, writeEndpoint } from "../shared.js"; const VOLUMES = "/var/lib/docker/volumes"; const WORKER_PORT = 7070; @@ -126,6 +126,10 @@ const platform = { }, path: (settings, hostPath) => hostPath, docker: cli, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `${VOLUMES}/${volume}/_data/global/store.sqlite`, homeNames: (settings) => homeNames(settings.home), async removeHome(settings, project) { rmSync(join(settings.home, project), { recursive: true, force: true }); }, diff --git a/api/runtime/src/platform/shared.js b/api/runtime/src/platform/shared.js index cb0b1dd7f1..8a1d826384 100644 --- a/api/runtime/src/platform/shared.js +++ b/api/runtime/src/platform/shared.js @@ -11,12 +11,24 @@ import { frame, readFrames } from "@layerfs/agent/internal"; export const run = promisify(execFile); export const refuse = (code, message, detail = null) => new LayerFSError(code, message, detail); +/** + * The refusal a Docker-shaped helper gives on a `vm:` home: host + * Docker holds none of that home's Stores or Sandboxes, so answering from it + * would describe another machine. + */ +export const noHostDocker = (runtime, what, instead) => + refuse("Unsupported", `${what} reads host Docker, and this home is placed in ${runtime.platform.name}: ${instead}`, null); + /** One Docker CLI call on the configured engine: its stdout, trimmed. */ export const docker = (settings, args, options = {}) => run("docker", ["-H", `unix://${settings.docker}`, ...args], { timeout: 5000, ...options }).then((r) => r.stdout.trim()); -/** Where a home's worker endpoint lives: `/worker/endpoint.json`. */ -export const endpointFile = (home) => join(home, "worker", "endpoint.json"); +/** + * Where a worker's endpoint lives: `/worker/endpoint.json` for a home + * whose one worker serves it (macOS, Windows), or `/worker/.json` + * when each project's Store is in its own VM (`vm:`). + */ +export const endpointFile = (home, project = undefined) => join(home, "worker", project ? `${project}.json` : "endpoint.json"); /** Writes an endpoint atomically, readable by this user only. */ export function writeEndpoint(file, info) { @@ -54,3 +66,39 @@ export function homeNames(home) { if (!existsSync(home)) return []; return readdirSync(home, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(home, e.name, "project.record"))).map((e) => e.name).sort(); } + +// The four Store hooks Init and project removal use, as Docker has them (the +// `local` placement): a project's Store is a labelled volume, its holders are +// containers, and removing the project removes the volume. A `vm:` placement +// implements the same four over the project's VM instead (vm/index.js). Each +// takes the runtime and the project `{ name, volume }`, so the hook uses that +// platform's own Docker CLI (Windows goes through WSL) or that project's VM. + +/** Container states that hold a volume but run nothing (Docker's; a namespace Sandbox's record has none). */ +export const STOPPED = new Set(["created", "exited", "dead"]); + +/** Init makes the project's Store volume before installing into it. */ +export async function makeStore(runtime, project) { + const { settings, platform } = runtime; + await platform.docker(settings, ["volume", "create", "--label", "layerfs.role=store", "--label", `layerfs.home=${settings.home}`, "--label", `layerfs.project=${project.name}`, project.volume], { timeout: 30_000 }); +} + +/** The containers holding the volume now: [{ id, state }], running or not. */ +export async function storeHolders(runtime, project) { + const { settings, platform } = runtime; + const rows = await platform.docker(settings, ["ps", "-a", "--filter", `volume=${project.volume}`, "--format", "{{.ID}} {{.State}}"]); + return rows.split("\n").filter(Boolean).map((line) => { const [id, state] = line.split(" "); return { id, state }; }); +} + +/** Removes the volume's holders: the running ones only when `force`. */ +export async function releaseStoreHolders(runtime, project, holders, { force = false } = {}) { + const { settings, platform } = runtime; + if (!holders.length) return; + await platform.docker(settings, ["rm", ...(force ? ["-f"] : []), ...holders.map((holder) => holder.id)], { timeout: 30_000 }); +} + +/** The volume itself, once nothing holds it. */ +export async function removeStore(runtime, project) { + const { settings, platform } = runtime; + await platform.docker(settings, ["volume", "rm", project.volume], { timeout: 30_000 }); +} \ No newline at end of file diff --git a/api/runtime/src/platform/vm/apple-container.js b/api/runtime/src/platform/vm/apple-container.js new file mode 100644 index 0000000000..dca57ac1e6 --- /dev/null +++ b/api/runtime/src/platform/vm/apple-container.js @@ -0,0 +1,151 @@ +// Apple Container (`container`, #515 §6, V3's third adapter). The VM is one +// container of the `container` CLI; the Store is a named volume the CLI keeps +// across stop and start, mounted at the Store directory; the private stream is +// `-p 127.0.0.1:H:G`. +// +// Verified against `container` 1.6.1 (build aa8d76b) on macOS 27.2 / M2, +// 2026-10-31, one VM per Store. What the CLI actually does, and how this file +// answers it: +// - `inspect NAME` prints JSON by default (it has no `--format`): an +// **array** of `[{ id, status: { state, +// startedDate, networks }, configuration: { image, labels, mounts, +// publishedPorts, creationDate, … } }]`. `status` is an object, not a +// string; mounts carry `destination`/`source`/`type.volume.name`; ports are +// `publishedPorts[].{containerPort,hostPort,hostAddress}`; the image +// reference and digest are `image.reference`/`image.descriptor.digest`. +// - a volume keeps its size: `volume create -s ` (the CLI's default is +// a sparse 504 GiB image), so `ensureStore` sizes it from `storeSize`; +// - `exec` needs `-i` for a caller's stdin (`cat > file` gets EOF at once +// without it), which `write` and `copy` rely on; +// - the guest's `/proc/sys` is mounted **read-only** and user namespaces are +// open by default: V4's block needs `mount -o remount,rw /proc/sys` first +// (CAP_SYS_ADMIN, which this adapter adds anyway), then the write; +// - `delete --force` ends a running container; `start`/`stop` keep the +// published port; `volume delete` removes the Store disk. +// It reads its facts strictly: an `inspect` answer that is not this shape is +// refused by name rather than guessed at, so a changed CLI shows up as an error +// and never as a VM this adapter believes is running. +import { answer, capture, readFile, writeFile } from "./cli.js"; + +const CLI = "container"; +const CREATE_MS = 120_000, CALL_MS = 30_000, EXEC_MS = 60_000; +/** The guest kernel shares FUSE, cgroup and loopback with these capabilities (#512: they were required). */ +const CAPABILITIES = ["--cap-add", "SYS_ADMIN", "--cap-add", "SYS_RESOURCE", "--cap-add", "NET_ADMIN"]; + +const call = (args, options = {}) => answer(CLI, args, { timeout: CALL_MS, ...options }); + +/** One `container exec`, plain streams, stdin kept open when the caller sends some. */ +const exec = (name, argv, { input = undefined, timeout = EXEC_MS } = {}) => + capture(CLI, ["exec", ...(input === undefined ? [] : ["-i"]), name, ...argv], { input, timeout }); + +const STRANGE = (what) => new Error(`the Apple Container CLI's ${what} is not the shape this adapter knows; check the CLI's output before using it`); + +/** The VM's record, or null when it does not exist. See the header: strict, never guessed. */ +export async function inspect(name) { + let text; + try { + text = await call(["inspect", name]); + } catch (error) { + const said = `${error.stderr ?? ""}${error.message ?? ""}`; + if (/not found|no such (container|sandbox)/i.test(said)) return null; + throw error; + } + const parsed = JSON.parse(text); + const record = Array.isArray(parsed) ? parsed[0] : parsed; + if (!record || typeof record !== "object") throw STRANGE("inspect"); + const configuration = record.configuration; + const status = record.status; + if (!configuration || typeof status !== "object" || typeof status?.state !== "string") throw STRANGE("inspect (status or configuration)"); + const mounts = configuration.mounts ?? []; + if (!Array.isArray(mounts)) throw STRANGE("inspect (mounts)"); + const mounted = (guest) => mounts.find((mount) => mount.destination === guest) ?? null; + const ports = configuration.publishedPorts ?? []; + if (!Array.isArray(ports)) throw STRANGE("inspect (publishedPorts)"); + return { + state: status.state.toLowerCase(), + image: configuration.image?.reference ?? null, + digest: configuration.image?.descriptor?.digest ?? null, + labels: configuration.labels ?? {}, + ports: ports.map((port) => ({ guest: Number(port.containerPort), host: Number(port.hostPort), bind: port.hostAddress ?? null })), + home: mounted("/layerfs/home")?.source ?? null, + // a named Store volume is named in the mount's type, and its source is the + // volume's image file on the host + store: mounted("/layerfs/store")?.type?.volume?.name ?? null, + created: configuration.creationDate ?? null, + }; +} + +/** Newly created named Store volumes in this session; cleaned up if create fails. */ +const newlyCreatedStores = new Set(); + +/** + * The Store disk: a named volume of `size`, made once. An existing volume keeps + * the size it was made with (the CLI has no resize), so only its presence is + * checked here. + */ +export async function ensureStore(name, size) { + const found = await capture(CLI, ["volume", "inspect", name]); + if (found.code === 0) return; + const made = await capture(CLI, ["volume", "create", "-s", size, name], { timeout: CREATE_MS }); + if (made.code !== 0) throw new Error(`could not create the Store volume ${name}: ${(made.stderr || made.stdout).trim()}`); + newlyCreatedStores.add(name); +} + +export async function create(config) { + const wasNew = newlyCreatedStores.has(config.storeName); + const existedBefore = wasNew ? false : (await capture(CLI, ["volume", "inspect", config.storeName])).code === 0; + try { + await call([ + "run", "--detach", "--name", config.name, "--cpus", String(config.cpus), "--memory", config.memory, + ...CAPABILITIES, + "--mount", `type=volume,source=${config.storeName},target=/layerfs/store`, + "--mount", `type=bind,source=${config.home},target=/layerfs/home`, + "--publish", `127.0.0.1:${config.hostPort}:${config.workerPort}`, + "--label", "layerfs.role=vm", "--label", `layerfs.home=${config.home}`, + config.image, "sleep", "infinity", + ], { timeout: CREATE_MS }); + newlyCreatedStores.delete(config.storeName); + return inspect(config.name); + } catch (error) { + if (wasNew || !existedBefore) { + newlyCreatedStores.delete(config.storeName); + await removeStore(config.storeName).catch(() => {}); + } + throw error; + } +} + +export const start = (name) => call(["start", name], { timeout: CREATE_MS }); +export const stop = (name) => call(["stop", name]); +export const restart = async (name) => { await stop(name); await start(name); }; +/** The container record only: the Store volume is a separate object and is never removed here. */ +export const remove = (name) => call(["delete", "--force", name], { timeout: CREATE_MS }); +export const removeStore = (storeName) => call(["volume", "delete", storeName]); + +/** + * V4: the VM-wide user-namespace block. The guest mounts `/proc/sys` + * read-only, so it is remounted read-write first; the caller reads the sysctl + * back, so a block that did not take is reported, not assumed. + */ +export const blockUserNamespaces = (name) => + exec(name, ["sh", "-c", "mount -o remount,rw /proc/sys 2>/dev/null; echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the VM's Store volume, or null when `df` cannot answer. */ +export async function storeFree(name) { + const { code, stdout } = await exec(name, ["df", "-kP", "/layerfs/store"]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +/** Copies a host folder into a guest folder. */ +export const copy = (name, source, destination) => call(["copy", source, `${name}:${destination}`], { timeout: CREATE_MS }); + +export const read = (name, path) => readFile(CLI, name, path, exec); +export const write = (name, path, text) => writeFile(CLI, name, path, text, exec); + +export default { + name: "apple-container", + defaultImage: "node:22-trixie-slim", + inspect, ensureStore, create, start, stop, restart, remove, removeStore, blockUserNamespaces, storeFree, read, write, copy, exec, +}; diff --git a/api/runtime/src/platform/vm/cli.js b/api/runtime/src/platform/vm/cli.js new file mode 100644 index 0000000000..e0675f6780 --- /dev/null +++ b/api/runtime/src/platform/vm/cli.js @@ -0,0 +1,45 @@ +// One VM provider CLI call, for the two providers in this folder: the command +// is spawned with pipes, its output stays apart from its error stream, and its +// exit code comes back. Nothing is retried, nothing is buffered beyond the +// call, and a call that runs past its timeout is killed and reported. +import { spawn } from "node:child_process"; +import { run } from "../shared.js"; + +/** + * `command args` with `input` on stdin, answered as { code, stdout, stderr }. + * `detach` marks a call whose started work outlives it and is not waited for. + */ +export function capture(command, args, { input = undefined, timeout = 60_000 } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] }); + const out = [], err = []; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`${command} ${args.slice(0, 2).join(" ")} did not end within ${timeout / 1000} s`)); + }, timeout); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => err.push(chunk)); + child.on("error", (error) => { clearTimeout(timer); reject(error); }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout: Buffer.concat(out).toString(), stderr: Buffer.concat(err).toString() }); + }); + if (input === undefined) child.stdin.end(); else child.stdin.end(input); + }); +} + +/** One call whose stdout, trimmed, is the command's answer. */ +export const answer = (command, args, options = {}) => + run(command, args, { timeout: 30_000, ...options }).then((result) => result.stdout.trim()); + +/** A guest file's text, or null when it does not exist. */ +export async function readFile(command, name, path, exec) { + const { code, stdout } = await exec(name, ["sh", "-c", "test -f \"$1\" && cat \"$1\"", "sh", path]); + return code === 0 ? stdout : null; +} + +/** Writes a guest file from stdin: the text never appears in a command line. */ +export async function writeFile(command, name, path, text, exec) { + const { code, stderr } = await exec(name, ["sh", "-c", "umask 077 && mkdir -p \"$(dirname \"$1\")\" && cat > \"$1\" && chmod 600 \"$1\"", "sh", path], { input: text }); + if (code !== 0) throw new Error(`could not write ${path} in ${name}: ${stderr.trim()}`); +} diff --git a/api/runtime/src/platform/vm/index.js b/api/runtime/src/platform/vm/index.js new file mode 100644 index 0000000000..6cf05cf905 --- /dev/null +++ b/api/runtime/src/platform/vm/index.js @@ -0,0 +1,453 @@ +// `vm:` placement (#515 §6, V9): **each of this home's projects — +// strictly, each Store — lives in one Linux VM of its own** (V1). That VM holds +// that project's Store on its own ext4 disk, shares that disk with the project's +// Sandboxes' daemons (Linux namespaces inside it, `LAYERFS_SANDBOX_BACKEND= +// namespaces`, H36), and runs its own worker with its own secret and its own +// private loopback channel (`worker/server.js`, one attempt per call, no replay). +// The host reaches only each VM's worker. +// +// This module is the platform object the runtime uses instead of the host's. +// Every step names the project, so the VM, its Store disk, its host port, its +// bundle copy and its worker secret are that project's alone: +// ① ensure the project's VM exists and runs, with its Store disk attached; +// ② copy the sealed bundle in, once per bundle hash and per VM; +// ③ start the worker as root with its secret, and block user namespaces +// (V4: `user.max_user_namespaces=0`); the namespaces backend refuses a +// host where that is not set, so step ③ also proves it; +// ④ open the private stream, `-p 127.0.0.1::`, and write the +// endpoint `useWorker` reads for that project; +// ⑤ stop the VM, or delete it: a delete never destroys a Store that was not +// pushed (V8), and there is no remote Store push yet (#506). +// The provider module behind it (`microsandbox.js`, `apple-container.js`) knows +// the platform; this file knows LayerFS's side and nothing platform-specific. +// Guest paths are in settings.js (GUEST): the home is a bind mount of the +// host's, the Store disk is mounted at the Store directory itself, and the +// bundle, Init sources, worker state and Sandbox directories live on the VM's +// root disk. +import { createHash, randomBytes } from "node:crypto"; +import { createServer } from "node:net"; +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { connection } from "@layerfs/agent/internal"; +import { endpointFile, homeNames, ping, ready, refuse, writeEndpoint } from "../shared.js"; +import { callHome, GUEST } from "../../settings.js"; +import { NAME, record as projectRecord } from "../../project/list.js"; +import microsandbox from "./microsandbox.js"; +import appleContainer from "./apple-container.js"; +import ssh from "./ssh.js"; + +const PROVIDERS = { microsandbox, "apple-container": appleContainer, ssh }; +const WORKER_PORT = 7070; +const SERVER = "api/runtime/src/worker/server.js"; +/** Bundle hashing in the worker comes first, so readiness takes a while. */ +const READY_MS = 60_000; + +const providerOf = (settings) => { + const provider = PROVIDERS[settings.vm.provider]; + if (!provider) throw refuse("Unsupported", `no VM provider is implemented for vm:${settings.vm.provider}`, null); + return provider; +}; + +/** The project's VM and its Store disk: one VM per Store means one of each per project. */ +const vmNameOf = (settings, project) => `${settings.containerPrefix}${project}-vm`; +const storeNameOf = (settings, project) => `${settings.containerPrefix}${project}-vm-store`; + +/** A project name a VM can be named after and its driver paths hold. */ +function checked(project) { + if (typeof project !== "string" || !NAME.test(project)) throw refuse("InvalidInput", "a project name is required: each project's Store runs in its own VM", null); + return project; +} + +/** The project's Store file inside its own VM: the disk is the Store's directory. */ +const storeFile = () => `${GUEST.store}/global/store.sqlite`; + +/** A host loopback port this process just freed: a VM's published worker port. */ +const freePort = () => new Promise((resolve, reject) => { + const server = createServer(); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); +}); + +/** + * Step ①: the project's VM runs, or it is created once from the home's settings. + * A record under our name that serves another home, or does not match what this + * project needs, is refused: nothing is replaced, restarted or repaired on a + * guess. + */ +async function ensure(settings, project) { + const provider = providerOf(settings), name = vmNameOf(settings, project), storeName = storeNameOf(settings, project); + const current = await provider.inspect(name); + if (current) { + if (current.labels?.["layerfs.home"] && current.labels["layerfs.home"] !== settings.home) { + throw refuse("Busy", `the VM ${name} serves the LayerFS home ${current.labels["layerfs.home"]}; give this home its own containerPrefix`, null); + } + if (current.store && current.store !== storeName) throw refuse("Failed", `the VM ${name} has the Store volume ${current.store}, not ${storeName}; delete it before using this project`, null); + if (current.home && current.home !== settings.home) throw refuse("Failed", `the VM ${name} mounts ${current.home} as its home, not ${settings.home}; delete it before using this project`, null); + const port = current.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (current.state === "running") { + if (!port) throw refuse("Failed", `the VM ${name} publishes no host port for ${WORKER_PORT}; delete it and start again`, null); + return { ...current, name, port }; + } + if (current.state !== "stopped") throw refuse("Failed", `the VM ${name} is ${current.state}; stop or delete it before using this project`, null); + await provider.start(name); + const started = await provider.inspect(name); + if (started?.state !== "running") throw refuse("Failed", `the VM ${name} did not start (${started?.state ?? "missing"})`, null); + const again = started.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (!again) throw refuse("Failed", `the VM ${name} publishes no host port for ${WORKER_PORT}; delete it and start again`, null); + return { ...started, name, port: again }; + } + if (provider.ensureStore) await provider.ensureStore(storeName, settings.vm.storeSize); + const hostPort = await freePort(); + const created = await provider.create({ + ...settings.vm, + name, image: settings.vm.image, cpus: settings.vm.cpus, memory: settings.vm.memory, + storeName, storeSize: settings.vm.storeSize, home: settings.home, hostPort, workerPort: WORKER_PORT, + }); + if (created?.state !== "running") throw refuse("Failed", `the VM ${name} was created but is ${created?.state ?? "missing"}; its record is kept for inspection, not replayed`, null); + const port = created.ports?.find((entry) => entry.guest === WORKER_PORT)?.host ?? null; + if (!port) throw refuse("Failed", `the VM ${name} was created without its port forward; delete it before using this project`, null); + return { ...created, name, port }; +} + +/** The sealed bundle's identity: its manifest hash, which the worker checks file by file. */ +function bundleIdentity(bundle) { + if (!bundle || !existsSync(join(bundle, "manifest.json"))) throw refuse("InvalidInput", "a VM placement needs a sealed Linux bundle (settings.bundle)", null); + const text = readFileSync(join(bundle, "manifest.json"), "utf8"); + const manifest = JSON.parse(text); + if (manifest.status !== "sealed" || manifest.platform !== "linux") throw refuse("InvalidInput", "a VM placement needs a sealed Linux bundle", null); + return { build: manifest.build_id, hash: createHash("sha256").update(text).digest("hex"), image: bundleImage(bundle) }; +} + +/** + * The Sandbox image a sealed bundle carries (#515 §4.7 V7, the H38 namespaces + * backend's convention): `/images/<64 hex>.tar`, an uncompressed + * `docker save` archive named by the image's config digest. It is copied in + * with the rest of the bundle — each VM has its own copy, because each VM is + * one Store — and the worker is told about it as a guest path. The namespaces + * backend requires an image, so a bundle of none leaves its Sandboxes to refuse + * the start (`deploy.image`/`deploy.imageArchive` are not configured). + */ +function bundleImage(bundle) { + const folder = join(bundle, "images"); + const found = existsSync(folder) ? readdirSync(folder).filter((name) => /^[0-9a-f]{64}\.tar$/.test(name)).sort() : []; + if (found.length > 1) throw refuse("InvalidInput", `the bundle holds ${found.length} Sandbox images (${found.join(", ")}); a VM placement describes one deployment image`, null); + if (!found.length) return null; + return { digest: `sha256:${found[0].slice(0, -4)}`, archive: `${GUEST.bundle}/images/${found[0]}` }; +} + +/** Step ②: the bundle, copied into this VM once per hash; a changed bundle replaces the copy. */ +async function install(settings, provider, vm) { + const { build, hash } = bundleIdentity(settings.bundle); + const marker = await provider.read(vm.name, `${GUEST.bundle}/.sealed`); + if (marker?.trim() === hash) return { build, copied: false }; + const { code, stderr } = await provider.exec(vm.name, ["rm", "-rf", GUEST.bundle]); + if (code !== 0) throw refuse("Failed", `could not replace ${GUEST.bundle} in ${vm.name}: ${stderr.trim()}`, null); + await provider.copy(vm.name, settings.bundle, GUEST.bundle); + await provider.write(vm.name, `${GUEST.bundle}/.sealed`, `${hash}\n`); + return { build, copied: true }; +} + +/** + * Step ③, the guest's precondition for the namespace backend: the Sandbox + * directory (`GUEST.local`, on the Store disk) and the VM-wide user-namespace + * block (V4). The block is read back — the namespaces backend refuses a host + * without it — so a VM that cannot block them is reported here, by fact. + */ +async function prepareGuest(provider, vm) { + // the namespaces backend wants a root owned by this user (root), not + // group/other writable, and traversable by others (0755): the nonroot + // command identity enters it + const made = await provider.exec(vm.name, ["sh", "-c", "umask 022 && mkdir -p \"$1\" && chmod 755 \"$1\"", "sh", GUEST.local]); + if (made.code !== 0) throw refuse("Failed", `could not make ${GUEST.local} in ${vm.name}: ${made.stderr.trim()}`, null); + const { code, stderr } = await provider.blockUserNamespaces(vm.name); + if (code !== 0) throw refuse("Unsupported", `the VM ${vm.name} cannot block user namespaces (${stderr.trim() || `exit ${code}`}); a Sandbox command could copy the Workspace mount into its own`, null); + const left = (await provider.read(vm.name, "/proc/sys/user/max_user_namespaces"))?.trim(); + if (left !== "0") throw refuse("Unsupported", `the VM ${vm.name} still allows user namespaces (${left ?? "unreadable"})`, null); +} + +/** The worker's own settings, read by `sh` from a 0600 file so the secret is in no command line. */ +const workerEnv = (secret, image) => [ + `LAYERFS_WORKER_SECRET=${secret}`, + `LAYERFS_WORKER_LISTEN=0.0.0.0:${WORKER_PORT}`, + `LAYERFS_HOME=${GUEST.home}`, + `LAYERFS_BUNDLE=${GUEST.bundle}`, + `LAYERFS_NATIVE=${GUEST.bundle}/layerfs.node`, + // a Sandbox reopened from its record has no daemon path to look beside + `LAYERFS_SANDBOX_HELPER=${GUEST.bundle}/layerfs-sandbox-helper`, + // #515 §4.7 V7 (H38's backend): this VM's own copy of the Sandbox image + ...(image ? [`LAYERFS_IMAGE=${image.digest}`, `LAYERFS_IMAGE_ARCHIVE=${image.archive}`] : []), + "", +].join("\n"); + +/** + * Step ③ and ④: the worker as root, its secret in a 0600 file, then the + * project's endpoint once it answers on the private port. A worker left by an + * earlier start that no longer answers is ended by the pid that start wrote (a + * bounded cleanup, not a replay); one launch, one readiness wait. A worker that + * does not answer reports the last lines of its log. + */ +async function launch(settings, provider, vm, project, bundle) { + const secret = randomBytes(32).toString("hex"); + const file = endpointFile(settings.home, project); + await provider.write(vm.name, `${GUEST.state}/worker.env`, workerEnv(secret, bundle.image)); + const script = [ + // one bounded cleanup: a worker an earlier start left is ended, and its + // port is waited for, so this start's worker can take it + `if [ -f ${GUEST.state}/worker.pid ]; then kill "$(cat ${GUEST.state}/worker.pid)" 2>/dev/null;`, + `for i in $(seq 1 50); do kill -0 "$(cat ${GUEST.state}/worker.pid)" 2>/dev/null || break; sleep 0.1; done; fi;`, + `set -a; . ${GUEST.state}/worker.env; set +a;`, + `nohup node ${GUEST.bundle}/${SERVER} >> ${GUEST.state}/worker.log 2>&1 &`, + `echo $! > ${GUEST.state}/worker.pid`, + ].join(" "); + const started = await provider.exec(vm.name, ["sh", "-c", script]); + if (started.code !== 0) throw refuse("Failed", `the worker did not start in ${vm.name}: ${(started.stderr || started.stdout).trim()}`, null); + const info = { home: GUEST.home, port: vm.port, secret, build: bundle.build, vm: { provider: settings.vm.provider, name: vm.name, project } }; + try { + await ready(info, READY_MS, `the worker in ${vm.name}`); + } catch (error) { + const log = (await provider.read(vm.name, `${GUEST.state}/worker.log`).catch(() => null))?.trim().split("\n").slice(-8).join("\n"); + throw refuse("Failed", `${error.message}${log ? `; its log ends: ${log}` : ""}`, null); + } + mkdirSync(join(settings.home, "worker"), { recursive: true, mode: 0o700 }); + writeEndpoint(file, info); + return info; +} + +/** The endpoint the project's worker was last written at, or null. */ +function endpoint(settings, project) { + try { return JSON.parse(readFileSync(endpointFile(settings.home, project), "utf8")); } catch { return null; } +} + +/** Nothing to start when the project's worker already answers with this build. */ +async function answering(settings, project, expected) { + const info = endpoint(settings, project); + if (!info || (expected && info.build !== expected)) return null; + return (await ping(info))?.ready ? info : null; +} + +/** One project's VM record and, when it runs, whether its worker answers. It starts nothing. `disk` + * asks each running VM for its Store disk's free bytes (one guest call), which `status` reports. */ +async function entry(settings, project, disk = false) { + const provider = providerOf(settings), name = vmNameOf(settings, project); + let found; + try { found = await provider.inspect(name); } catch (error) { + return { project, name, state: "unknown", error: error.message, worker: { state: "unknown", error: error.message } }; + } + if (!found) return { project, name, state: "missing", worker: { state: "missing" } }; + const vm = { project, name, state: found.state, image: found.image, home: found.home, store: found.store, created: found.created }; + if (found.state !== "running") return { ...vm, worker: { state: found.state, build: null } }; + const info = endpoint(settings, project); + const reply = info ? await ping(info) : null; + const worker = { state: reply?.ready ? "running" : "unknown", answering: !!reply?.ready, build: reply?.build ?? info?.build ?? null, port: info?.port ?? null }; + if (!disk) return { ...vm, worker }; + return { ...vm, worker, disk: { free: await provider.storeFree(name, GUEST.store).catch(() => null) } }; +} + +/** `{ project, worker, vm }` of one project: what `worker.start` and `vm.stop` answer. */ +async function stateOf(settings, project) { + const vm = await entry(settings, checked(project)); + return { project: vm.project, worker: vm.worker, vm }; +} + +/** The runtime's `worker.start(project)`: steps ①–④ for that project's VM. */ +async function start(settings, project) { + checked(project); + const provider = providerOf(settings), bundle = bundleIdentity(settings.bundle); + const held = await answering(settings, project, bundle.build); + if (held) return stateOf(settings, project); + const vm = await ensure(settings, project); + await install(settings, provider, vm); + await prepareGuest(provider, vm); + await launch(settings, provider, vm, project, bundle); + return stateOf(settings, project); +} + +/** One `provider restart`, then the steps a rebooted VM needs again (③, ④). */ +async function restart(settings, project) { + checked(project); + const provider = providerOf(settings), bundle = bundleIdentity(settings.bundle); + const vm = await ensure(settings, project); + if (provider.restart) await provider.restart(vm.name); + else { await provider.stop(vm.name); await provider.start(vm.name); } + rmSync(endpointFile(settings.home, project), { force: true }); + await install(settings, provider, vm); + await prepareGuest(provider, vm); + await launch(settings, provider, vm, project, bundle); + return stateOf(settings, project); +} + +/** Every project's VM, or all of this home's: `status()` of the runtime and of the `vm` group. */ +async function status(settings, projects = []) { + const provider = providerOf(settings); + const vms = []; + for (const project of projects) vms.push(await entry(settings, checked(project), true)); + return { worker: null, vm: { provider: settings.vm.provider, placement: settings.placement.name, vms } }; +} + +/** Step ⑤: stop one project's VM; its Store disk and its Store stay. */ +async function stop(settings, project) { + const provider = providerOf(settings), name = vmNameOf(settings, checked(project)); + const found = await provider.inspect(name); + if (!found) return { project, name, state: "missing" }; + if (found.state !== "stopped") await provider.stop(name); + const after = await provider.inspect(name); + rmSync(endpointFile(settings.home, project), { force: true }); + return { project, name, state: after?.state ?? "missing" }; +} + +/** + * Step ⑤, delete (V8): one project's VM record and, only with `discardStore`, + * its Store disk. A delete that would leave a project without its only Store + * copy is refused: that project's Store lies on this VM's disk and nowhere + * else, and no remote Store push exists (#506). + */ +async function remove(settings, project, options = {}) { + const provider = providerOf(settings), name = vmNameOf(settings, checked(project)); + if (options?.confirm !== "delete") throw refuse("InvalidInput", `a VM delete ends the project's VM; confirm it with { confirm: "delete" }`, null); + const found = await provider.inspect(name); + if (!found) return { project, name, state: "missing", store: "kept" }; + if (exists(settings, project)) { + throw refuse("Unsupported", `the Store of ${project} is on ${name}'s disk and nowhere else; a VM delete never destroys a Store that was not pushed, and this build has no remote Store push (#506). Remove the project first`, null); + } + await provider.remove(name); + if (options.discardStore) await provider.removeStore(storeNameOf(settings, project)); + rmSync(endpointFile(settings.home, project), { force: true }); + return { project, name, state: "deleted", store: options.discardStore ? "discarded" : "kept" }; +} + +/** Whether this home still holds the project: its record, or its entry in the home. */ +function exists(settings, project) { + if (projectRecord(settings, project)) return true; + try { + return readdirSync(settings.home, { withFileTypes: true }).some((e) => e.isDirectory() && e.name === project && existsSync(join(settings.home, e.name, "project.record"))); + } catch { return false; } +} + +/** The four hooks Init and project removal use instead of host Docker (§4.2 B2, B9). */ +const stores = { + /** + * Init examines the Store's directory before it installs into it (the kernel + * guard checks its filesystem), so the project's Store disk is mounted and + * its directory made here, as a Docker volume is on a local placement. + */ + async makeStore(runtime, project) { + const provider = providerOf(runtime.settings), name = vmNameOf(runtime.settings, project.name); + const { code, stderr } = await provider.exec(name, ["sh", "-c", "umask 022 && mkdir -p \"$1\" && chmod 755 \"$1\"", "sh", GUEST.store]); + if (code !== 0) throw refuse("Failed", `could not make ${GUEST.store} in ${name}: ${stderr.trim()}`, null); + }, + /** The Sandboxes holding the project's Store now: its running Sandboxes, by number. A stopped VM runs none. */ + async storeHolders(runtime, project) { + const { settings } = runtime, provider = providerOf(settings); + const found = await provider.inspect(vmNameOf(settings, project.name)); + if (!found || found.state !== "running") return []; + const held = await runtime.clients.use(project, (client) => client.project.list_sandboxes(1000, 0)); + return held.items.map(({ sandbox }) => ({ id: sandbox, state: "running" })); + }, + /** Stops them the way the SDK does: `sandbox.close`, one per Sandbox. */ + async releaseStoreHolders(runtime, project, holders) { + const { settings } = runtime, provider = providerOf(settings); + const found = await provider.inspect(vmNameOf(settings, project.name)); + if (!found || found.state !== "running") return; + await runtime.clients.use(project, async (client) => { + for (const holder of holders) await client.sandbox.close(holder.id); + }); + }, + /** Removes the project's VM and its Store disk: no Store copy is left anywhere. */ + async removeStore(runtime, project) { + const { settings } = runtime, provider = providerOf(settings), name = vmNameOf(settings, project.name); + const found = await provider.inspect(name); + if (found) await provider.remove(name); + await provider.removeStore(storeNameOf(settings, project.name)); + rmSync(endpointFile(settings.home, project.name), { force: true }); + }, +}; + +/** The guest home, and the deployment inside the project's VM: native Sandboxes, the namespace backend. */ +export function deploymentEnv(settings, project) { + const image = settings.bundle ? bundleImage(settings.bundle) : null; + return { + HOME: callHome(settings), + SANDBOX: "native", + SANDBOX_BACKEND: process.env.LAYERFS_SANDBOX_BACKEND || "namespaces", + DAEMON: GUEST.daemon, + LOCAL_DIR: GUEST.local, + // the Sandbox image this VM's bundle carries, when it carries one (V7): + // the digest and, for the namespaces backend, the archive in this VM's + // own copy of the bundle that holds it (both guest paths) + ...(image ? { IMAGE: image.digest, IMAGE_ARCHIVE: image.archive } : {}), + STORE_PATH: storeFile(), + CONTAINER_PREFIX: settings.containerPrefix, + // a process an integration launches for this project reaches this VM's worker + WORKER_ENDPOINT: endpointFile(settings.home, project.name), + }; +} + +/** + * The platform object of a `vm:` home. `hostDocker`/`hostSandboxes` + * tell the runtime that host Docker is not this home's: its Stores are + * directories on each VM's own disk and its Sandboxes are Linux namespaces + * there, so `volume.list`, `volume.remove`, `sandbox.list`, + * `sandbox.list_leftovers` and `sandbox.clean_leftovers` refuse with that + * reason instead of answering from the host, and `sandbox.sweep` answers + * `{ host: false }`. `endpoints` answers, per project, the endpoint file + * `@layerfs/agent` sends that project's calls to. + */ +export function vmPlatform(settingName) { + if (!PROVIDERS[settingName]) throw refuse("Unsupported", `no VM provider is implemented for vm:${settingName}`, null); + return { + name: `vm:${settingName}`, + hostDocker: false, + hostSandboxes: false, + endpoints: (settings) => (project) => endpointFile(settings.home, project), + status, + start, + restart, + /** Init runs in that project's worker; the source folder is copied into its VM first. */ + async init(settings, project) { + await start(settings, project.name); + const provider = providerOf(settings), name = vmNameOf(settings, project.name); + const target = importPath(project.source); + const { code, stderr } = await provider.exec(name, ["rm", "-rf", target]); + if (code !== 0) throw refuse("Failed", `could not clear ${target} in ${name}: ${stderr.trim()}`, null); + await provider.copy(name, project.source, target); + }, + /** Where the SDK inside the VM reads a host folder: its copy under `/layerfs/import`. */ + path: (settings, hostPath) => (hostPath === settings.home || hostPath.startsWith(`${settings.home}/`) ? hostPath.replace(settings.home, GUEST.home) : importPath(hostPath)), + deploymentEnv, + docker: () => { + throw refuse("Unsupported", `this home is placed in vm:${settingName}: its Stores and Sandboxes are inside each project's VM, so host Docker does not describe it`, null); + }, + storePath: () => storeFile(), + homeNames: (settings) => homeNames(settings.home), + async removeHome(settings, name) { + rmSync(join(settings.home, name), { recursive: true, force: true }); + }, + /** Before a Store is deleted, the project's worker closes the Store it holds open. */ + async releaseStore(settings, project) { + const info = endpoint(settings, project.name); + if (!info) return; + const answer = await connection(info).call({ fn: "releaseStore", args: { home: GUEST.home, storePath: storeFile() } }); + if (answer.refused) throw refuse(answer.refused.code, answer.refused.message, answer.refused.detail); + }, + /** Free bytes of every project VM's Store disk, as that VM sees them. */ + async free(settings) { + const provider = providerOf(settings); + let total = null; + for (const project of homeNames(settings.home)) { + const free = await provider.storeFree(vmNameOf(settings, project), GUEST.store).catch(() => null); + if (free !== null) total = (total ?? 0) + free; + } + return total; + }, + ...stores, + vm: { status, stop, delete: remove }, + }; +} + +/** A host folder's copy inside the VM, named by its path so the mapping needs no state. */ +export function importPath(hostPath) { + return `${GUEST.import}/${createHash("sha256").update(hostPath).digest("hex").slice(0, 16)}`; +} diff --git a/api/runtime/src/platform/vm/microsandbox.js b/api/runtime/src/platform/vm/microsandbox.js new file mode 100644 index 0000000000..38cd030fc5 --- /dev/null +++ b/api/runtime/src/platform/vm/microsandbox.js @@ -0,0 +1,115 @@ +// Microsandbox (`msb` 0.7.8, libkrun microVMs): the VM is one sandbox record +// and the Store is a named `disk` volume that survives stop and start (V8). +// #512 read this provider's facts; the gaps it named are handled here: +// - the name is the only identity: every read goes through `msb inspect` by +// name, and a record that is not ours (another home) is refused, never +// reused; +// - a failed create can leave hidden state under the name, so `create` is +// called once and its failure is reported as it is (the message says what +// `msb` suggests); nothing is replayed or replaced on a guess; +// - `msb exec -u` capability drop is unverified, so the worker, the helper +// and the daemon run as root (the guest's only identity) and the +// namespaces backend drops each command itself (#515 §4.3). +// One CLI call per call, one attempt, no retry. +import { existsSync, rmSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { answer, capture, readFile, writeFile } from "./cli.js"; + +const CLI = "msb"; +const CREATE_MS = 120_000, CALL_MS = 30_000, EXEC_MS = 60_000; + +const call = (args, options = {}) => answer(CLI, args, { timeout: CALL_MS, ...options }); + +/** + * One `msb exec` in the VM. `--no-tty` keeps the streams plain (stdout and + * stderr arrive separately). The worker detaches itself with a shell, so this + * call ends when its shell does. + */ +const exec = (name, argv, { input = undefined, timeout = EXEC_MS } = {}) => + capture(CLI, ["exec", name, "--no-tty", "--", ...argv], { input, timeout }); + +/** The VM's record as `msb inspect --format json` gives it, or null when it does not exist. */ +export async function inspect(name) { + let text; + try { + text = await call(["inspect", "--format", "json", name]); + } catch (error) { + const said = `${error.stderr ?? ""}${error.message ?? ""}`; + if (/sandbox not found|no such sandbox/i.test(said)) return null; + throw error; + } + const record = JSON.parse(text); + const config = record.active_config ?? record.config ?? {}; + const mounted = (guest) => (config.mounts ?? []).find((mount) => mount.guest === guest) ?? null; + return { + state: String(record.status ?? "").toLowerCase(), + image: config.image?.Oci?.reference ?? null, + digest: config.image?.Oci?.manifest_digest ?? null, + labels: config.labels ?? {}, + ports: (config.network?.ports ?? []).map((port) => ({ guest: port.guest_port, host: port.host_port, bind: port.host_bind })), + home: mounted("/layerfs/home")?.host ?? null, + store: mounted("/layerfs/store")?.name ?? null, + created: record.created_at ?? null, + }; +} + +/** + * Step ①: the VM, from `config` when it does not exist yet. Answers the record + * read back after the call, so a create that ended without a usable VM is + * reported by the caller from fact, not from the command's exit code. + */ +export async function create(config) { + const dir = join(homedir(), ".microsandbox", "sandboxes", config.name); + const existedBefore = existsSync(dir); + try { + await call([ + "create", "--name", config.name, "--cpus", String(config.cpus), "--memory", config.memory, + "--mount-named", `${config.storeName}:/layerfs/store:kind=disk,size=${config.storeSize}`, + "-v", `${config.home}:/layerfs/home`, + "-p", `127.0.0.1:${config.hostPort}:${config.workerPort}`, + "--label", "layerfs.role=vm", "--label", `layerfs.home=${config.home}`, + config.image, + ], { timeout: CREATE_MS }); + return inspect(config.name); + } catch (error) { + if (!existedBefore && existsSync(dir)) { + rmSync(dir, { recursive: true, force: true }); + } + throw error; + } +} + +export const start = (name) => call(["start", name], { timeout: CREATE_MS }); +export const stop = (name) => call(["stop", name]); +export const restart = (name) => call(["restart", name], { timeout: CREATE_MS }); +/** The VM record only: its Store volume is a separate object and is never removed here. */ +export const remove = (name) => call(["remove", "--force", name], { timeout: CREATE_MS }); +export const removeStore = (storeName) => call(["volume", "remove", storeName]); + +/** + * V4: the VM-wide user-namespace block. Microsandbox's guest keeps + * `/proc/sys` writable; the caller reads the sysctl back, so a block that did + * not take is reported, not assumed. + */ +export const blockUserNamespaces = (name) => exec(name, ["sh", "-c", "echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the VM's Store disk, or null when `df` cannot answer. */ +export async function storeFree(name) { + const { code, stdout } = await exec(name, ["df", "-kP", "/layerfs/store"]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +/** Copies a host folder's contents into a guest folder. */ +export const copy = (name, source, destination) => call(["copy", source, `${name}:${destination}`], { timeout: CREATE_MS }); + +export const read = (name, path) => readFile(CLI, name, path, exec); +export const write = (name, path, text) => writeFile(CLI, name, path, text, exec); + +export default { + name: "microsandbox", + defaultImage: "node:22-trixie-slim", + inspect, create, start, stop, restart, remove, removeStore, blockUserNamespaces, storeFree, read, write, copy, exec, +}; diff --git a/api/runtime/src/platform/vm/ssh.js b/api/runtime/src/platform/vm/ssh.js new file mode 100644 index 0000000000..1008956867 --- /dev/null +++ b/api/runtime/src/platform/vm/ssh.js @@ -0,0 +1,395 @@ +// `vm:ssh` placement (#515 §6, V3): any Linux host reachable over SSH — an EC2 +// instance or a VM the user made, instead of one this runtime creates. The host +// and its Store disk are the user's: this adapter never makes a disk, it only +// proves that the path given is on ext4, on that host's own kernel. +// +// The five adapter steps (§6) on this provider: +// ① the "VM" is the host; the Store disk is the path `storePath` (a block +// device mounted at /layerfs/store, or a directory bind-mounted there); +// ② the bundle is copied in over SSH (one `tar` stream per bundle hash); +// ③ the worker runs as root with its secret, exactly as on a microVM; +// ④ the private stream is `ssh -L 127.0.0.1::127.0.0.1:7070`; +// ⑤ stop ends the forward and the worker; delete never touches the disk. +// +// Facts this file depends on, and why they are checked rather than assumed: +// - the host must answer `uname -s` with Linux: a macOS/BSD host has no +// namespaces and no /proc/sys/user; +// - the Store path must be an ext4 filesystem on that host: LayerFS's Store +// names its files and relies on that filesystem's ordering; +// - key login only, from a file; a password never reaches argv or a log; +// - host key checking stays on (`StrictHostKeyChecking=yes` unless the +// setting says otherwise, `accept-new` at the loosest); +// - `-o ExitOnForwardFailure=yes` so a busy local port fails at once, before +// anything is uploaded or started, instead of leaving a half-made VM. +// +// The forward runs with its stderr in a log file, not a pipe: a host process +// (the CLI, a test) must be able to exit while the user's VM keeps running, and +// a referenced stdio pipe would hold its event loop open. `unref` on the child +// alone does not release the pipes, so they are files. +import { spawn } from "node:child_process"; +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { LayerFSError } from "@layerfs/agent"; +import { readFile, writeFile } from "./cli.js"; + +const EXEC_MS = 60_000, COPY_MS = 120_000, FORWARD_READY_MS = 600, END_MS = 3_000; +const WORKER_PORT_DEFAULT = 7070; +const STORE_PATH_DEFAULT = "/layerfs/store"; + +const SESSIONS = new Map(); + +const refuse = (code, message) => new LayerFSError(code, message, null); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Where this provider keeps its session records: one JSON file per VM name. */ +export const stateDir = () => process.env.LAYERFS_SSH_STATE_DIR || join(homedir(), ".layerfs", "ssh"); + +function sessionFile(name) { + const dir = stateDir(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return join(dir, `${name}.json`); +} + +function forwardLog(name) { + const dir = stateDir(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return join(dir, `${name}-forward.log`); +} + +function loadSession(name) { + if (SESSIONS.has(name)) return SESSIONS.get(name); + const file = sessionFile(name); + if (!existsSync(file)) return null; + try { + const session = JSON.parse(readFileSync(file, "utf8")); + SESSIONS.set(name, session); + return session; + } catch { + return null; + } +} + +function saveSession(session) { + SESSIONS.set(session.name, session); + writeFileSync(sessionFile(session.name), JSON.stringify(session, null, 2), { mode: 0o600 }); +} + +function deleteSession(name) { + SESSIONS.delete(name); + rmSync(sessionFile(name), { force: true }); + rmSync(forwardLog(name), { force: true }); +} + +/** Whether a pid still runs. Only a plain liveness probe; it proves nothing else. */ +export function isAlive(pid) { + if (!pid || typeof pid !== "number") return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error.code === "EPERM"; + } +} + +/** One argv token for the remote POSIX shell: never left to the shell's word splitting. */ +export function shellEscape(arg) { + const text = String(arg); + if (/^[A-Za-z0-9_\-./=:@%+,]+$/.test(text)) return text; + return `'${text.replace(/'/g, "'\\''")}'`; +} + +/** + * The ssh arguments both `exec` and the forward share. `BatchMode=yes` keeps a + * password prompt from ever being tried (this provider does not support one); + * the identity file is a path here, its content stays in the file. + */ +function sshArgs(session) { + const args = ["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"]; + if (session.port && session.port !== 22) args.push("-p", String(session.port)); + if (session.identityFile) args.push("-i", session.identityFile); + const strict = session.strictHostKeyChecking; + args.push("-o", `StrictHostKeyChecking=${strict === false ? "accept-new" : typeof strict === "string" ? strict : "yes"}`); + if (session.knownHostsFile) args.push("-o", `UserKnownHostsFile=${session.knownHostsFile}`); + if (session.controlMaster && session.controlPath) { + args.push("-o", "ControlMaster=auto", "-o", `ControlPath=${session.controlPath}`, "-o", "ControlPersist=60s"); + } + return args; +} + +const destinationOf = (session) => (session.user ? `${session.user}@${session.host}` : session.host); + +/** One command on the host, its streams kept apart; a call past `timeout` is killed and reported. */ +function remote(session, argv, { input = undefined, timeout = EXEC_MS } = {}) { + const args = [...sshArgs(session), destinationOf(session), "--", ...argv.map(shellEscape)]; + return new Promise((resolve, reject) => { + const child = spawn("ssh", args, { stdio: ["pipe", "pipe", "pipe"] }); + const out = [], err = []; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`ssh ${argv.join(" ")} did not end within ${timeout / 1000} s`)); + }, timeout); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => err.push(chunk)); + child.on("error", (error) => { clearTimeout(timer); reject(error); }); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout: Buffer.concat(out).toString(), stderr: Buffer.concat(err).toString() }); + }); + if (input === undefined) child.stdin.end(); else child.stdin.end(input); + }); +} + +/** The last `lines` lines of the forward's log, for an error message. */ +function logTail(name, lines = 3) { + try { + return readFileSync(forwardLog(name), "utf8").trim().split("\n").slice(-lines).join("; "); + } catch { + return ""; + } +} + +/** + * Step ④: the private stream. `-o ExitOnForwardFailure=yes` makes a busy local + * port an immediate failure, so nothing is uploaded or started on a VM that + * cannot be reached. A forward that exits within the readiness window is + * reported with its own reason; one that answers will be ended by `endForward`. + */ +async function startForward(session, hostPort, workerPort) { + const log = forwardLog(session.name); + const fd = openSync(log, "w", 0o600); + const child = spawn("ssh", [ + "-N", + "-o", "ExitOnForwardFailure=yes", + ...sshArgs(session), + "-L", `127.0.0.1:${hostPort}:127.0.0.1:${workerPort}`, + destinationOf(session), + ], { stdio: ["ignore", "ignore", fd] }); + closeSync(fd); + + const ended = new Promise((resolve) => child.once("exit", (code, signal) => resolve({ code, signal }))); + const first = await Promise.race([ended.then((value) => ({ ended: true, ...value })), delay(FORWARD_READY_MS).then(() => ({ ended: false }))]); + child.removeAllListeners("exit"); + if (first.ended) { + child.unref(); + throw new Error(`ssh -L 127.0.0.1:${hostPort} exited (${first.code ?? first.signal}): ${logTail(session.name) || "no reason given"}`); + } + child.on("exit", () => { SESSIONS.delete(session.name); }); + child.unref(); + return child; +} + +/** Ends a forward, bounded: SIGTERM, then SIGKILL; never waits on it indefinitely. */ +async function endForward(session) { + const pid = session.forwardPid; + if (!isAlive(pid)) { session.forwardPid = null; return; } + try { process.kill(pid, "SIGTERM"); } catch { /* already gone */ } + for (let waited = 0; waited < END_MS && isAlive(pid); waited += 50) await delay(50); + if (isAlive(pid)) { + try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } + for (let waited = 0; waited < 1_000 && isAlive(pid); waited += 50) await delay(50); + } + session.forwardPid = null; +} + +/** The record as the runtime reads it: a forward that no longer runs is a stopped VM. */ +export async function inspect(name) { + const session = loadSession(name); + if (!session) return null; + const running = isAlive(session.forwardPid); + return { + state: running ? "running" : "stopped", + image: null, + digest: null, + labels: session.labels ?? {}, + ports: [{ guest: session.workerPort, host: session.hostPort, bind: "127.0.0.1" }], + home: session.home ?? null, + store: session.store ?? null, + created: session.created ?? null, + }; +} + +/** + * One host serves one Store: the Store path is a single directory on a disk the + * user owns, and two projects naming it would write one Store file between + * them. This is the provider's own rule, checked here because the runtime's + * per-project VM names would otherwise hide the collision. + */ +function sharedWith(name, host, storePath) { + for (const file of existsSync(stateDir()) ? readdirSync(stateDir()).filter((entry) => entry.endsWith(".json")) : []) { + if (file === `${name}.json`) continue; + try { + const other = JSON.parse(readFileSync(join(stateDir(), file), "utf8")); + if (other.host === host && other.storePath === storePath) return other.name; + } catch { /* a record that cannot be read is not one of ours to judge */ } + } + return null; +} + +/** + * Step ①–④: the host, its Store path, and the private stream. Everything that + * can refuse is checked before a session is written, so a create that fails + * leaves no session, no forward and nothing uploaded. + */ +export async function create(config) { + const given = config.ssh ?? {}; + const host = String(config.host ?? given.host ?? "").trim(); + if (!host) throw refuse("InvalidInput", "vm:ssh needs the host to reach: settings.vm.host, or LAYERFS_SSH_HOST"); + if (config.password || given.password) throw refuse("InvalidInput", "vm:ssh does not take a password; give a key file as identityFile"); + + const session = { + name: config.name, + host, + user: String(config.user ?? given.user ?? "root").trim(), + port: Number(config.port ?? given.port ?? 22), + identityFile: config.identityFile ?? given.identityFile ?? null, + strictHostKeyChecking: config.strictHostKeyChecking ?? given.strictHostKeyChecking ?? "yes", + knownHostsFile: config.knownHostsFile ?? given.knownHostsFile ?? null, + controlMaster: config.controlMaster ?? given.controlMaster ?? false, + storePath: String(config.storePath ?? given.storePath ?? STORE_PATH_DEFAULT).trim(), + hostPort: config.hostPort, + workerPort: config.workerPort ?? WORKER_PORT_DEFAULT, + home: config.home, + store: config.storeName, + labels: { "layerfs.home": config.home, "layerfs.role": "vm" }, + created: new Date().toISOString(), + }; + if (session.identityFile && !existsSync(session.identityFile)) { + throw refuse("InvalidInput", `the SSH identity file does not exist: ${session.identityFile}`); + } + const shared = sharedWith(session.name, session.host, session.storePath); + if (shared) { + throw refuse("Busy", `${session.host}:${session.storePath} already holds the Store of ${shared}; one host serves one Store, so give this project its own host or disk`, null); + } + + // the host, by fact: a Linux kernel, and the Store path on ext4 + const kernel = await remote(session, ["uname", "-s"]).catch((error) => { throw refuse("Failed", `could not reach ${session.host} over SSH: ${error.message}`); }); + if (kernel.code !== 0) throw refuse("Failed", `SSH to ${session.host} failed (exit ${kernel.code}): ${kernel.stderr.trim()}`); + if (kernel.stdout.trim() !== "Linux") throw refuse("Unsupported", `${session.host} answers ${kernel.stdout.trim() || "nothing"}, not Linux; vm:ssh needs a Linux host`); + + const exists = await remote(session, ["sh", "-c", "test -e \"$1\"", "sh", session.storePath]); + if (exists.code !== 0) throw refuse("InvalidInput", `the Store path ${session.storePath} does not exist on ${session.host}; vm:ssh uses a disk the user provides`); + const block = (await remote(session, ["sh", "-c", "test -b \"$1\"", "sh", session.storePath])).code === 0; + const type = block + ? (await remote(session, ["sh", "-c", "blkid -s TYPE -o value \"$1\" 2>/dev/null || lsblk -no FSTYPE \"$1\"", "sh", session.storePath])).stdout.trim() + : (await remote(session, ["findmnt", "-T", session.storePath, "-o", "FSTYPE", "-n"])).stdout.trim(); + if (type !== "ext4") { + throw refuse("Unsupported", `the Store path ${session.storePath} on ${session.host} is ${type || "an unreadable filesystem"}, not ext4; LayerFS requires an ext4 Store disk`); + } + + // the guest's own view: /layerfs/store is that path (mounted, or bind-mounted) + const attached = await remote(session, ["sh", "-c", + "if [ \"$1\" = /layerfs/store ]; then exit 0; fi; mkdir -p /layerfs/store && mountpoint -q /layerfs/store || { if [ -b \"$1\" ]; then mount -t ext4 \"$1\" /layerfs/store; else mount --bind \"$1\" /layerfs/store; fi; }", + "sh", session.storePath]); + if (attached.code !== 0) throw refuse("Failed", `could not attach ${session.storePath} at /layerfs/store on ${session.host}: ${attached.stderr.trim()}`); + + try { + const forward = await startForward(session, session.hostPort, session.workerPort); + session.forwardPid = forward.pid; + } catch (error) { + throw refuse("Failed", `could not open the private stream to ${session.host}: ${error.message}`); + } + saveSession(session); + return inspect(session.name); +} + +/** A stopped VM is started again by re-opening its stream on the port it kept. */ +export async function start(name) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + if (isAlive(session.forwardPid)) return inspect(name); + const forward = await startForward(session, session.hostPort, session.workerPort); + session.forwardPid = forward.pid; + saveSession(session); + return inspect(name); +} + +/** Step ⑤, stop: the stream and the worker end; the Store disk and the session record stay. */ +export async function stop(name) { + const session = loadSession(name); + if (!session) return; + await endForward(session); + await remote(session, ["sh", "-c", "if [ -f /layerfs/state/worker.pid ]; then kill \"$(cat /layerfs/state/worker.pid)\" 2>/dev/null; fi"]).catch(() => {}); + saveSession(session); +} + +export const restart = async (name) => { + await stop(name); + return start(name); +}; + +/** + * Step ⑤, delete: the record, the forward and what this adapter put on the host. + * The Store disk is the user's and is never unmounted, formatted or removed; a + * delete while the project still holds its Store is refused by the runtime. + */ +export async function remove(name) { + const session = loadSession(name); + if (!session) return; + await endForward(session); + await remote(session, ["sh", "-c", + "if [ -f /layerfs/state/worker.pid ]; then kill \"$(cat /layerfs/state/worker.pid)\" 2>/dev/null; fi; rm -rf /layerfs/state /layerfs/bundle"]).catch(() => {}); + deleteSession(name); +} + +/** The Store disk is the user's: this adapter has nothing of its own to delete. */ +export const removeStore = async () => {}; + +/** V4: the host-wide user-namespace block; the caller reads it back. */ +export const blockUserNamespaces = (name) => + exec(name, ["sh", "-c", "echo 0 > /proc/sys/user/max_user_namespaces"]); + +/** Free bytes of the Store disk, as the host sees them. */ +export async function storeFree(name, guestStorePath = STORE_PATH_DEFAULT) { + const { code, stdout } = await exec(name, ["df", "-kP", guestStorePath]); + if (code !== 0) return null; + const kb = Number(stdout.trim().split("\n").pop().trim().split(/\s+/)[3]); + return Number.isFinite(kb) ? kb * 1024 : null; +} + +export function exec(name, argv, options = {}) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + return remote(session, argv, options); +} + +/** + * Step ②: the bundle (or any folder) as one `tar` stream, unpacked on the host. + * Both ends are killed if the transfer passes its budget, so no copy outlives + * its call. + */ +export function copy(name, source, destination) { + const session = loadSession(name); + if (!session) throw refuse("Failed", `there is no vm:ssh session called ${name}`); + const script = `mkdir -p ${shellEscape(destination)} && tar -C ${shellEscape(destination)} -xf -`; + return new Promise((resolve, reject) => { + const pack = spawn("tar", ["-C", source, "-cf", "-", "."], { stdio: ["ignore", "pipe", "pipe"] }); + const unpack = spawn("ssh", [...sshArgs(session), destinationOf(session), "--", script], { stdio: ["pipe", "pipe", "pipe"] }); + let err = "", tarErr = ""; + const timer = setTimeout(() => { + pack.kill("SIGKILL"); + unpack.kill("SIGKILL"); + reject(new Error(`copy into ${name}:${destination} did not end within ${COPY_MS / 1000} s`)); + }, COPY_MS); + unpack.stderr.on("data", (chunk) => { err += chunk.toString(); }); + pack.stderr.on("data", (chunk) => { tarErr += chunk.toString(); }); + pack.on("error", (error) => { clearTimeout(timer); reject(error); }); + unpack.on("error", (error) => { clearTimeout(timer); reject(error); }); + unpack.on("close", (code) => { + clearTimeout(timer); + if (code === 0) resolve(); + else reject(new Error(`copy into ${name}:${destination} failed (exit ${code}): ${(err || tarErr).trim()}`)); + }); + pack.stdout.pipe(unpack.stdin); + }); +} + +export const read = (name, path) => readFile("ssh", name, path, exec); +export const write = (name, path, text) => writeFile("ssh", name, path, text, exec); + +export default { + name: "ssh", + defaultImage: null, + inspect, create, start, stop, restart, remove, removeStore, + blockUserNamespaces, storeFree, read, write, copy, exec, isAlive, +}; diff --git a/api/runtime/src/platform/windows/index.js b/api/runtime/src/platform/windows/index.js index 6351f9ed2c..8bd4b693ce 100644 --- a/api/runtime/src/platform/windows/index.js +++ b/api/runtime/src/platform/windows/index.js @@ -11,7 +11,7 @@ import { createWriteStream, mkdirSync, readFileSync, rmSync } from "node:fs"; import { join, posix } from "node:path"; import { fileURLToPath } from "node:url"; import { connection, frame, readFrames } from "@layerfs/agent/internal"; -import { endpointFile, ping, ready, refuse, run, writeEndpoint } from "../shared.js"; +import { endpointFile, ping, ready, refuse, run, writeEndpoint, makeStore, releaseStoreHolders, removeStore, storeHolders } from "../shared.js"; import { absoluteLinux, projectHome, wsl, wslArgs } from "./paths.js"; const SERVER = "api/runtime/src/worker/server.js"; @@ -131,6 +131,10 @@ export default { async init(settings) { await start(settings); }, path: (settings, hostPath) => hostPath, docker, + makeStore, + storeHolders, + releaseStoreHolders, + removeStore, storePath: (settings, volume) => `/var/lib/docker/volumes/${volume}/_data/global/store.sqlite`, async homeNames(settings) { const { stdout } = await wsl(settings, ["find", settings.wsl.home, "-mindepth", "2", "-maxdepth", "2", "-name", "project.record"], { timeout: 5000 }).catch(() => ({ stdout: "" })); @@ -139,7 +143,8 @@ export default { // rm gets one checked child directory as an argv item, never shell text async removeHome(settings, name) { await wsl(settings, ["rm", "-rf", "--", projectHome(settings.wsl, name)], { timeout: 10_000 }); }, /** Before a volume is deleted, the worker closes its cached Store of it. */ - async releaseStore(settings, volume) { + async releaseStore(settings, project) { + const { volume } = project; if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(volume)) throw refuse("InvalidInput", "invalid Store volume name"); const info = JSON.parse(readFileSync(endpointFile(settings.home), "utf8")); const answer = await connection(info).call({ fn: "releaseStore", args: { home: settings.wsl.home, storePath: `/var/lib/docker/volumes/${volume}/_data/global/store.sqlite` } }); diff --git a/api/runtime/src/project/init.js b/api/runtime/src/project/init.js index f11645958c..f9899c0769 100644 --- a/api/runtime/src/project/init.js +++ b/api/runtime/src/project/init.js @@ -80,9 +80,10 @@ export async function runInit(runtime, prepared, report = () => {}) { report({ step: "scan" }); const totals = await scanSource(project.source, (counts) => report({ scanned: counts })); report({ step: "volume", totals }); - await platform.init(settings); - // its owner, for runtime.volume.list: Docker sets a volume's labels only when it creates it - await platform.docker(settings, ["volume", "create", "--label", "layerfs.role=store", "--label", `layerfs.home=${settings.home}`, "--label", `layerfs.project=${project.name}`, project.volume], { timeout: 30_000 }); + await platform.init(settings, project); + // its owner, for runtime.volume.list on a local placement (Docker sets a + // volume's labels only when it creates it); a VM's Store is a directory + await platform.makeStore(runtime, project); report({ step: "import" }); let client; try { @@ -96,7 +97,7 @@ export async function runInit(runtime, prepared, report = () => {}) { throw error; } if (error?.code !== "Exists") await platform.removeHome(settings, project.name).catch(() => {}); - await platform.docker(settings, ["volume", "rm", project.volume]).catch(() => {}); + await platform.removeStore(runtime, project).catch(() => {}); throw error; } await client.close(); diff --git a/api/runtime/src/project/remove.js b/api/runtime/src/project/remove.js index 036c099819..914880234d 100644 --- a/api/runtime/src/project/remove.js +++ b/api/runtime/src/project/remove.js @@ -5,12 +5,13 @@ // containers on the volume, left by failed starts and idle Sandboxes, go // with it. The volume is the runtime's record's, else `volume`. The home // entry is the project, so deleting it frees the name at once. One attempt -// each, no retry; a failed step leaves the rest in place. +// each, no retry; a failed step leaves the rest in place. The four Store hooks +// are the platform's: Docker volumes and containers on a local placement, the +// VM's own directory and namespace Sandboxes under `vm:`. import { LayerFSError } from "@layerfs/agent"; +import { STOPPED } from "../platform/shared.js"; import { NAME, dropRecord, record } from "./list.js"; -/** Container states that hold the volume but run nothing. */ -const STOPPED = new Set(["created", "exited", "dead"]); const refuse = (code, message) => new LayerFSError(code, message, null); export async function remove(runtime, name, options = {}) { @@ -21,13 +22,15 @@ export async function remove(runtime, name, options = {}) { if (typeof volume !== "string" || !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(volume)) throw refuse("InvalidInput", `the runtime has no record of ${name}'s volume; name it as { volume }`); if (!record(settings, name) && !(await platform.homeNames(settings)).includes(name)) throw refuse("NoSuchProject", `no LayerFS project named ${name}`); await runtime.clients.forget(name); - const rows = await platform.docker(settings, ["ps", "-a", "--filter", `volume=${volume}`, "--format", "{{.ID}} {{.State}}"]); - const containers = rows.split("\n").filter(Boolean).map((line) => line.split(" ")); - const users = containers.filter(([, state]) => !STOPPED.has(state)).length; + // the hooks name the project as well as its volume: a `vm:` placement's Store is + // the project's own VM and disk, a local placement's is a Docker volume + const project = { name, volume }; + const holders = await platform.storeHolders(runtime, project); + const users = holders.filter((holder) => !STOPPED.has(holder.state)).length; if (users && !options.stopSandboxes) throw refuse("Busy", `${users} running container(s) use ${volume}; a Sandbox closes 2 minutes after its last Workspace is unmounted, or pass stopSandboxes`); - if (containers.length) await platform.docker(settings, ["rm", ...(users ? ["-f"] : []), ...containers.map(([id]) => id)], { timeout: 30_000 }); - await platform.releaseStore(settings, volume); - await platform.docker(settings, ["volume", "rm", volume], { timeout: 30_000 }); + await platform.releaseStoreHolders(runtime, project, holders, { force: users > 0 }); + await platform.releaseStore(settings, project); + await platform.removeStore(runtime, project); await platform.removeHome(settings, name); dropRecord(settings, name); return { kind: "Removed", project: name, volume }; diff --git a/api/runtime/src/sandbox/leftovers.js b/api/runtime/src/sandbox/leftovers.js index b4ca7eec7c..caf5528565 100644 --- a/api/runtime/src/sandbox/leftovers.js +++ b/api/runtime/src/sandbox/leftovers.js @@ -1,10 +1,12 @@ // Sandbox containers of this home that no project owns, or that were left // stopped, and their removal. From part of the dsh Host's runtime.js (#428). import { LayerFSError } from "@layerfs/agent"; +import { noHostDocker } from "../platform/shared.js"; /** This home's Sandbox containers (`-`, label layerfs.role=sandbox), running or not. */ export async function containers(runtime) { const { settings, platform } = runtime; + if (platform.hostSandboxes === false) throw noHostDocker(runtime, "runtime.sandbox", "its Sandboxes are Linux namespaces inside the VM; the SDK's project.list_sandboxes reads them"); const out = await platform.docker(settings, ["ps", "-a", "--no-trunc", "--filter", "label=layerfs.role=sandbox", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Label \"layerfs.project\"}}\t{{.Label \"layerfs.sandbox\"}}\t{{.Image}}"]); return out.split("\n").filter(Boolean).map((row) => { const [name, state, status, project, sandbox, image] = row.split("\t"); diff --git a/api/runtime/src/sandbox/list.js b/api/runtime/src/sandbox/list.js index f1b5c710c8..5316b78f65 100644 --- a/api/runtime/src/sandbox/list.js +++ b/api/runtime/src/sandbox/list.js @@ -5,6 +5,7 @@ // there). It goes only when the container is deleted (sandbox.close, an idle // close followed by a reclaim, or runtime.sandbox.sweep). One `docker ps` // and one `docker inspect --size`; it changes nothing. +import { noHostDocker } from "../platform/shared.js"; /** A container gone between the listing and the inspection is left out: inspect still prints the others. */ async function inspect(runtime, ids) { @@ -21,6 +22,7 @@ async function inspect(runtime, ids) { /** `[{ name, project, sandbox, state, running, startedAt, finishedAt, image, ports, scratch, owned }]`, by name. */ export async function list(runtime) { const { settings, platform } = runtime; + if (platform.hostSandboxes === false) throw noHostDocker(runtime, "runtime.sandbox.list", "its Sandboxes are Linux namespaces inside the VM; the SDK's project.list_sandboxes reads them"); const ids = (await platform.docker(settings, ["ps", "-a", "-q", "--no-trunc", "--filter", "label=layerfs.role=sandbox"])).split("\n").filter(Boolean); if (!ids.length) return []; const known = new Set((await runtime.project.list()).map((p) => p.name)); diff --git a/api/runtime/src/sandbox/sweep.js b/api/runtime/src/sandbox/sweep.js index bf1e0d4161..32b3a469a5 100644 --- a/api/runtime/src/sandbox/sweep.js +++ b/api/runtime/src/sandbox/sweep.js @@ -7,6 +7,8 @@ // project of this home records stay for list_leftovers / clean_leftovers. // The runtime runs it once when it opens (settings.sweep, default true); // status().sweep reports that run. One attempt per project, no retry. +// A `vm:` home keeps no host containers, so its sweep reads none +// and says so: `{ reclaimed: {}, failed: [], host: false }`. import { containers } from "./leftovers.js"; /** States of a container whose daemon has ended and that nothing restarts. */ @@ -15,6 +17,7 @@ const ENDED = new Set(["exited", "dead"]); /** `{ reclaimed: { project: [ids] }, failed: [{ project, code, message }] }` */ export async function sweep(runtime) { const { settings } = runtime; + if (runtime.platform.hostSandboxes === false) return { reclaimed: {}, failed: [], host: false }; const known = new Set((await runtime.project.list()).filter((p) => p.volume).map((p) => p.name)); const ended = (await containers(runtime)).filter((c) => ENDED.has(c.state) && known.has(c.project) && c.name.startsWith(`${settings.containerPrefix}${c.project}-`)); const reclaimed = {}, failed = []; diff --git a/api/runtime/src/settings.js b/api/runtime/src/settings.js index 070816dac8..b1ce588cb9 100644 --- a/api/runtime/src/settings.js +++ b/api/runtime/src/settings.js @@ -9,6 +9,59 @@ import { LayerFSError } from "@layerfs/agent"; const PLATFORMS = { linux: "linux", darwin: "macos", win32: "windows" }; const DEFAULT_DOCKER = "/var/run/docker.sock"; +/** Where a home's Store and Sandboxes live: this machine (`local`, today) or one Linux VM per home (V9). */ +const PROVIDERS = ["microsandbox", "apple-container", "ssh"]; +/** The VM's own settings: the image that holds node and the guest base, and its size. */ +const VM = { image: "node:22-trixie-slim", cpus: 2, memory: "2G", storeSize: "20G" }; +/** Where the VM sees the home, the Store disk, the bundle and its working state (§5, §6). */ +export const GUEST = Object.freeze({ + home: "/layerfs/home", + store: "/layerfs/store", + bundle: "/layerfs/bundle", + import: "/layerfs/import", + state: "/layerfs/state", + // The Sandbox directory is on the Store disk, beside the Store's own + // `global/` folder, not on the VM's root: a namespace Sandbox's root is an + // overlay whose writable layer must be on a local filesystem, and a + // provider's own root may be an overlay itself (Microsandbox's is), while + // the Store disk is an ext4 disk on both providers (H38's + // `require_overlay_upper`). Its `parent` is what the backend reads, so the + // Store's own directory `/layerfs/store/global` stays outside it. + local: "/layerfs/store/local", + daemon: "/layerfs/bundle/layerfs-daemon", +}); + +/** + * `placement`: absent or `local` (this machine's Docker, macOS worker container + * or WSL2 worker; behavior unchanged), or `vm:` (one Linux VM per + * home, §6). A VM provider name that is not implemented is refused here, not + * later, so a typo cannot look like `local`. + */ +function placement(given) { + const text = given ?? process.env.LAYERFS_PLACEMENT ?? "local"; + if (typeof text !== "string" || !text) throw new LayerFSError("InvalidInput", "placement must be `local` or `vm:`", null); + if (text === "local") return { kind: "local", name: "local" }; + const provider = /^vm:(.+)$/.exec(text)?.[1]; + if (!provider) throw new LayerFSError("InvalidInput", `placement is \`local\` or \`vm:\`, not ${text}`, null); + if (!PROVIDERS.includes(provider)) throw new LayerFSError("Unsupported", `no VM provider is implemented for ${text}; it is vm:${PROVIDERS.join(" or vm:")}`, null); + return { kind: "vm", name: text, provider }; +} + +/** The VM's size and image, from `settings.vm` else `LAYERFS_VM_*` else the defaults. */ +function vmSettings(given = {}) { + const NAMES = { image: "LAYERFS_VM_IMAGE", cpus: "LAYERFS_VM_CPUS", memory: "LAYERFS_VM_MEMORY", storeSize: "LAYERFS_VM_STORE_SIZE" }; + const read = (name) => given[name] ?? (process.env[NAMES[name]] || undefined); + const image = read("image") ?? VM.image; + if (typeof image !== "string" || !/^[^\s]+$/.test(image)) throw new LayerFSError("InvalidInput", "the VM image is an image reference, such as node:22-trixie-slim", null); + const cpus = Number(read("cpus") ?? VM.cpus); + if (!Number.isSafeInteger(cpus) || cpus < 1) throw new LayerFSError("InvalidInput", "the VM's cpus is a whole number of at least 1", null); + const size = (name, fallback) => { + const value = read(name) ?? fallback; + if (typeof value !== "string" || !/^[1-9][0-9]{0,5}[MG]$/.test(value)) throw new LayerFSError("InvalidInput", `the VM's ${name} is a size such as 2G or 512M`, null); + return value; + }; + return { image, cpus, memory: size("memory", VM.memory), storeSize: size("storeSize", VM.storeSize) }; +} /** * The Docker socket when none is given: the engine's usual path, or on macOS @@ -35,6 +88,10 @@ function defaultDocker(platform) { * @param {string} [given.containerPrefix] this home's container names (default `layerfs-`) * @param {string} [given.stateDir] the runtime's project records (default /.runtime) * @param {object} [given.wsl] Windows: { distribution, user, node, home } + * @param {string} [given.placement] `local` (default) or `vm:`; also read from + * LAYERFS_PLACEMENT when not given (V9). A VM holds the home's Stores and Sandboxes. + * @param {object} [given.vm] VM placement: { image, cpus, memory, storeSize } (also + * LAYERFS_VM_IMAGE, LAYERFS_VM_CPUS, LAYERFS_VM_MEMORY, LAYERFS_VM_STORE_SIZE) * @param {string} [given.platform] linux | macos | windows (default this process's) * @param {boolean} [given.sweep] sweep this home's stopped Sandboxes once at open (default true) */ @@ -44,12 +101,15 @@ export function resolveSettings(given = {}) { const platform = given.platform ?? PLATFORMS[process.platform]; if (!Object.values(PLATFORMS).includes(platform)) throw new LayerFSError("Unsupported", `LayerFS runs on Linux, macOS and Windows, not ${process.platform}`, null); const bundle = given.bundle || null; + const where = placement(given.placement); const settings = { platform, + placement: where, home: given.home, image: given.image || null, bundle, - daemon: given.daemon || (bundle ? (platform === "windows" ? posix.join(bundle, "layerfs-daemon") : join(bundle, "layerfs-daemon")) : null), + // a VM runs the bundle's daemon beside its helper, at the guest's path + daemon: where.kind === "vm" ? GUEST.daemon : given.daemon || (bundle ? (platform === "windows" ? posix.join(bundle, "layerfs-daemon") : join(bundle, "layerfs-daemon")) : null), docker: given.docker || defaultDocker(platform), volumes: given.volumes || "/var/lib/docker/volumes", traceDir: given.traceDir || join(given.home, "traces"), @@ -58,6 +118,34 @@ export function resolveSettings(given = {}) { pollMs: given.pollMs ?? 400, sweep: given.sweep !== false, }; + if (where.kind === "vm") { + settings.vm = { ...vmSettings(given.vm), ...where, home: GUEST.home, store: GUEST.store }; + if (where.provider === "ssh") { + const sshGiven = given.ssh ?? given.vm ?? {}; + if (sshGiven.password) throw new LayerFSError("InvalidInput", "vm:ssh does not support password authentication; use an identityFile instead", null); + const host = sshGiven.host ?? process.env.LAYERFS_SSH_HOST ?? null; + const port = Number(sshGiven.port ?? process.env.LAYERFS_SSH_PORT ?? 22); + if (!Number.isSafeInteger(port) || port < 1 || port > 65535) throw new LayerFSError("InvalidInput", "the SSH port must be a number between 1 and 65535", null); + const identityFile = sshGiven.identityFile ?? sshGiven.identity ?? process.env.LAYERFS_SSH_IDENTITY ?? process.env.LAYERFS_SSH_KEY ?? null; + if (identityFile && typeof identityFile !== "string") throw new LayerFSError("InvalidInput", "the SSH identityFile must be a file path", null); + const storePath = sshGiven.storePath ?? sshGiven.store ?? process.env.LAYERFS_SSH_STORE ?? GUEST.store; + settings.vm.ssh = { + host: host ? String(host).trim() : null, + user: String(sshGiven.user ?? process.env.LAYERFS_SSH_USER ?? "root").trim(), + port, + identityFile: identityFile ? String(identityFile).trim() : null, + storePath: String(storePath).trim(), + strictHostKeyChecking: sshGiven.strictHostKeyChecking ?? true, + knownHostsFile: sshGiven.knownHostsFile ?? process.env.LAYERFS_SSH_KNOWN_HOSTS ?? null, + controlMaster: sshGiven.controlMaster ?? false, + }; + settings.vm.host = settings.vm.ssh.host; + settings.vm.user = settings.vm.ssh.user; + settings.vm.port = settings.vm.ssh.port; + settings.vm.identityFile = settings.vm.ssh.identityFile; + settings.vm.storePath = settings.vm.ssh.storePath; + } + } if (platform === "windows") { const hash = createHash("sha256").update(given.home).digest("hex").slice(0, 16); settings.wsl = { @@ -74,10 +162,15 @@ export function resolveSettings(given = {}) { return settings; } -/** Settings Init and Workspaces need that are not given. macOS and Windows need a bundle too. */ -export const missing = (settings) => [ - ...["image", "daemon", ...(settings.platform === "linux" ? [] : ["bundle"])].filter((name) => !settings[name]), -]; +/** Settings Init and Workspaces need that are not given. macOS, Windows and a VM need a bundle too. */ +export const missing = (settings) => + [ + ...(settings.placement.kind === "vm" ? [] : ["image"]), + "daemon", + ...(settings.platform === "linux" && settings.placement.kind !== "vm" ? [] : ["bundle"]), + // a host this home cannot reach is a missing setting, listed before a start + ...(settings.placement.provider === "ssh" ? ["host"] : []), + ].filter((name) => (name === "host" ? !settings.vm?.host : !settings[name])); -/** The LayerFS home as LayerFS calls see it (Windows: the WSL one). */ -export const callHome = (settings) => settings.wsl?.home ?? settings.home; +/** The LayerFS home as LayerFS calls see it (Windows: the WSL one; a VM: the guest's). */ +export const callHome = (settings) => settings.vm?.home ?? settings.wsl?.home ?? settings.home; diff --git a/api/runtime/src/status.js b/api/runtime/src/status.js index 45ba3b79d4..ed4ace61e3 100644 --- a/api/runtime/src/status.js +++ b/api/runtime/src/status.js @@ -1,23 +1,34 @@ // runtime.status(): Docker reachability, the worker's state (null on Linux), // free disk and missing settings. From part of the dsh Host's runtime.js -// (#428). It starts nothing. +// (#428). It starts nothing. A `vm:` home has no host Docker in its +// data path (#515 §3): its Stores and Sandboxes are inside the VM, so `docker` +// says so instead of reporting the host engine down, and `vm` carries the VM's +// own state. import { statfsSync } from "node:fs"; import { missing } from "./settings.js"; export async function status(runtime) { const { settings, platform } = runtime; let docker; - try { docker = { level: "ok", version: await platform.docker(settings, ["version", "--format", "{{.Server.Version}}"]) }; } catch (error) { - docker = { level: "down", error: error.stderr?.trim() || error.message }; + if (platform.hostDocker === false) { + docker = { level: "not-used", detail: `this home is placed in ${platform.name}; its Stores and Sandboxes are inside each project's VM` }; + } else { + try { docker = { level: "ok", version: await platform.docker(settings, ["version", "--format", "{{.Server.Version}}"]) }; } catch (error) { + docker = { level: "down", error: error.stderr?.trim() || error.message }; + } } - const { worker } = await platform.status(settings).catch((error) => ({ worker: { state: "unknown", error: error.message } })); + // a `vm:` placement has one VM per project, so it is asked about each project + const projects = await runtime.project.list().then((all) => all.map((project) => project.name), () => []); + const reported = await platform.status(settings, projects).catch((error) => ({ worker: { state: "unknown", error: error.message } })); let home = null; try { const s = statfsSync(settings.home); home = s.bavail * s.bsize; } catch { /* unreadable */ } return { - platform: platform.name, + platform: settings.platform, + placement: settings.placement.name, missing: missing(settings), docker, - worker, - disk: { home, volumes: docker.level === "ok" ? await platform.free(settings) : null }, + worker: reported.worker ?? null, + vm: reported.vm ?? null, + disk: { home, volumes: docker.level === "not-used" || docker.level === "ok" ? await platform.free(settings) : null }, }; } diff --git a/api/runtime/src/volume.js b/api/runtime/src/volume.js index dc45d9b142..370922a79c 100644 --- a/api/runtime/src/volume.js +++ b/api/runtime/src/volume.js @@ -14,6 +14,7 @@ // One `docker system df -v`, plus one `docker volume inspect` of the // labelled volumes; listing changes nothing. import { LayerFSError } from "@layerfs/agent"; +import { noHostDocker } from "./platform/shared.js"; const UNITS = { B: 1, kB: 1e3, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12 }; /** "84.05MB" -> 84050000; anything else -> null. */ @@ -25,6 +26,7 @@ export const bytesOf = (text) => { /** `[{ name, size, containers, owner, project, live, removable }]`, by name. */ export async function list(runtime) { const { settings, platform } = runtime; + if (platform.hostDocker === false) throw noHostDocker(runtime, "runtime.volume.list", "its Stores are directories on the VM's disk, not host Docker volumes"); const df = JSON.parse(await platform.docker(settings, ["system", "df", "-v", "--format", "{{json .}}"], { timeout: 60_000 })); const volumes = df.Volumes ?? []; const labelled = volumes.filter((v) => v.Labels).map((v) => v.Name); diff --git a/api/runtime/test/fixture/fake-container.cjs b/api/runtime/test/fixture/fake-container.cjs new file mode 100644 index 0000000000..d61636842e --- /dev/null +++ b/api/runtime/test/fixture/fake-container.cjs @@ -0,0 +1,176 @@ +// The `container` stand-in for @layerfs/runtime's Apple Container test: no VM. +// It answers the calls `platform/vm/apple-container.js` makes **in the shape the +// real CLI prints**, copied from `container` 1.6.1 (build aa8d76b) on macOS 27.2, +// 2026-10-31: `inspect --format json` is an array of +// `{ id, status: { state, startedDate, networks }, configuration: { image, +// labels, mounts, publishedPorts, creationDate } }`, mounts carry +// `type.volume.name`, and it needs `-i` on `exec` for a caller's stdin. What the +// stand-in cannot prove is the real CLI's behaviour under load; the real run is +// `api/runtime/test/verify/vm_real.mjs apple-container`. +// +// container inspect --format json NAME +// container run --detach --name N … IMAGE sleep infinity +// container start|stop NAME, container delete --force NAME +// container volume inspect NAME | create -s SIZE NAME | delete NAME +// container copy HOST NAME:DEST +// container exec [-i] NAME sh -c SCRIPT sh ARGS… +// Records live in FAKE_CONTAINER_STATE/vms/.json, volumes in +// FAKE_CONTAINER_STATE/volumes/.json, guest paths under +// FAKE_CONTAINER_STATE/guest/. +const { appendFileSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_CONTAINER_STATE; +if (!state) { + console.error("fake container: FAKE_CONTAINER_STATE is not set"); + process.exit(2); +} +const recordFile = (name) => join(state, "vms", `${name}.json`); +const volumeFile = (name) => join(state, "volumes", `${name}.json`); +const guestIn = (name) => (path) => join(state, "guest", name, path.replace(/^\//, "")); +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const fail = (message) => { console.error(`error: ${message}`); process.exit(1); }; +const at = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); +const all = (flag) => argv.flatMap((value, index) => (value === flag ? [argv[index + 1]] : [])); +const read = (name) => (existsSync(recordFile(name)) ? JSON.parse(readFileSync(recordFile(name), "utf8")) : null); +const save = (value) => { mkdirSync(dirname(recordFile(value.name)), { recursive: true }); writeFileSync(recordFile(value.name), JSON.stringify(value, null, 1)); }; + +function endWorker(vm) { + if (!vm?.pid) return; + try { process.kill(vm.pid); } catch { /* already gone */ } + vm.pid = null; + save(vm); +} + +function launchWorker(vm) { + endWorker(vm); // a start ends the worker an earlier start left, as the launch script does + const guest = guestIn(vm.name); + const env = Object.fromEntries(readFileSync(guest("/layerfs/state/worker.env"), "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const child = spawn(process.execPath, [guest("/layerfs/bundle/api/runtime/src/worker/server.js")], { + env: { ...process.env, ...env, LAYERFS_HOME: guest(env.LAYERFS_HOME), LAYERFS_BUNDLE: guest(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${vm.port}`, LAYERFS_NATIVE: process.env.FAKE_CONTAINER_NATIVE }, + detached: true, stdio: ["ignore", "ignore", openSync(guest("/layerfs/state/worker.log"), "a")], + }); + child.unref(); + vm.pid = child.pid; + save(vm); + const pid = guest("/layerfs/state/worker.pid"); + mkdirSync(dirname(pid), { recursive: true }); + writeFileSync(pid, `${child.pid}\n`); +} + +const command = argv[0]; +if (command === "inspect") { + const name = argv.at(-1); + const vm = read(name) ?? fail(`container not found: ${name}`); + process.stdout.write(JSON.stringify([{ + id: vm.name, + status: { + state: vm.state, + startedDate: vm.started ?? vm.created, + networks: [{ network: "default", ipv4Address: "192.168.64.2/24", ipv4Gateway: "192.168.64.1", variant: "reserved" }], + }, + configuration: { + creationDate: vm.created, + image: { reference: vm.image, descriptor: { digest: `sha256:${"1".repeat(64)}` } }, + labels: vm.labels, + mounts: [ + { source: vm.home, destination: "/layerfs/home", type: { virtiofs: {} } }, + { source: join(state, "volumes", `${vm.store}.img`), destination: "/layerfs/store", type: { volume: { name: vm.store, format: "ext4" } } }, + ], + publishedPorts: [{ hostPort: vm.port, containerPort: 7070, hostAddress: "127.0.0.1", count: 1, proto: "tcp" }], + }, + }])); +} else if (command === "run") { + const name = at("--name"); + if (read(name)) fail(`container ${name} already exists`); + const mounts = all("--mount").map((value) => Object.fromEntries(value.split(",").map((pair) => pair.split("=")))); + const store = mounts.find((entry) => entry.target === "/layerfs/store")?.source ?? ""; + const home = mounts.find((entry) => entry.target === "/layerfs/home")?.source ?? ""; + // H39v's finding: `container run` makes a named volume it was given before it + // fails on a busy port, and the container record never appears + if (process.env.FAKE_CONTAINER_FAIL_RUN === "port") { + if (!existsSync(volumeFile(store))) { + mkdirSync(join(state, "volumes"), { recursive: true }); + writeFileSync(volumeFile(store), JSON.stringify({ name: store, size: "20G", format: "ext4", creationDate: new Date().toISOString(), labels: {} })); + } + fail(`failed to publish 127.0.0.1:${String(at("--publish") ?? "").split(":")[1]}:7070: port is already allocated`); + } + if (!existsSync(volumeFile(store))) fail(`volume not found: ${store}`); + const labels = Object.fromEntries(all("--label").map((pair) => pair.split("="))); + save({ + name, state: "running", created: new Date().toISOString(), started: new Date().toISOString(), pid: null, + labels, home, store, port: Number(String(at("--publish") ?? "").split(":")[1]), + image: argv[argv.length - 3], + }); +} else if (command === "start") { + const vm = read(argv[1]) ?? fail(`container not found: ${argv[1]}`); + vm.state = "running"; + vm.started = new Date().toISOString(); + save(vm); +} else if (command === "stop") { + const vm = read(argv[1]) ?? fail(`container not found: ${argv[1]}`); + endWorker(vm); + vm.state = "stopped"; + save(vm); +} else if (command === "delete") { + const vm = read(argv.at(-1)) ?? fail(`container not found: ${argv.at(-1)}`); + endWorker(vm); + rmSync(recordFile(vm.name), { force: true }); + rmSync(join(state, "guest", vm.name), { recursive: true, force: true }); +} else if (command === "volume") { + const [action, ...rest] = argv.slice(1); + const name = action === "create" ? rest.at(-1) : rest.at(-1); + if (action === "inspect") { + if (!existsSync(volumeFile(name))) fail(`volume not found: ${name}`); + process.stdout.write(JSON.stringify([{ id: name, configuration: JSON.parse(readFileSync(volumeFile(name), "utf8")) }])); + } else if (action === "create") { + if (existsSync(volumeFile(name))) fail(`volume ${name} already exists`); + mkdirSync(join(state, "volumes"), { recursive: true }); + const size = at("-s") ?? "549755813888"; + writeFileSync(volumeFile(name), JSON.stringify({ name, sizeInBytes: Number(String(size).replace(/[KMGTP]$/, "")), size, format: "ext4", creationDate: new Date().toISOString(), labels: {} })); + } else if (action === "delete") { + rmSync(volumeFile(name), { force: true }); + } +} else if (command === "copy") { + const target = argv[2]; + const guest = guestIn(target.slice(0, target.indexOf(":"))); + const destination = guest(target.slice(target.indexOf(":") + 1)); + mkdirSync(destination, { recursive: true }); + for (const entry of readdirSync(argv[1], { withFileTypes: true })) cpSync(join(argv[1], entry.name), join(destination, entry.name), { recursive: entry.isDirectory() }); +} else if (command === "exec") { + // `exec [-i] NAME argv…` + const interactive = argv.includes("-i"); + const rest = argv.slice(1).filter((value) => value !== "-i"); + const [name, ...command_] = rest; + const vm = read(name) ?? fail(`container not found: ${name}`); + const guest = guestIn(vm.name); + const put = (path, text) => { mkdirSync(dirname(guest(path)), { recursive: true }); writeFileSync(guest(path), text); }; + const [tool, flag, script, ...args] = command_; + if (tool === "rm") { + rmSync(guest(script), { recursive: true, force: true }); + } else if (tool === "df") { + const volume = JSON.parse(readFileSync(volumeFile(vm.store), "utf8")); + const blocks = Math.floor(volume.sizeInBytes / 1024); + process.stdout.write(`Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/vdc ${blocks} 1048576 ${blocks - 1048576} 1% /layerfs/store\n`); + } else if (tool === "sh" && flag === "-c") { + if (/test -f "\$1" && cat "\$1"/.test(script)) { + if (!existsSync(guest(args[1]))) process.exit(1); + process.stdout.write(readFileSync(guest(args[1]))); + } else if (/cat > "\$1"/.test(script)) { + if (!interactive) fail("stdin was not kept open: the adapter must pass -i"); + put(args[1], readFileSync(0)); + } else if (/umask 022/.test(script)) { + mkdirSync(guest(args[1]), { recursive: true, mode: 0o755 }); + } else if (/max_user_namespaces/.test(script)) { + if (!/remount,rw \/proc\/sys/.test(script)) fail("the guest mounts /proc/sys read-only: remount it first"); + put("/proc/sys/user/max_user_namespaces", "0\n"); + } else if (/nohup node/.test(script)) { + launchWorker(vm); + } + } +} else { + fail(`fake container does not know ${command}`); +} diff --git a/api/runtime/test/fixture/fake-msb.cjs b/api/runtime/test/fixture/fake-msb.cjs new file mode 100644 index 0000000000..aecf4a3d7b --- /dev/null +++ b/api/runtime/test/fixture/fake-msb.cjs @@ -0,0 +1,151 @@ +// The `msb` stand-in for @layerfs/runtime's VM tests (#515 §6): no microVM. +// It answers the calls `platform/vm/microsandbox.js` makes, keeps one record +// per VM (one project's Store = one VM), and maps each guest's absolute paths +// under FAKE_MSB_STATE/guest/, so the adapter's guest file reads and writes +// are real file operations and two VMs never share one. +// +// msb inspect --format json NAME the record, else "sandbox not found" +// msb create --name N … IMAGE record it as Running with the -p host port +// msb start|stop|restart NAME flip the record's state +// msb remove --force NAME forget it (and end its worker) +// msb volume remove NAME log it; nothing else happens +// msb copy HOST NAME:DEST copy HOST's contents into DEST +// msb exec NAME --no-tty -- sh -c SCRIPT sh ARGS… +// the scripts the adapter sends: a guest read, a guest write from stdin, +// the user-namespace block, `rm -rf PATH`, and the worker's launch (which +// starts the real worker/server.js of the bundle, with the stand-in +// binding, listening on the VM's published port). +// Every call is appended to FAKE_MSB_STATE/calls.jsonl as { at, argv }. +const { appendFileSync, cpSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { homedir } = require("node:os"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_MSB_STATE; +if (!state) { + console.error("fake msb: FAKE_MSB_STATE is not set"); + process.exit(2); +} +const vmRoot = (name) => join(state, "vms", `${name}.json`); +const guestIn = (name) => (path) => join(state, "guest", name, path.replace(/^\//, "")); +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const fail = (message) => { console.error(`error: ${message}`); process.exit(1); }; +const at = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); +const read = (name) => (existsSync(vmRoot(name)) ? JSON.parse(readFileSync(vmRoot(name), "utf8")) : null); +const save = (name, value) => { mkdirSync(dirname(vmRoot(name)), { recursive: true }); writeFileSync(vmRoot(name), JSON.stringify(value, null, 1)); }; + +function endWorker(vm) { + if (!vm?.pid) return; + try { process.kill(vm.pid); } catch { /* already gone */ } + vm.pid = null; + save(vm.name, vm); +} + +/** The worker the adapter launched: the bundle's own server.js, on the VM's published port. */ +function launchWorker(vm) { + endWorker(vm); // a start ends the worker an earlier start left, as the launch script does + const guest = guestIn(vm.name); + const env = Object.fromEntries(readFileSync(guest("/layerfs/state/worker.env"), "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const script = guest("/layerfs/bundle/api/runtime/src/worker/server.js"); + if (!existsSync(script)) fail("the bundle holds no worker/server.js"); + mkdirSync(dirname(guest("/layerfs/state/worker.log")), { recursive: true }); + const child = spawn(process.execPath, [script], { + env: { ...process.env, ...env, LAYERFS_HOME: guest(env.LAYERFS_HOME), LAYERFS_BUNDLE: guest(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${vm.port}`, LAYERFS_NATIVE: process.env.FAKE_MSB_NATIVE }, + detached: true, stdio: ["ignore", "ignore", openSync(guest("/layerfs/state/worker.log"), "a")], + }); + child.unref(); + vm.pid = child.pid; + save(vm.name, vm); + const pid = guest("/layerfs/state/worker.pid"); + mkdirSync(dirname(pid), { recursive: true }); + writeFileSync(pid, `${child.pid}\n`); +} + +const command = argv[0]; +if (command === "inspect") { + const vm = read(argv.at(-1)) ?? fail(`sandbox not found: ${argv.at(-1)}`); + process.stdout.write(JSON.stringify({ + name: vm.name, + status: vm.state === "running" ? "Running" : "Stopped", + created_at: vm.created, + active_config: { + image: { Oci: { reference: vm.image, manifest_digest: `sha256:${"0".repeat(64)}` } }, + labels: vm.labels, + mounts: [ + { guest: "/layerfs/home", host: vm.home, type: "Bind", options: { readonly: false } }, + { guest: "/layerfs/store", name: vm.store, type: "Named", options: { readonly: false } }, + ], + network: { ports: [{ guest_port: 7070, host_port: vm.port, host_bind: "127.0.0.1", protocol: "tcp" }] }, + }, + })); +} else if (command === "create") { + const name = at("--name"); + if (read(name)) fail(`sandbox already exists: sandbox '${name}' already exists; remove it, start the stopped sandbox, or recreate with .replace()`); + // H39v's finding: a create that fails on its mounts leaves the sandbox's own + // directory behind, invisible to `msb list` and uncleaned by `msb remove` + if (process.env.FAKE_MSB_FAIL_CREATE === "mount") { + mkdirSync(join(homedir(), ".microsandbox", "sandboxes", name), { recursive: true }); + fail("the host path for one of the mounts does not exist"); + } + const labels = Object.fromEntries(argv.flatMap((value, index) => (value === "--label" ? [argv[index + 1].split("=")] : []))); + const published = String(at("-p") ?? "").split(":"); + save(name, { + name, state: "running", created: new Date().toISOString(), image: argv.at(-1), pid: null, + labels, home: String(at("-v") ?? "").split(":")[0], store: String(at("--mount-named") ?? "").split(":")[0], + port: Number(published[1]), + }); +} else if (command === "start" || command === "restart") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + vm.state = "running"; + save(vm.name, vm); +} else if (command === "stop") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + endWorker(vm); + vm.state = "stopped"; + save(vm.name, vm); +} else if (command === "remove") { + const vm = read(argv.at(-1)) ?? fail(`sandbox not found: ${argv.at(-1)}`); + endWorker(vm); + rmSync(vmRoot(vm.name), { force: true }); + rmSync(join(state, "guest", vm.name), { recursive: true, force: true }); +} else if (command === "volume") { + // `volume remove NAME`: nothing is stored for volumes here +} else if (command === "copy") { + const [source, target] = [argv[1], argv[2]]; + const name = target.slice(0, target.indexOf(":")); + const guest = guestIn(name); + const destination = guest(target.slice(target.indexOf(":") + 1)); + mkdirSync(destination, { recursive: true }); + for (const entry of readdirSync(source, { withFileTypes: true })) { + const from = join(source, entry.name), to = join(destination, entry.name); + if (entry.isDirectory()) cpSync(from, to, { recursive: true }); else cpSync(from, to); + } +} else if (command === "exec") { + const vm = read(argv[1]) ?? fail(`sandbox not found: ${argv[1]}`); + const guest = guestIn(vm.name); + const rest = argv.slice(argv.indexOf("--") + 1); + const [tool, flag, script, ...args] = rest; + const put = (path, text) => { mkdirSync(dirname(guest(path)), { recursive: true }); writeFileSync(guest(path), text); }; + if (tool === "rm") { + rmSync(guest(rest[2]), { recursive: true, force: true }); + } else if (tool === "df") { + process.stdout.write("Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/vdc 20971520 1048576 15728640 6% /layerfs/store\n"); + } else if (tool === "sh" && flag === "-c") { + if (/test -f "\$1" && cat "\$1"/.test(script)) { + if (!existsSync(guest(args[1]))) process.exit(1); + process.stdout.write(readFileSync(guest(args[1]))); + } else if (/cat > "\$1"/.test(script)) { + put(args[1], readFileSync(0)); + } else if (/umask 022/.test(script)) { + mkdirSync(guest(args[1]), { recursive: true, mode: 0o755 }); + } else if (/max_user_namespaces/.test(script)) { + put("/proc/sys/user/max_user_namespaces", "0\n"); + } else if (/nohup node/.test(script)) { + launchWorker(vm); + } + } +} else { + fail(`fake msb does not know ${command}`); +} diff --git a/api/runtime/test/fixture/fake-ssh.cjs b/api/runtime/test/fixture/fake-ssh.cjs new file mode 100644 index 0000000000..f2616f42af --- /dev/null +++ b/api/runtime/test/fixture/fake-ssh.cjs @@ -0,0 +1,184 @@ +// The `ssh` stand-in for @layerfs/runtime's vm:ssh tests (#515 §6): no host. +// It answers exactly the calls `platform/vm/ssh.js` makes, keeps one guest root +// under FAKE_SSH_STATE/guest, and serves the private stream itself: +// +// ssh -N -o ExitOnForwardFailure=yes -L 127.0.0.1:H:127.0.0.1:G … the forward +// ssh … -- uname -s | findmnt | blkid | df -kP | sh -c SCRIPT sh ARGS… +// tar -C -cf - . | ssh … -- mkdir -p DEST && tar -C DEST -xf - +// +// Facts worth knowing before changing it: +// - the forward is lazy: the worker's port is read from state/worker.port when +// a client connects, so the forward may start before the worker does, and +// two test runs never fight over one fixed port; +// - it exits 255 with `Address already in use` when its local port is taken, +// which is what `-o ExitOnForwardFailure=yes` reports; +// - `FAKE_SSH_UNREACHABLE=1` makes every call fail like a dead network, +// `FAKE_SSH_STORE_MISSING=/path` makes `test -e` fail for that path, and +// `FAKE_SSH_STORE_NON_EXT4=xfs` answers that filesystem instead of ext4; +// - every call is appended to FAKE_SSH_STATE/calls.jsonl as { at, argv }. +const { appendFileSync, existsSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync } = require("node:fs"); +const { spawn } = require("node:child_process"); +const { createServer, connect } = require("node:net"); +const { dirname, join } = require("node:path"); + +const state = process.env.FAKE_SSH_STATE; +if (!state) { + console.error("fake ssh: FAKE_SSH_STATE is not set"); + process.exit(2); +} + +const argv = process.argv.slice(2); +appendFileSync(join(state, "calls.jsonl"), `${JSON.stringify({ at: Date.now(), argv })}\n`); + +const guestIn = (path) => join(state, "guest", path.replace(/^\//, "")); +const workerPortFile = join(state, "worker.port"); +const workerPidFile = () => guestIn("/layerfs/state/worker.pid"); +const log = (...parts) => process.stderr.write(`${parts.join(" ")}\n`); + +/** A port nothing listens on now: the stand-in worker's own, chosen per launch. */ +function freePort() { + return new Promise((resolve) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +function endWorker() { + const file = workerPidFile(); + if (!existsSync(file)) return; + try { process.kill(Number(readFileSync(file, "utf8").trim())); } catch { /* already gone */ } +} + +/** The bundle's own worker, on a free port this stand-in records. */ +async function launchWorker() { + endWorker(); + const envFile = guestIn("/layerfs/state/worker.env"); + const env = Object.fromEntries(readFileSync(envFile, "utf8").split("\n").filter(Boolean).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)])); + const script = guestIn("/layerfs/bundle/api/runtime/src/worker/server.js"); + if (!existsSync(script)) { log("the bundle holds no worker/server.js"); process.exit(1); } + const port = await freePort(); + writeFileSync(workerPortFile, `${port}\n`); + const logFile = guestIn("/layerfs/state/worker.log"); + mkdirSync(dirname(logFile), { recursive: true }); + const child = spawn(process.execPath, [script], { + env: { ...process.env, ...env, LAYERFS_HOME: guestIn(env.LAYERFS_HOME), LAYERFS_BUNDLE: guestIn(env.LAYERFS_BUNDLE), LAYERFS_WORKER_LISTEN: `127.0.0.1:${port}`, LAYERFS_NATIVE: process.env.FAKE_SSH_NATIVE }, + detached: true, + stdio: ["ignore", "ignore", openSync(logFile, "a")], + }); + child.unref(); + mkdirSync(dirname(workerPidFile()), { recursive: true }); + writeFileSync(workerPidFile(), `${child.pid}\n`); +} + +/** + * The private stream. Lazy on purpose: a client that arrives before the worker + * exists waits up to a second for state/worker.port rather than failing. + */ +function forward(hostPort) { + const server = createServer((client) => { + client.on("error", () => {}); + let tries = 0; + const dial = () => { + const port = existsSync(workerPortFile) ? Number(readFileSync(workerPortFile, "utf8").trim()) : NaN; + if (!Number.isFinite(port)) { + if (tries++ < 20) return setTimeout(dial, 50); + return client.destroy(); + } + const upstream = connect(port, "127.0.0.1"); + upstream.on("error", () => client.destroy()); + client.pipe(upstream); + upstream.pipe(client); + }; + dial(); + }); + server.on("error", (error) => { + if (error.code === "EADDRINUSE") log(`bind [127.0.0.1]:${hostPort}: Address already in use`); + else log(`forward error: ${error.message}`); + process.exit(255); + }); + server.listen(hostPort, "127.0.0.1"); +} + +/** `sh -c SCRIPT sh ARG…`: the handful of scripts the adapter sends. `argv[0]` is + * the `sh` that names the script's `$0`, so the script's `$1` is `args[1]`. */ +function shellScript(script, args) { + const path = args[1]; + if (script.includes("max_user_namespaces")) { + const file = guestIn("/proc/sys/user/max_user_namespaces"); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, "0\n"); + return 0; + } + if (script.includes("test -f \"$1\" && cat \"$1\"")) { + const file = guestIn(path); + if (!existsSync(file)) return 1; + process.stdout.write(readFileSync(file)); + return 0; + } + if (script.includes("cat > \"$1\"")) { + const file = guestIn(path); + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + writeFileSync(file, readFileSync(0), { mode: 0o600 }); + return 0; + } + if (script.includes("test -e \"$1\"")) return process.env.FAKE_SSH_STORE_MISSING === path ? 1 : 0; + if (script.includes("blkid")) { process.stdout.write(`${process.env.FAKE_SSH_STORE_NON_EXT4 || "ext4"}\n`); return 0; } + if (script.includes("test -b \"$1\"")) return 1; + if (script.includes("mount") && script.includes("/layerfs/store")) return 0; + if (script.includes("nohup node")) return launchWorker().then(() => 0); + if (script.includes("worker.pid") && script.includes("rm -rf")) { + endWorker(); + rmSync(guestIn("/layerfs/state"), { recursive: true, force: true }); + rmSync(guestIn("/layerfs/bundle"), { recursive: true, force: true }); + return 0; + } + if (script.includes("worker.pid")) { endWorker(); return 0; } + return 0; +} + +/** The unpack end of a bundle copy: `mkdir -p DEST && tar -C DEST -xf -`. */ +function unpack(script) { + const match = /tar -C ('([^']+)'|\S+) -xf -/.exec(script); + const destination = guestIn(match ? match[2] ?? match[1] : "/layerfs/bundle"); + mkdirSync(destination, { recursive: true }); + const tar = spawn("tar", ["-C", destination, "-xf", "-"], { stdio: ["pipe", "ignore", "inherit"] }); + process.stdin.pipe(tar.stdin); + tar.on("close", (code) => process.exit(code ?? 0)); +} + +async function main() { + if (process.env.FAKE_SSH_UNREACHABLE) { + log("ssh: connect to host: Network is unreachable"); + process.exit(255); + } + const dash = argv.indexOf("--"); + const remoteArgv = dash === -1 ? [] : argv.slice(dash + 1); + const joined = remoteArgv.join(" "); + + if (argv.includes("-N")) { + const index = argv.indexOf("-L"); + const port = index === -1 ? NaN : Number(/^127\.0\.0\.1:(\d+):/.exec(argv[index + 1])?.[1]); + if (!Number.isFinite(port)) { log("ssh: no local port to forward"); process.exit(255); } + return forward(port); + } + if (!remoteArgv.length) { log("ssh: no command"); process.exit(255); } + if (remoteArgv[0] === "uname") { process.stdout.write(`${process.env.FAKE_SSH_OS || "Linux"}\n`); return; } + if (remoteArgv[0] === "findmnt") { process.stdout.write(`${process.env.FAKE_SSH_STORE_NON_EXT4 || "ext4"}\n`); return; } + if (remoteArgv[0] === "df") { + process.stdout.write("Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/sda1 100000000 5000000 95000000 5% /layerfs/store\n"); + return; + } + if (remoteArgv[0] === "tar" || /tar -C/.test(joined)) return unpack(joined); + if (remoteArgv[0] === "rm") { rmSync(guestIn(remoteArgv.at(-1)), { recursive: true, force: true }); return; } + if (remoteArgv[0] === "sh") { + const code = await shellScript(remoteArgv[2] ?? "", remoteArgv.slice(3)); + process.exit(code); + } + log(`fake ssh does not know ${joined}`); + process.exit(1); +} + +main(); diff --git a/api/runtime/test/verify/ssh_faults.mjs b/api/runtime/test/verify/ssh_faults.mjs new file mode 100644 index 0000000000..fdc300837a --- /dev/null +++ b/api/runtime/test/verify/ssh_faults.mjs @@ -0,0 +1,342 @@ +// The `vm:ssh` failure matrix (#515 §6, V3; task H40 §5), with the stand-in +// transport of `vm_placement.mjs` and the real provider code: no host, no VM. +// Each scenario states what the user sees and what is left behind; every await +// has a deadline and the whole run has a watchdog, so a stuck provider call +// fails this file instead of hanging a shell. +// node api/runtime/test/verify/ssh_faults.mjs [port|store|ext4|host|drop|disconnect|remote|delete] +// (no selector: every scenario, one fresh state folder each) +import assert from "node:assert/strict"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createServer, connect } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SELF = fileURLToPath(import.meta.url); +const ROOT = fileURLToPath(new URL("../../..", import.meta.url)); +const FAKE = fileURLToPath(new URL("../../../agent/test/fixture/fake-native.cjs", import.meta.url)); +const SOURCE = fileURLToPath(new URL("../../src", import.meta.url)); +const SSH = fileURLToPath(new URL("../fixture/fake-ssh.cjs", import.meta.url)); +process.env.LAYERFS_NATIVE = FAKE; +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const provider = (await import("../../src/platform/vm/ssh.js")).default; +const { endpointFile, ping } = await import("../../src/platform/shared.js"); +const { deploymentEnv } = await import("../../src/clients.js"); + +const SCENARIOS = ["port", "store", "ext4", "host", "drop", "disconnect", "remote", "delete", "shared"]; + +/** One await's ceiling: a stuck call fails with its own name. */ +const deadline = (ms, what, promise) => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not finish within ${ms / 1000} s`)), ms).unref()), + ]); + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const sleep = delay; +const fails = (promise, matcher) => + promise.then((value) => assert.fail(`expected a refusal, got ${JSON.stringify(value)}`), (error) => { + assert.match(`${error.code} ${error.message}`, matcher, `${error.code}: ${error.message}`); + return error; + }); + +/** A sealed Linux bundle with this adapter's worker and the real SDK packages. */ +function bundleAt(folder) { + rmSync(folder, { recursive: true, force: true }); + cpSync(join(SOURCE, "worker"), join(folder, "api/runtime/src/worker"), { recursive: true }); + cpSync(join(ROOT, "node_modules/@layerfs"), join(folder, "api/node_modules/@layerfs"), { recursive: true, dereference: true }); + mkdirSync(join(folder, "images"), { recursive: true }); + writeFileSync(join(folder, "images", `${"a".repeat(64)}.tar`), "an uncompressed docker save archive\n"); + writeFileSync(join(folder, "manifest.json"), JSON.stringify({ version: 1, status: "sealed", platform: "linux", build_id: "faults", files: {} })); + return folder; +} + +/** One scenario's own world: a bundle, a home, a stand-in SSH on PATH, a state folder. */ +function world(tag) { + const work = mkdtempSync(join(tmpdir(), `layerfs-ssh-${tag}-`)); + const state = join(work, "state"); + mkdirSync(state, { recursive: true }); + const bin = join(work, "bin"); + mkdirSync(bin, { recursive: true }); + const ssh = join(bin, "ssh"); + writeFileSync(ssh, `#!/bin/sh\nexec "${process.execPath}" "${SSH}" "$@"\n`); + chmodSync(ssh, 0o755); + const saved = { path: process.env.PATH, state: process.env.FAKE_SSH_STATE, native: process.env.FAKE_SSH_NATIVE, dir: process.env.LAYERFS_SSH_STATE_DIR }; + process.env.PATH = `${bin}:${saved.path}`; + process.env.FAKE_SSH_STATE = state; + process.env.FAKE_SSH_NATIVE = FAKE; + process.env.LAYERFS_SSH_STATE_DIR = join(state, "sessions"); + return { + work, + state, + home: join(work, "home"), + bundle: bundleAt(join(work, "bundle")), + sessions: () => (existsSync(join(state, "sessions")) ? readdirSync(join(state, "sessions")).filter((file) => file.endsWith(".json")) : []), + calls: () => (existsSync(join(state, "calls.jsonl")) ? readFileSync(join(state, "calls.jsonl"), "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line).argv) : []), + guest: (...parts) => join(state, "guest", ...parts), + done: () => { + for (const [key, value] of [["PATH", saved.path], ["FAKE_SSH_STATE", saved.state], ["FAKE_SSH_NATIVE", saved.native], ["LAYERFS_SSH_STATE_DIR", saved.dir]]) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + killStandIns(state); + rmSync(work, { recursive: true, force: true }); + }, + }; +} + +/** Ends every process this world started: forwards from their records, workers from their pid files. + * A process that refuses to end is reported: a leaked stand-in is a test failure, not a silent one. */ +function killStandIns(state) { + const pids = []; + const sessions = join(state, "sessions"); + if (existsSync(sessions)) { + for (const file of readdirSync(sessions).filter((name) => name.endsWith(".json"))) { + try { + const { forwardPid } = JSON.parse(readFileSync(join(sessions, file), "utf8")); + if (forwardPid) pids.push(forwardPid); + } catch { /* already gone */ } + } + } + const pid = join(state, "guest/layerfs/state/worker.pid"); + if (existsSync(pid)) pids.push(Number(readFileSync(pid, "utf8").trim())); + for (const one of pids) { + try { process.kill(one, "SIGKILL"); } catch { /* already gone */ } + } + const left = pids.filter((one) => provider.isAlive(one)); + if (left.length) process.stderr.write(`ssh_faults: residue, these stand-ins did not end: ${left.join(", ")}\n`); +} + +/** A project of the home, as Init leaves it: an entry and the runtime's record. */ +function project(home, name) { + mkdirSync(join(home, name), { recursive: true }); + writeFileSync(join(home, name, "project.record"), "x"); + mkdirSync(join(home, ".runtime", "projects"), { recursive: true }); + writeFileSync(join(home, ".runtime", "projects", `${name}.json`), JSON.stringify({ name, volume: `lfs-${name}-store-1`, created: 1 })); +} + +/** A local port nothing listens on now, for a forward that must succeed. */ +const freePort = () => new Promise((resolve) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { const { port } = server.address(); server.close(() => resolve(port)); }); +}); + +/** Whether something accepts a connection on a loopback port now. */ +const answers = (port) => new Promise((resolve) => { + const socket = connect({ host: "127.0.0.1", port }); + const done = (value) => { socket.destroy(); resolve(value); }; + socket.setTimeout(500); + socket.on("connect", () => done(true)); + socket.on("timeout", () => done(false)); + socket.on("error", () => done(false)); +}); + +const config = (world, overrides = {}) => ({ + name: "vm-a", host: "127.0.0.1", user: "root", port: 22, storePath: "/layerfs/store", + home: world.home, storeName: "store-a", workerPort: 7070, hostPort: 0, ...overrides, +}); + +const scenarios = { + // ① a local forward port someone else holds: create fails at once, and nothing + // was uploaded or started on the host + async port(world) { + const taken = await freePort(); + const server = createServer(() => {}); + await new Promise((resolve) => server.listen(taken, "127.0.0.1", resolve)); + try { + const error = await deadline(40_000, "create with a busy port", fails(provider.create(config(world, { hostPort: taken })), /could not open the private stream.*Address already in use|address already in use/i)); + assert.match(error.message, new RegExp(`127\\.0\\.0\\.1:${taken}`), error.message); + assert.deepEqual(world.sessions(), [], "no session record is written"); + const calls = world.calls(); + assert.ok(!calls.some((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))), "nothing is uploaded"); + assert.ok(!calls.some((argv) => argv.some((value) => typeof value === "string" && value.includes("nohup node"))), "no worker is started"); + // the same VM on a free port is made: the refusal was about the port, not the host + const made = await deadline(60_000, "create on a free port", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(made.state, "running"); + await deadline(30_000, "remove", provider.remove("vm-a")); + assert.deepEqual(world.sessions(), [], "a delete clears the record"); + } finally { + server.close(); + } + }, + + // ② the Store path does not exist: refused before the forward, with nothing left + async store(world) { + process.env.FAKE_SSH_STORE_MISSING = "/mnt/nope"; + try { + await deadline(40_000, "create with a missing Store path", fails(provider.create(config(world, { storePath: "/mnt/nope" })), /InvalidInput.*does not exist/)); + } finally { + delete process.env.FAKE_SSH_STORE_MISSING; + } + assert.deepEqual(world.sessions(), [], "no session record"); + assert.deepEqual(world.calls().filter((argv) => argv.includes("-N")), [], "no forward is opened"); + }, + + // ③ the Store path is not ext4: refused by name, with the filesystem it is + async ext4(world) { + process.env.FAKE_SSH_STORE_NON_EXT4 = "xfs"; + try { + const error = await deadline(40_000, "create on a non-ext4 path", fails(provider.create(config(world, { storePath: "/mnt/store" })), /Unsupported.*not ext4/)); + assert.match(error.message, /xfs/); + } finally { + delete process.env.FAKE_SSH_STORE_NON_EXT4; + } + assert.deepEqual(world.sessions(), []); + }, + + // ④ a host that is not Linux: refused by what it answers, not by a guess + async host(world) { + process.env.FAKE_SSH_OS = "Darwin"; + try { + await deadline(40_000, "create against a non-Linux host", fails(provider.create(config(world)), /Unsupported.*not Linux/)); + } finally { + delete process.env.FAKE_SSH_OS; + } + assert.deepEqual(world.sessions(), []); + }, + + // ⑤ the network is down, then it is back: the first call fails clearly, the + // second one makes the VM + async drop(world) { + process.env.FAKE_SSH_UNREACHABLE = "1"; + try { + const error = await deadline(40_000, "create while unreachable", fails(provider.create(config(world)), /Failed.*(SSH to 127\.0\.0\.1 failed|could not reach)/)); + assert.match(error.message, /Network is unreachable/, "the refusal carries the transport's own reason"); + assert.match(error.message, /127\.0\.0\.1/, "and names the host"); + } finally { + delete process.env.FAKE_SSH_UNREACHABLE; + } + assert.deepEqual(world.sessions(), [], "an unreachable host leaves no record"); + const made = await deadline(60_000, "create after reconnect", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(made.state, "running"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, + + // ⑥ the SSH forward dies under a running VM (a dropped connection): the VM + // reads stopped, its endpoint refuses, and start brings it back + async disconnect(world) { + const hostPort = await freePort(); + const made = await deadline(60_000, "create", provider.create(config(world, { hostPort }))); + assert.equal(made.state, "running"); + assert.equal(await answers(hostPort), true, "the forward answers while it runs"); + const pid = made.ports[0].host; + const record = JSON.parse(readFileSync(join(world.state, "sessions", "vm-a.json"), "utf8")); + process.kill(record.forwardPid); + for (let waited = 0; waited < 2_000 && (await answers(pid)); waited += 50) await sleep(50); + assert.equal(await answers(pid), false, "the dropped forward refuses, so a call fails with a clear connection error"); + assert.equal((await deadline(20_000, "inspect after the drop", provider.inspect("vm-a"))).state, "stopped"); + const started = await deadline(60_000, "start after the drop", provider.start("vm-a")); + assert.equal(started.state, "running"); + assert.equal(await answers(pid), true, "the same port answers again"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, + + // ⑦ the host (or its worker) is gone, then back: the runtime re-opens the + // worker without copying the bundle again + async remote(world) { + project(world.home, "shop"); + const runtime = await LayerFSRuntime.open({ + home: world.home, bundle: world.bundle, placement: "vm:ssh", + containerPrefix: "lfs-fault-", platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + const first = await deadline(90_000, "worker.start", runtime.worker.start("shop")); + assert.deepEqual([first.vm.state, first.worker.state], ["running", "running"]); + const copies = () => world.calls().filter((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))).length; + assert.equal(copies(), 1, "the bundle is uploaded once"); + const endpoint = JSON.parse(readFileSync(endpointFile(world.home, "shop"), "utf8")); + assert.equal((await deadline(5_000, "ping a running VM", ping(endpoint)))?.ready, true); + // the host reboots: its worker and the forward are gone, the disk stays + await deadline(30_000, "stop", runtime.vm.stop("shop")); + const pid = join(world.state, "guest/layerfs/state/worker.pid"); + assert.equal(existsSync(pid), true, "the stand-in recorded the worker it started"); + assert.equal((await deadline(2_000, "ping a stopped VM", ping(endpoint))), null, "a stopped VM's worker does not answer"); + const back = await deadline(90_000, "worker.start after the reboot", runtime.worker.start("shop")); + assert.deepEqual([back.vm.state, back.worker.state], ["running", "running"]); + assert.equal(copies(), 1, "the bundle on the host's disk was reused, not uploaded again"); + assert.equal((await deadline(5_000, "ping after the restart", ping(JSON.parse(readFileSync(endpointFile(world.home, "shop"), "utf8")))))?.ready, true); + // an un-pushed Store is never destroyed by a delete + await deadline(20_000, "delete with a project", fails(runtime.vm.delete("shop", { confirm: "delete" }), /Unsupported.*never destroys a Store/)); + assert.equal(world.sessions().length, 1, "the refused delete left the VM alone"); + } finally { + await runtime.close(); + } + }, + + // ⑧ delete after the project is gone: the record, the forward and this + // adapter's remote copy go; the user's Store path is untouched + async delete(world) { + mkdirSync(world.guest("layerfs/store/global"), { recursive: true }); + writeFileSync(world.guest("layerfs/store/global/store.sqlite"), "the user's Store\n"); + project(world.home, "shop"); + const runtime = await LayerFSRuntime.open({ + home: world.home, bundle: world.bundle, placement: "vm:ssh", + containerPrefix: "lfs-fault-", platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + await deadline(90_000, "worker.start", runtime.worker.start("shop")); + // the project goes: its entry and the runtime's record + rmSync(join(world.home, "shop"), { recursive: true, force: true }); + rmSync(join(world.home, ".runtime", "projects", "shop.json"), { force: true }); + const gone = await deadline(60_000, "vm.delete", runtime.vm.delete("shop", { confirm: "delete" })); + assert.deepEqual([gone.state, gone.store], ["deleted", "kept"], "the Store disk is kept by default"); + assert.deepEqual(world.sessions(), [], "the session record is gone"); + assert.equal(existsSync(world.guest("layerfs/bundle")), false, "the adapter's bundle copy is removed"); + assert.equal(existsSync(world.guest("layerfs/state")), false, "its worker state is removed"); + assert.equal(readFileSync(world.guest("layerfs/store/global/store.sqlite"), "utf8"), "the user's Store\n", "the user's Store path is untouched"); + } finally { + await runtime.close(); + } + }, + + // ⑨ an un-pushed Store is refused by the runtime, and one host never serves two + async shared(world) { + const first = await deadline(60_000, "create the first VM", provider.create(config(world, { hostPort: await freePort() }))); + assert.equal(first.state, "running"); + const second = await deadline(30_000, "create a second VM on the same host", fails(provider.create(config(world, { name: "vm-b", storeName: "store-b", hostPort: await freePort() })), /Busy.*one host serves one Store/)); + assert.match(second.message, /vm-a/, "the refusal names the VM that already holds it"); + assert.deepEqual(world.sessions(), ["vm-a.json"], "the refused create wrote no record"); + await deadline(30_000, "remove", provider.remove("vm-a")); + }, +}; + +const selected = process.argv[2]; +if (!selected) { + // One child per scenario: a runtime binds this process's worker route once + // per project (`useWorker`), so two runtimes cannot share a process, and a + // scenario that hangs ends its own child instead of the whole file. + const { spawn } = await import("node:child_process"); + const results = []; + for (const name of SCENARIOS) { + const code = await new Promise((resolve) => { + const child = spawn(process.execPath, [SELF, name], { stdio: ["ignore", "inherit", "inherit"], env: process.env }); + const timer = setTimeout(() => { child.kill("SIGKILL"); }, 120_000); + child.on("close", (value) => { clearTimeout(timer); resolve(value); }); + }); + results.push([name, code]); + } + const failed = results.filter(([, code]) => code !== 0); + process.stdout.write(`${results.map(([name, code]) => `${code === 0 ? "PASS" : "FAIL"} ssh_faults/${name}`).join("\n")}\n`); + if (failed.length) { + process.stderr.write(`ssh_faults: ${failed.length} of ${results.length} scenarios failed: ${failed.map(([name]) => name).join(", ")}\n`); + process.exit(1); + } + process.stdout.write("PASS ssh_faults\n", () => process.exit(0)); +} else { + if (!scenarios[selected]) throw new Error(`select ${SCENARIOS.join(", ")}`); + setTimeout(() => { + process.stderr.write(`ssh_faults/${selected}: FAIL, the scenario did not finish in 120 s\n`); + process.exit(1); + }, 120_000).unref(); + + const box = world(selected); + const started = Date.now(); + try { + await deadline(110_000, `scenario ${selected}`, scenarios[selected](box)); + console.log(`PASS ssh_faults/${selected} (${Date.now() - started} ms)`); + } finally { + box.done(); + } + process.stdout.write("PASS ssh_faults\n", () => process.exit(0)); +} diff --git a/api/runtime/test/verify/vm_real.mjs b/api/runtime/test/verify/vm_real.mjs new file mode 100644 index 0000000000..e5079dbea9 --- /dev/null +++ b/api/runtime/test/verify/vm_real.mjs @@ -0,0 +1,363 @@ +// @layerfs/runtime on a `vm:` placement, against real VMs (#515 §6, +// V9, V1): **one Linux VM per Store**. The runtime makes each project's VM, +// copies the sealed bundle into it, blocks user namespaces, starts its worker as +// root and writes that project's endpoint; each project's agent process reaches +// only its own VM, where the Sandboxes are Linux namespaces (H36's backend). +// Two projects run side by side: neither sees the other's Store, removing one +// leaves the other alone, and V8 refuses a delete while its Store exists. +// The Sandbox root is the bundle's image (#515 §4.7 V7, H38): a command sees the +// image's own `/etc/os-release` and marker and never the VM's tree, the image's +// directories are not writable by it while its own overlay layer is, two +// Sandboxes' writes to the image stay each one's, one digest is unpacked once, +// and `LAYERFS_LOCAL_DIR` (their overlay uppers) is on the VM's ext4 disk. +// A Sandbox left behind by a VM restart is a stopped record: it is not reopened +// onto a root that boot did not start, the next `sandbox.create` reclaims it and +// reuses its number, and that Sandbox runs from the same unpacked image. Everything it made (both VMs, their Store disks, the home and +// the sources) is removed afterwards. +// node api/runtime/test/verify/vm_real.mjs [microsandbox|apple-container] +// Needs that provider's CLI and the bundle built for the host's Linux VM +// (aarch64 on an M-series Mac), with a Sandbox image at `images/<64 hex>.tar`. +// Under 2 minutes for small source folders; the first run pulls the image. +// LAYERFS_TEST_DIR (default ~/.cache/layerfs-test) holds the run's home, sources +// and traces. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SELF = fileURLToPath(import.meta.url); +const CLI = { microsandbox: "msb", "apple-container": "container" }; + +/** An agent process: it names its project, and the endpoint of that project's VM comes from the deployment. */ +async function agent(project, deployment, phase = undefined, extra = {}) { + const env = { ...process.env, ...deployment, PROJECT: project, ...(phase ? { AGENT_PHASE: phase } : {}), ...extra }; + return new Promise((resolve) => { + const child = spawn(process.execPath, [SELF, "agent"], { env, stdio: ["ignore", "pipe", "pipe"] }); + const out = []; + const timer = setTimeout(() => child.kill("SIGKILL"), 100_000); + child.stdout.on("data", (chunk) => out.push(chunk)); + child.stderr.on("data", (chunk) => out.push(chunk)); + child.on("close", (code) => { clearTimeout(timer); resolve({ code, text: Buffer.concat(out).toString() }); }); + }); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "resume") { + // after the project's VM was restarted and its worker restarted: the committed + // lane is still there, and the Sandbox the previous boot left is a stopped + // record, never reopened onto a root this boot did not start + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const left = process.env.AGENT_LEFTOVER; + try { + assert.equal((await shop.fs.read("lane", "/fresh.txt")).text, "two\n"); + if (left) { + // the Sandbox the previous boot left: it is not running, and it is not + // reopened onto a root this boot did not start + assert.ok(!(await shop.project.list_sandboxes(1000, 0)).items.some((entry) => entry.sandbox === left), `the Sandbox ${left} of the previous boot is not running`); + await assert.rejects(shop.workspace.mount(left, "lane"), (error) => { + console.log(`reopening the restarted VM's Sandbox was refused: ${error.code}: ${error.message}`); + return true; + }); + } + // the record whose init the reboot ended is reclaimed by the next create: + // its number comes back, and the same digest is started from the cache the + // VM kept rather than unpacked again + const { sandbox } = await shop.sandbox.create(); + if (left) { + console.log(`reclaimed=${left} reused=${sandbox}`); + assert.equal(sandbox, left, `the reclaimed record's number is free again: ${sandbox}`); + } + const ws = await shop.workspace.mount(sandbox, "lane"); + const ran = await shop.workspace.exec(ws, "cat fresh.txt; cat /etc/layerfs-image"); + assert.equal(ran.stdout, "two\nh39-sandbox-image\n", ran.stderr); + await shop.workspace.unmount(ws); + await shop.sandbox.close(sandbox); + } finally { + await shop.close(); + } + console.log(`${process.env.PROJECT} after the worker restart: ok`); + process.exit(0); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "third") { + // the bundle's Sandbox image (#515 §4.7 V7): a command's root is the image, + // not the VM's tree; the image's directories are read-only to it while its + // own writable layer is not, and two Sandboxes' writes stay each one's + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const open = []; + try { + const { sandbox } = await shop.sandbox.create(); + const ws = await shop.workspace.mount(sandbox, "lane"); + open.push(() => shop.sandbox.close(sandbox), () => shop.workspace.unmount(ws, true)); + const image = await shop.workspace.exec(ws, "cat /etc/os-release; cat /etc/layerfs-image; ls /layerfs"); + assert.equal(image.exitCode, 0, image.stderr); + assert.match(image.stdout, /^PRETTY_NAME="LayerFS Sandbox Image \(busybox static\)"$/m, image.stdout); + assert.match(image.stdout, /^h39-sandbox-image$/m, image.stdout); + assert.equal(image.stdout.split("\n").at(-2), "store", `only the Store's directory is bound into the image, not the VM's /layerfs: ${image.stdout}`); + // the image is not the VM's root: the VM's node and bundle are absent + const shape = await shop.workspace.exec(ws, "command -v node; echo node=$?; ls /bin/busybox; id -u"); + assert.match(shape.stdout, /^node=[1-9]/m, shape.stdout); + assert.match(shape.stdout, /^\/bin\/busybox$/m, shape.stdout); + assert.match(shape.stdout, /^501$/m, shape.stdout); + // the image's own directories are root's: a command cannot write them, and + // what it writes goes to its own writable layer instead + const writes = await shop.workspace.exec(ws, "printf x > /etc/layerfs-probe; echo etc-write=$?; test -w /etc; echo etc-w=$?; printf mine > /var/tmp/mine; echo tmp-write=$?"); + assert.match(writes.stdout, /^etc-write=[1-9]/m, writes.stdout); + assert.match(writes.stdout, /^etc-w=[1-9]/m, writes.stdout); + assert.match(writes.stdout, /^tmp-write=0$/m, writes.stdout); + // a second Sandbox on the same image reads the image, not the first one's layer + const { sandbox: other } = await shop.sandbox.create(); + const ws2 = await shop.workspace.mount(other, "lane"); + open.push(() => shop.sandbox.close(other), () => shop.workspace.unmount(ws2, true)); + const apart = await shop.workspace.exec(ws2, "cat /var/tmp/mine 2>&1; echo mine=$?; printf other > /var/tmp/mine; cat /var/tmp/mine"); + assert.match(apart.stdout, /No such file|not found/, apart.stdout); + assert.match(apart.stdout, /^mine=[1-9]$/m, apart.stdout); + assert.match(apart.stdout, /^other$/m, apart.stdout); + } finally { + for (const step of open.reverse()) await step().catch((error) => console.error(`teardown: ${error.message}`)); + await shop.close(); + } + console.log("the Sandbox root is the bundle's image: ok"); + process.exit(0); +} + +if (process.argv[2] === "agent" && process.env.AGENT_PHASE === "blog") { + // the other project's agent: its own VM, its own Store + const { LayerFS } = await import("@layerfs/agent"); + const blog = await LayerFS.open(process.env.PROJECT); + try { + assert.equal((await blog.fs.read("main", "/README.md")).text, "# Blog\n"); + // the first project's Branch and files are not in this Store at all + assert.ok(!(await blog.project.list_branches(1000, 0)).items.some((branch) => branch.name === "lane"), "the other project's Branch is not here"); + await assert.rejects(blog.fs.read("lane", "/note.txt"), { code: "NoSuchBranch" }); + await blog.branch.fork("blog-work"); + const { sandbox } = await blog.sandbox.create(); + const ws = await blog.workspace.mount(sandbox, "blog-work"); + try { + const ran = await blog.workspace.exec(ws, "id -u; cat README.md"); + assert.match(ran.stdout, /^501\n# Blog\n$/m, ran.stdout); + } finally { + await blog.workspace.unmount(ws, true); + await blog.sandbox.close(sandbox); + } + } finally { + await blog.close(); + } + console.log("blog over its own VM worker: ok"); + process.exit(0); +} + +if (process.argv[2] === "agent") { + // the first project's agent: only the deployment's endpoint is set, never LAYERFS_WORKER_ENDPOINT by hand + const { LayerFS } = await import("@layerfs/agent"); + const shop = await LayerFS.open(process.env.PROJECT); + const undone = []; + try { + await shop.branch.fork("lane"); + await shop.fs.write("lane", "/note.txt", "one\n", "Add note"); + const { sandbox } = await shop.sandbox.create(); + const ws = await shop.workspace.mount(sandbox, "lane"); + undone.push(() => shop.sandbox.close(sandbox), () => shop.workspace.unmount(ws, true)); + // the command runs in a namespace Sandbox inside the project's VM: nonroot, + // no capabilities, and the VM's home and Store around it + const ran = await shop.workspace.exec(ws, "id -u; cat README.md note.txt && echo two > fresh.txt"); + assert.equal(ran.exitCode, 0, ran.stderr); + assert.match(ran.stdout, /^501\n# Shop\none\n$/m); + // V4: the VM blocks user namespaces, so a command cannot copy the Workspace + // mount into one of its own; the command's own identity is nonroot + const blocked = await shop.workspace.exec(ws, "cat /proc/sys/user/max_user_namespaces; unshare -U /bin/true 2>&1; echo rc=$?"); + assert.match(blocked.stdout, /^0\n/m, blocked.stdout); + assert.match(blocked.stdout, /rc=[1-9]/, blocked.stdout); + assert.equal((await shop.workspace.commit(ws, "Add fresh")).kind, "Committed"); + await shop.workspace.exec(ws, "echo hidden > secret.txt"); + await shop.workspace.unmount(ws, true); + await shop.sandbox.close(sandbox); + undone.length = 0; + assert.equal((await shop.fs.read("lane", "/fresh.txt")).text, "two\n"); + // a second Sandbox on the same Store has its own mount + const { sandbox: second } = await shop.sandbox.create(); + const ws2 = await shop.workspace.mount(second, "lane"); + undone.push(() => shop.sandbox.close(second), () => shop.workspace.unmount(ws2, true)); + const isolated = await shop.workspace.exec(ws2, "test -e secret.txt && echo leaked || echo isolated"); + assert.equal(isolated.stdout.trim(), "isolated", isolated.stdout); + const again = await shop.workspace.exec(ws2, "id -u; cat fresh.txt"); + assert.match(again.stdout, /^501\ntwo\n$/m, again.stdout); + // this Sandbox is left open on purpose: the parent stops the VM under it, so + // the reboot leaves a stopped record for the resume phase to close. Its + // Workspace is unmounted first: a stopped Sandbox's daemon is gone, so a + // Workspace still recorded as mounted there could not be unmounted with it. + await shop.workspace.unmount(ws2, true); + undone.length = 0; + console.log(`left=${second}`); + } finally { + for (const step of undone.reverse()) await step().catch((error) => console.error(`teardown: ${error.message}`)); + await shop.close(); + } + console.log("shop over its own VM worker: ok"); + process.exit(0); +} + +const [bundle, provider = "microsandbox"] = process.argv.slice(2); +if (!CLI[provider] || !bundle || process.platform !== "darwin") { + console.error("usage: vm_real.mjs [microsandbox|apple-container] (macOS host)"); + process.exit(2); +} +if (!existsSync(bundle)) { + console.error(`no sealed bundle at ${bundle}`); + process.exit(2); +} +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const { deploymentEnv } = await import("../../src/clients.js"); +const { endpointFile } = await import("../../src/platform/shared.js"); +// the provider module of the VM this run uses: the verify script reads the VM's +// own disk (filesystem, image cache), which no public call answers +const DRIVER = { microsandbox: "../../src/platform/vm/microsandbox.js", "apple-container": "../../src/platform/vm/apple-container.js" }; +const driver = (await import(DRIVER[provider])).default; + +const shared = process.env.LAYERFS_TEST_DIR || join(homedir(), ".cache", "layerfs-test"); +mkdirSync(shared, { recursive: true }); +const work = mkdtempSync(join(shared, `lfs-${provider}-`)); +const home = join(work, "home"), id = Date.now().toString(36), prefix = `lfs-${provider}-${id}-`; +const sources = {}; +for (const [name, text] of [["shop", "# Shop\n"], ["blog", "# Blog\n"]]) { + sources[name] = join(work, name); + mkdirSync(sources[name]); + writeFileSync(join(sources[name], "README.md"), text); +} +const projectEnv = (runtime, name, volume) => Object.fromEntries(Object.entries(deploymentEnv(runtime.settings, runtime.platform, { name, volume })).map(([key, value]) => [`LAYERFS_${key}`, value])); +const initProject = async (runtime, name) => { + let job = await runtime.project.start_init(name, sources[name]); + for (let i = 0; !job.ended && i < 900; i++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + job = await runtime.project.get_init(job); + } + assert.equal(job.step, "done", `${name}'s Init: ${JSON.stringify(job.error)}`); + return (await runtime.project.list()).find((project) => project.name === name); +}; + +/** One `sh -c` line as the named project's VM answers it. */ +const guest = async (project, command) => (await driver.exec(`${prefix}${project}-vm`, ["sh", "-c", command])).stdout.trim(); + +/** + * The project's VM as its disk shows it: the filesystem `LAYERFS_LOCAL_DIR` is on + * (H38's overlay upper needs a local one, not the VM's own overlay root) and the + * one image cache directory — its digest, inode, mtime and mode. An unpack writes + * inside that directory, so an unchanged inode and mtime after another Sandbox + * start is the proof that the same digest was unpacked once. + */ +async function disk(project) { + const digest = await guest(project, "ls -1 /layerfs/store/local/images 2>/dev/null | grep -E '^[0-9a-f]{64}$'"); + return { + digest, + stat: await guest(project, `stat -c '%i %Y' /layerfs/store/local/images/${digest}`), + mode: await guest(project, `stat -c '%a' /layerfs/store/local/images/${digest}`), + partial: await guest(project, "ls -A /layerfs/store/local/images | grep -c '^\\.tmp-' || true"), + local: await guest(project, "findmnt -T /layerfs/store/local -o FSTYPE -n"), + }; +} + +const started = Date.now(); +let runtime; +try { + runtime = await LayerFSRuntime.open({ home, bundle, placement: `vm:${provider}`, containerPrefix: prefix, sweep: false }); + // a project's VM is made on demand, for that project + const first = await runtime.worker.start("shop"); + assert.equal(first.project, "shop"); + assert.equal(first.worker.state, "running", JSON.stringify(first)); + assert.equal(first.vm.state, "running"); + assert.equal(first.vm.name, `${prefix}shop-vm`); + const shopEndpoint = JSON.parse(readFileSync(endpointFile(home, "shop"), "utf8")); + console.log(`${first.vm.name} running; its worker on 127.0.0.1:${shopEndpoint.port}`); + + const shop = await initProject(runtime, "shop"); + const blog = await initProject(runtime, "blog"); + const listed = await runtime.vm.status(); + assert.deepEqual(listed.vms.map((vm) => [vm.project, vm.state, vm.worker.state]).sort(), [["blog", "running", "running"], ["shop", "running", "running"]]); + const blogEndpoint = JSON.parse(readFileSync(endpointFile(home, "blog"), "utf8")); + assert.notEqual(blogEndpoint.port, shopEndpoint.port, "each project's VM owns its host port"); + assert.notEqual(blogEndpoint.secret, shopEndpoint.secret, "each project's VM has its own secret"); + assert.notEqual(blogEndpoint.vm.name, shopEndpoint.vm.name); + console.log(`two projects, two VMs: ok (${shop.volume}, ${blog.volume})`); + + // V8: each Store is on its own VM's disk and nowhere else, so a delete is refused + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + + const shopEnv = projectEnv(runtime, "shop", shop.volume); + const blogEnv = projectEnv(runtime, "blog", blog.volume); + assert.equal(shopEnv.LAYERFS_WORKER_ENDPOINT, endpointFile(home, "shop"), "the deployment names this project's worker"); + assert.equal(process.env.LAYERFS_WORKER_ENDPOINT, undefined, "the runtime hands its routes over in-process, not through its own environment"); + const shopRun = await agent("shop", shopEnv); + assert.equal(shopRun.code, 0, shopRun.text); + console.log(shopRun.text.trim()); + const left = /^left=(\S+)$/m.exec(shopRun.text)?.[1]; + assert.ok(left, `the shop agent left its Sandbox open for the restart: ${shopRun.text}`); + // #515 §4.7 V7: the Sandbox root is the bundle's image, unpacked once on the + // VM's ext4 disk (H38's require_overlay_upper), under its config digest + const unpacked = await disk("shop"); + assert.equal(unpacked.local, "ext4", `LAYERFS_LOCAL_DIR is on the VM's Store disk: ${JSON.stringify(unpacked)}`); + assert.match(unpacked.digest, /^[0-9a-f]{64}$/, `one image, under its digest: ${JSON.stringify(unpacked)}`); + assert.equal(unpacked.mode, "555", `an unpacked image is read-only: ${JSON.stringify(unpacked)}`); + assert.equal(unpacked.partial, "0", `no half-unpacked image is left behind: ${JSON.stringify(unpacked)}`); + console.log(`the image ${unpacked.digest.slice(0, 12)} is unpacked on the VM's ${unpacked.local} disk: ${unpacked.mode}`); + + // the Sandbox's root really is that image, and one more Sandbox reuses the unpack + const third = await agent("shop", shopEnv, "third"); + assert.equal(third.code, 0, third.text); + console.log(third.text.trim()); + const reused = await disk("shop"); + assert.deepEqual([reused.digest, reused.stat], [unpacked.digest, unpacked.stat], `the same digest was unpacked once: ${JSON.stringify([unpacked, reused])}`); + + const blogRun = await agent("blog", blogEnv, "blog"); + assert.equal(blogRun.code, 0, blogRun.text); + console.log(blogRun.text.trim()); + const otherDisk = await disk("blog"); + assert.equal(otherDisk.local, "ext4", `the other project's VM has its own ext4 disk: ${JSON.stringify(otherDisk)}`); + assert.equal(otherDisk.digest, unpacked.digest, "each VM unpacks the bundle's one image"); + + // a stopped VM is started again, and its worker comes back on the same port + assert.equal((await runtime.vm.stop("shop")).state, "stopped"); + assert.equal((await runtime.vm.status("shop")).state, "stopped"); + const again = await runtime.worker.start("shop"); + assert.equal(again.worker.state, "running", JSON.stringify(again)); + assert.equal(again.vm.worker.port, shopEndpoint.port, "the same VM keeps its published port"); + // the reboot ended the Sandbox's init process but not its image: the cache the + // VM kept is the same directory, and the record it left is reclaimed by the + // resume phase, which then starts a Sandbox on that same unpack + assert.deepEqual((await disk("shop")).stat, unpacked.stat, "the image cache survived the VM restart unchanged"); + // the Sandbox the reboot ended is still a record in the home, not a running Sandbox + const record = join(home, "shop", "sandboxes", left, "sandbox.record"); + assert.ok(existsSync(record), `the Sandbox record survived the VM restart: ${record}`); + console.log("stopped and restarted the project's VM: ok"); + + // the restarted worker takes a new secret, and the project's Commit is still readable + const restarted = await runtime.worker.restart("shop", { confirm: "restart" }); + assert.equal(restarted.worker.state, "running"); + const resumed = await agent("shop", shopEnv, "resume", { AGENT_LEFTOVER: left }); + assert.equal(resumed.code, 0, resumed.text); + console.log(resumed.text.trim()); + assert.ok(!existsSync(record), "the record the previous boot left was reclaimed by the resumed agent"); + + // removing one project takes its own VM and Store disk, and leaves the other alone + const removed = await runtime.project.remove("blog", { confirm: "blog", stopSandboxes: true }); + assert.equal(removed.kind, "Removed"); + assert.equal((await runtime.vm.status("blog")).state, "missing"); + const other = await runtime.vm.status("shop"); + assert.deepEqual([other.project, other.state, other.worker.state], ["shop", "running", "running"], "the other project's VM is untouched"); + const shopAgain = await agent("shop", shopEnv, "resume"); + assert.equal(shopAgain.code, 0, shopAgain.text); + console.log("removed one project, the other still runs: ok"); +} finally { + // whatever the assertions decided: every project still here keeps its VM and Store + // disk to itself, so each one's removal takes exactly those + for (const project of (await runtime?.project.list().catch(() => [])) ?? []) { + const volume = project.volume ?? `${project.name}-store`; + await runtime?.project.remove(project.name, { confirm: project.name, volume, stopSandboxes: true }).catch((error) => console.error(`remove ${project.name}: ${error.message}`)); + await runtime?.vm.delete(project.name, { confirm: "delete", discardStore: true }).catch((error) => console.error(`vm delete ${project.name}: ${error.message}`)); + } + await runtime?.close(); + rmSync(work, { recursive: true, force: true }); +} +console.log(`vm real: ok in ${((Date.now() - started) / 1000).toFixed(1)} s`); diff --git a/api/runtime/test/vm_placement.mjs b/api/runtime/test/vm_placement.mjs new file mode 100644 index 0000000000..641ce82332 --- /dev/null +++ b/api/runtime/test/vm_placement.mjs @@ -0,0 +1,501 @@ +// @layerfs/runtime's `vm:` placement (#515 §6, V9) with a stand-in +// provider CLI and the real worker: no microVM, no Store, no daemon. One Linux +// VM per Store (#515 V1): every case names a project, and its VM, Store disk, +// host port, bundle copy and worker secret are that project's alone. The +// fixture keeps one VM record per project, maps each guest's absolute paths +// under a temp folder, and starts the adapter's bundle worker so step ④'s +// readiness is real. +// node api/runtime/test/vm_placement.mjs settings|flow|two|bundle|delete|hosts|apple-container +import assert from "node:assert/strict"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const FAKE = fileURLToPath(new URL("../../agent/test/fixture/fake-native.cjs", import.meta.url)); +const SOURCE = fileURLToPath(new URL("../src", import.meta.url)); +const MSB = fileURLToPath(new URL("fixture/fake-msb.cjs", import.meta.url)); +const CONTAINER = fileURLToPath(new URL("fixture/fake-container.cjs", import.meta.url)); +const SSH = fileURLToPath(new URL("fixture/fake-ssh.cjs", import.meta.url)); +process.env.LAYERFS_NATIVE = FAKE; +const { LayerFSRuntime } = await import("@layerfs/runtime"); +const { resolveSettings, missing, callHome, GUEST } = await import("../src/settings.js"); +const { deploymentEnv } = await import("../src/clients.js"); +const { endpointFile } = await import("../src/platform/shared.js"); + +const selected = process.argv[2]; +const work = mkdtempSync(join(tmpdir(), "layerfs-vm-")); +const home = join(work, "home"), bundle = join(work, "bundle"), prefix = "lfs-vm-test-"; +/** A project's VM and Store disk names: one VM per Store. */ +const vmName = (project) => `${prefix}${project}-vm`; +const storeName = (project) => `${prefix}${project}-vm-store`; + +/** A stand-in provider CLI on PATH: `name` shells out to the fixture with this state folder. */ +function standIn(name, fixture, state) { + const bin = join(work, "bin"); + mkdirSync(bin, { recursive: true }); + const file = join(bin, name); + writeFileSync(file, `#!/bin/sh\nexec "${process.execPath}" "${fixture}" "$@"\n`); + chmodSync(file, 0o755); + const saved = process.env.PATH; + process.env.PATH = `${bin}:${saved}`; + const envKey = name === "msb" ? "FAKE_MSB" : name === "container" ? "FAKE_CONTAINER" : "FAKE_SSH"; + process.env[`${envKey}_STATE`] = state; + process.env[`${envKey}_NATIVE`] = FAKE; + if (name === "ssh") process.env.LAYERFS_SSH_STATE_DIR = join(state, "sessions"); + return () => { + process.env.PATH = saved; + if (name === "ssh") delete process.env.LAYERFS_SSH_STATE_DIR; + }; +} + +/** A sealed Linux bundle with the adapter's worker and the real SDK packages it imports. + * `image` adds a Sandbox image archive at `images/.tar` (H38's convention, V7). */ +function makeBundle(tag = "build-one", image = null) { + rmSync(bundle, { recursive: true, force: true }); + const api = join(bundle, "api"); + cpSync(join(SOURCE, "worker"), join(api, "runtime/src/worker"), { recursive: true }); + cpSync(join(SOURCE, "../../node_modules/@layerfs"), join(api, "node_modules/@layerfs"), { recursive: true, dereference: true }); + if (image) { + mkdirSync(join(bundle, "images"), { recursive: true }); + writeFileSync(join(bundle, "images", `${image}.tar`), "an uncompressed docker save archive\n"); + } + writeFileSync(join(bundle, "manifest.json"), JSON.stringify({ version: 1, status: "sealed", platform: "linux", build_id: tag, files: {} })); + return bundle; +} + +const calls = (state) => (existsSync(join(state, "calls.jsonl")) ? readFileSync(join(state, "calls.jsonl"), "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line).argv) : []); + +/** Every await that could wait on a provider has a ceiling here, so a stuck + * call fails this case with a name instead of hanging the harness. */ +const deadline = (ms, what, promise) => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error(`${what} did not finish within ${ms / 1000} s`)), ms).unref()), + ]); + +// The whole case has a ceiling too: an unref'd watchdog that exits non-zero +// rather than letting CI or a shell wait forever. +setTimeout(() => { + process.stderr.write(`vm_placement/${selected}: FAIL, the case did not finish in 300 s\n`); + process.exit(1); +}, 300_000).unref(); +/** A guest path inside one project's VM. */ +const guestPath = (state, vm, path) => join(state, "guest", vm, path.replace(/^\//, "")); +/** The project is gone from this home: its entry and the runtime's record of it. */ +const forget = (name) => { + rmSync(join(home, name), { recursive: true, force: true }); + rmSync(join(home, ".runtime", "projects", `${name}.json`), { force: true }); +}; + +/** A project of the home: its entry, and the runtime's record of its volume. */ +const project = (name) => { + mkdirSync(join(home, name), { recursive: true }); + writeFileSync(join(home, name, "project.record"), "x"); + mkdirSync(join(home, ".runtime", "projects"), { recursive: true }); + writeFileSync(join(home, ".runtime", "projects", `${name}.json`), JSON.stringify({ name, volume: `lfs-${name}-store-1`, created: 1 })); +}; +const endpointAt = (name) => JSON.parse(readFileSync(endpointFile(home, name), "utf8")); + +try { + if (selected === "settings") { + const local = resolveSettings({ home, platform: "linux", image: `sha256:${"a".repeat(64)}` }); + assert.deepEqual([local.placement.kind, local.placement.name, local.vm ?? null], ["local", "local", null]); + assert.deepEqual(missing(local), ["daemon"], "a local Linux home needs its daemon only"); + // LAYERFS_PLACEMENT is the V9 name; the settings object wins when both are set + process.env.LAYERFS_PLACEMENT = "vm:microsandbox"; + const vm = resolveSettings({ home, platform: "linux", bundle: "/b", containerPrefix: prefix }); + assert.deepEqual([vm.placement.kind, vm.placement.provider, vm.vm.provider, vm.vm.image], ["vm", "microsandbox", "microsandbox", "node:22-trixie-slim"]); + assert.equal(vm.daemon, GUEST.daemon, "the daemon is the guest's, beside its helper"); + assert.deepEqual([vm.vm.cpus, vm.vm.memory, vm.vm.storeSize], [2, "2G", "20G"]); + assert.equal(callHome(vm), GUEST.home, "LayerFS calls see the VM's home"); + assert.deepEqual(missing(vm), [], "a VM home needs no runtime image and no daemon setting"); + // sizes and cpus come from settings.vm, else LAYERFS_VM_* + process.env.LAYERFS_VM_CPUS = "3"; + process.env.LAYERFS_VM_STORE_SIZE = "64G"; + const sized = resolveSettings({ home, platform: "linux", bundle: "/b" }); + assert.deepEqual([sized.vm.cpus, sized.vm.storeSize], [3, "64G"]); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "vm:nope" }), { code: "Unsupported", message: /vm:microsandbox or vm:apple-container/ }); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "nope" }), { code: "InvalidInput", message: /local.*vm:/ }); + assert.throws(() => resolveSettings({ home, platform: "linux", placement: "vm:microsandbox", vm: { memory: "8" } }), { code: "InvalidInput", message: /memory/ }); + // vm:ssh settings + const sshSettings = resolveSettings({ + home, platform: "linux", bundle: "/b", placement: "vm:ssh", + vm: { host: "10.0.0.1", port: 2222, identityFile: "/path/to/key.pem", storePath: "/mnt/store" } + }); + assert.deepEqual([sshSettings.placement.kind, sshSettings.placement.provider, sshSettings.vm.provider], ["vm", "ssh", "ssh"]); + assert.deepEqual( + [sshSettings.vm.ssh.host, sshSettings.vm.ssh.port, sshSettings.vm.ssh.identityFile, sshSettings.vm.ssh.storePath, sshSettings.vm.ssh.strictHostKeyChecking], + ["10.0.0.1", 2222, "/path/to/key.pem", "/mnt/store", true] + ); + assert.deepEqual(missing(sshSettings), []); + // missing host in vm:ssh + const sshNoHost = resolveSettings({ home, platform: "linux", bundle: "/b", placement: "vm:ssh" }); + assert.deepEqual(missing(sshNoHost), ["host"]); + // password rejected + assert.throws( + () => resolveSettings({ home, platform: "linux", bundle: "/b", placement: "vm:ssh", vm: { host: "10.0.0.1", password: "secret" } }), + { code: "InvalidInput", message: /password/ } + ); + delete process.env.LAYERFS_PLACEMENT; + delete process.env.LAYERFS_VM_CPUS; + delete process.env.LAYERFS_VM_STORE_SIZE; + } else if (selected === "flow") { + const image = "a".repeat(64); + makeBundle("build-one", image); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + assert.equal(existsSync(endpointFile(home, "shop")), false); + // a project's VM is named after the project: there is no home-wide VM + await assert.rejects(runtime.worker.start(), { code: "InvalidInput", message: /project name/ }); + project("shop"); + const started = await runtime.worker.start("shop"); + assert.equal(started.project, "shop"); + assert.equal(started.worker.state, "running"); + assert.deepEqual([started.vm.state, started.vm.store], ["running", storeName("shop")]); + const endpoint = endpointAt("shop"); + assert.equal(endpoint.home, GUEST.home, "the endpoint names the VM's home"); + assert.equal(endpoint.vm.name, vmName("shop")); + assert.equal(endpoint.vm.project, "shop"); + assert.deepEqual(Object.keys(endpoint.vm).sort(), ["name", "project", "provider"]); + assert.equal(process.env.LAYERFS_WORKER_ENDPOINT, undefined, "the endpoint is handed over in-process"); + + const first = calls(state); + const create = first.find((argv) => argv[0] === "create"); + assert.ok(create, "step ① created the project's VM"); + assert.ok(create.includes("--name") && create[create.indexOf("--name") + 1] === vmName("shop")); + assert.ok(create.includes("--mount-named"), "the Store is a named disk volume of its own"); + assert.equal(create[create.indexOf("--mount-named") + 1], `${storeName("shop")}:/layerfs/store:kind=disk,size=20G`); + assert.equal(create[create.indexOf("-v") + 1], `${home}:/layerfs/home`); + assert.equal(create[create.indexOf("-p") + 1], `127.0.0.1:${endpoint.port}:7070`); + assert.equal(create.at(-1), "node:22-trixie-slim"); + assert.ok(first.some((argv) => argv[0] === "copy" && argv[2] === `${vmName("shop")}:/layerfs/bundle`), "step ② copied the bundle into that VM"); + assert.ok(first.some((argv) => argv[0] === "exec" && /max_user_namespaces/.test(argv.join(" "))), "step ③ blocked user namespaces"); + assert.ok(first.some((argv) => argv[0] === "exec" && /nohup node/.test(argv.join(" "))), "step ③ started the worker"); + assert.equal(readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8").includes(endpoint.secret), true, "the secret reached the VM in a file, not in a command line"); + assert.ok(!JSON.stringify(first).includes(endpoint.secret), "no call carried the secret in its arguments"); + + // the Sandbox image the bundle carries (V7, H38's convention): this VM's own copy, + // named for the worker as a guest path, and the deployment's image digest + const workerEnv = readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"); + assert.match(workerEnv, new RegExp(`^LAYERFS_IMAGE=sha256:${image}$`, "m")); + assert.match(workerEnv, new RegExp(`^LAYERFS_IMAGE_ARCHIVE=/layerfs/bundle/images/${image}\\.tar$`, "m")); + assert.equal(existsSync(guestPath(state, vmName("shop"), `/layerfs/bundle/images/${image}.tar`)), true, "the image archive came in with the bundle"); + + // the client side: this project's deployment, and this project's endpoint + const env = deploymentEnv(runtime.settings, runtime.platform, { name: "shop" }); + assert.equal(env.WORKER_ENDPOINT, endpointFile(home, "shop"), "agents an integration launches reach this project's VM"); + assert.equal(env.STORE_PATH, `${GUEST.store}/global/store.sqlite`, "the Store disk is the Store's directory"); + assert.equal(env.SANDBOX_BACKEND, "namespaces"); + assert.equal(env.IMAGE, `sha256:${image}`, "a bundle with an image gives the deployment its digest"); + assert.equal(env.IMAGE_ARCHIVE, `${GUEST.bundle}/images/${image}.tar`, "and the archive in this VM's own copy of the bundle"); + assert.equal(env.LOCAL_DIR, `${GUEST.store}/local`, "Sandbox directories are on the Store disk, never the VM's own root"); + assert.match(readFileSync(join(state, "calls.jsonl"), "utf8"), /\/layerfs\/store\/local/, "the guest's Sandbox directory was made on the Store disk"); + + // the same project again: the VM and its bundle are reused, and only the VM is read + const counted = () => calls(state).filter((argv) => argv[0] === "create" || argv[0] === "copy" || argv[0] === "exec").length; + const before = counted(); + await runtime.worker.start("shop"); + assert.equal(counted(), before, "a running worker with its bundle is reused"); + + // a stopped VM is started, not created + assert.deepEqual(await runtime.vm.stop("shop"), { project: "shop", name: vmName("shop"), state: "stopped" }); + assert.equal(existsSync(endpointFile(home, "shop")), false, "a stopped VM's endpoint is gone"); + const restarted = await runtime.worker.start("shop"); + assert.equal(restarted.worker.state, "running"); + assert.equal(calls(state).filter((argv) => argv[0] === "create").length, 1, "the VM was started, not created again"); + + // status: host Docker is not this home's, and each project's VM is listed + const status = await runtime.status(); + assert.deepEqual([status.placement, status.docker.level], ["vm:microsandbox", "not-used"]); + assert.equal(status.worker, null, "no single worker serves the home"); + assert.deepEqual(status.vm.vms.map((vm) => [vm.project, vm.state, vm.worker.state]), [["shop", "running", "running"]]); + assert.ok(status.disk.volumes > 0, "the project's Store disk has free bytes"); + assert.equal((await runtime.vm.status("shop")).name, vmName("shop")); + assert.deepEqual((await runtime.vm.status()).vms.map((vm) => vm.project), ["shop"]); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "two") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + project("shop"); + project("blog"); + await runtime.worker.start("shop"); + await runtime.worker.start("blog"); + // one VM, one Store disk and one endpoint per project, and they all differ + const shop = endpointAt("shop"), blog = endpointAt("blog"); + assert.notEqual(shop.port, blog.port, "each VM owns its host port"); + assert.notEqual(shop.secret, blog.secret, "each VM has its own worker secret"); + assert.deepEqual([shop.vm.name, blog.vm.name], [vmName("shop"), vmName("blog")]); + const created = calls(state).filter((argv) => argv[0] === "create").map((argv) => argv[argv.indexOf("--mount-named") + 1]); + assert.deepEqual(created.sort(), [`${storeName("blog")}:/layerfs/store:kind=disk,size=20G`, `${storeName("shop")}:/layerfs/store:kind=disk,size=20G`], "each project's Store is on its own disk"); + assert.equal(calls(state).filter((argv) => argv[0] === "copy" && argv[2] === `${vmName("blog")}:/layerfs/bundle`).length, 1, "the bundle is copied into each VM"); + // neither VM's Store disk holds the other's Store: they are different disks + const shopStore = guestPath(state, vmName("shop"), "/layerfs/store/global/store.sqlite"); + mkdirSync(join(shopStore, ".."), { recursive: true }); + writeFileSync(shopStore, "shop"); + assert.equal(existsSync(guestPath(state, vmName("blog"), "/layerfs/store/global/store.sqlite")), false, "the other project's VM has its own Store disk"); + // and one VM's secrets, ports and bundle copy are its own + assert.notDeepEqual(readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"), readFileSync(guestPath(state, vmName("blog"), "/layerfs/state/worker.env"), "utf8")); + const listed = await runtime.vm.status(); + assert.deepEqual(listed.vms.map((vm) => [vm.project, vm.state, vm.worker.port]).sort(), [["blog", "running", blog.port], ["shop", "running", shop.port]]); + // the hook project.remove uses takes that project's VM and disk, and nothing else + // (the whole call, a Sandbox closing included, is vm_real.mjs's, with a real VM) + await runtime.platform.removeStore(runtime, { name: "blog", volume: "lfs-blog-store-1" }); + assert.equal(existsSync(join(state, "vms", `${vmName("blog")}.json`)), false, "the project's VM is gone"); + assert.ok(calls(state).some((argv) => argv[0] === "volume" && argv[1] === "remove" && argv[2] === storeName("blog"))); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), true, "the other project's VM is untouched"); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)) && endpointAt("shop").vm.name, vmName("shop")); + assert.equal((await runtime.worker.start("shop")).worker.state, "running", "and the other project's worker still answers"); + // V8 still holds for the project that is left + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + forget("shop"); + assert.deepEqual(await runtime.vm.delete("shop", { confirm: "delete", discardStore: true }), { project: "shop", name: vmName("shop"), state: "deleted", store: "discarded" }); + assert.ok(calls(state).some((argv) => argv[0] === "volume" && argv[1] === "remove" && argv[2] === storeName("shop"))); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "bundle") { + makeBundle("build-one"); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + await runtime.worker.start("shop"); + const copies = (vm) => calls(state).filter((argv) => argv[0] === "copy" && argv[2] === `${vm}:/layerfs/bundle`).length; + assert.equal(copies(vmName("shop")), 1); + // the same bundle hash: step ② is not repeated + await runtime.worker.start("shop"); + assert.equal(copies(vmName("shop")), 1); + // a second project's VM gets its own copy + await runtime.worker.start("blog"); + assert.equal(copies(vmName("shop")), 1); + assert.equal(copies(vmName("blog")), 1); + // a changed bundle is copied again into the VM that restarts, and the worker reports it + makeBundle("build-two"); + const restarted = await runtime.worker.start("shop"); + assert.equal(copies(vmName("shop")), 2, "a changed bundle hash is copied once more"); + assert.equal(restarted.worker.build, "build-two"); + assert.equal(readFileSync(guestPath(state, vmName("shop"), "/layerfs/bundle/.sealed"), "utf8").trim().length, 64); + // a bundle without a Sandbox image sets neither: the backend needs none there + const plain = readFileSync(guestPath(state, vmName("shop"), "/layerfs/state/worker.env"), "utf8"); + assert.ok(!/LAYERFS_IMAGE/.test(plain), "no image is named when the bundle carries none"); + assert.equal(copies(vmName("blog")), 1, "the other project's VM keeps its copy until it restarts"); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "delete") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + await runtime.worker.start("shop"); + // V8: a Store that lives only on the project's VM disk is never destroyed by a delete + project("shop"); + await assert.rejects(runtime.vm.delete("shop"), { code: "InvalidInput", message: /confirm/ }); + await assert.rejects(runtime.vm.delete("shop", { confirm: "delete" }), { code: "Unsupported", message: /never destroys a Store that was not pushed/ }); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), true, "the VM is untouched by the refusal"); + // with no project left, the VM record goes and its Store disk stays unless asked + forget("shop"); + assert.deepEqual(await runtime.vm.delete("shop", { confirm: "delete" }), { project: "shop", name: vmName("shop"), state: "deleted", store: "kept" }); + assert.equal(existsSync(join(state, "vms", `${vmName("shop")}.json`)), false); + assert.equal(calls(state).some((argv) => argv[0] === "volume"), false, "the Store disk is kept by default"); + await runtime.vm.delete("shop", { confirm: "delete", discardStore: true }); + // a store whose VM is gone answers "missing" rather than inventing one + assert.equal((await runtime.vm.delete("shop", { confirm: "delete" })).state, "missing"); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "hosts") { + makeBundle(); + const state = join(work, "msb"); + mkdirSync(state); + const restore = standIn("msb", MSB, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:microsandbox", containerPrefix: prefix, platform: "linux", sweep: false }); + try { + project("shop"); + await runtime.worker.start("shop"); + // host Docker describes another machine: the Docker-shaped helpers refuse by name + await assert.rejects(runtime.sandbox.list(), { code: "Unsupported", message: /runtime\.sandbox\.list reads host Docker.*project\.list_sandboxes/ }); + await assert.rejects(runtime.sandbox.list_leftovers(), { code: "Unsupported", message: /Linux namespaces inside the VM/ }); + await assert.rejects(runtime.volume.list(), { code: "Unsupported", message: /directories on the VM's disk/ }); + assert.deepEqual(await runtime.sandbox.sweep(), { reclaimed: {}, failed: [], host: false }); + // status lists a project's VM even before it was started + project("late"); + assert.deepEqual((await runtime.vm.status()).vms.map((vm) => [vm.project, vm.state]), [["late", "missing"], ["shop", "running"]]); + // a local home has no vm group at all + const local = await LayerFSRuntime.open({ home: join(work, "local"), image: `sha256:${"a".repeat(64)}`, daemon: "/bin/daemon", platform: "linux", sweep: false }); + assert.equal(local.vm, undefined); + await local.close(); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "apple-container") { + makeBundle(); + const state = join(work, "container"); + mkdirSync(state); + const restore = standIn("container", CONTAINER, state); + const runtime = await LayerFSRuntime.open({ home, bundle, placement: "vm:apple-container", containerPrefix: prefix, platform: "macos", sweep: false }); + try { + const started = await runtime.worker.start("shop"); + assert.equal(started.vm.state, "running"); + const first = calls(state); + // the Store volume is made first, at the configured size, and mounted by name + const volume = first.find((argv) => argv[0] === "volume" && argv[1] === "create"); + assert.ok(volume, "the Store volume is made before the VM"); + assert.equal(volume[volume.indexOf("-s") + 1], "20G"); + assert.equal(volume.at(-1), storeName("shop")); + const run = first.find((argv) => argv[0] === "run"); + assert.ok(run, "the VM was created with `container run`"); + const capabilities = run.flatMap((value, index) => (value === "--cap-add" ? [run[index + 1]] : [])); + assert.deepEqual(capabilities, ["SYS_ADMIN", "SYS_RESOURCE", "NET_ADMIN"], "#512's capabilities for FUSE, cgroups and loopback"); + assert.ok(run.includes("--publish")); + assert.equal(run[run.indexOf("--mount") + 1], `type=volume,source=${storeName("shop")},target=/layerfs/store`); + // the guest's /proc/sys is read-only there: the block remounts it first + const block = first.find((argv) => argv[0] === "exec" && /max_user_namespaces/.test(argv.join(" "))); + assert.match(block.join(" "), /remount,rw \/proc\/sys/); + assert.ok(first.some((argv) => argv[0] === "exec" && /nohup node/.test(argv.join(" ")))); + // the verified 1.6.1 inspect shape: the adapter reads this VM's state, store and home + const entry = await runtime.vm.status("shop"); + assert.deepEqual([entry.state, entry.store, entry.home, entry.worker.state], ["running", storeName("shop"), home, "running"]); + // copying the bundle into an Apple Container VM is `container copy HOST NAME:DEST` + assert.ok(first.some((argv) => argv[0] === "copy" && argv[2] === `${vmName("shop")}:/layerfs/bundle`)); + } finally { + restore(); + await runtime.close(); + restoreWorker(work); + } + } else if (selected === "ssh") { + makeBundle(); + const state = join(work, "ssh"); + mkdirSync(state); + const restore = standIn("ssh", SSH, state); + const runtime = await LayerFSRuntime.open({ + home, bundle, placement: "vm:ssh", containerPrefix: prefix, platform: "linux", sweep: false, + vm: { host: "127.0.0.1", port: 22, storePath: "/layerfs/store" }, + }); + try { + const started = await deadline(60_000, "worker.start", runtime.worker.start("shop")); + assert.equal(started.vm.state, "running"); + assert.equal(started.worker.state, "running"); + const first = calls(state); + assert.ok(first.some((argv) => argv.includes("uname")), "the host's kernel is read before anything else"); + assert.ok(first.some((argv) => argv.includes("findmnt")), "the Store path's filesystem is checked"); + assert.ok(first.some((argv) => argv.includes("-N") && argv.includes("-L")), "the private stream is an ssh -L forward"); + assert.ok(first.some((argv) => argv.includes("StrictHostKeyChecking=yes")), "host key checking stays on"); + assert.ok(!first.some((argv) => argv.includes("StrictHostKeyChecking=no")), "never turned off behind the user's back"); + assert.ok(first.some((argv) => argv.some((value) => typeof value === "string" && value.includes("tar -C"))), "the bundle arrives as one tar stream"); + const entry = await deadline(30_000, "vm.status", runtime.vm.status("shop")); + assert.deepEqual([entry.state, entry.worker.state], ["running", "running"]); + // the endpoint the worker was written at is this host's forwarded port + const endpoint = endpointAt("shop"); + assert.equal(endpoint.vm.provider, "ssh"); + assert.ok(endpoint.port > 1024); + // a second start is served by the same forward: no second copy, no second forward + await deadline(30_000, "worker.start again", runtime.worker.start("shop")); + assert.equal(calls(state).filter((argv) => argv.includes("-N")).length, 1, "one forward per VM"); + // stop ends the forward; the record stays and start re-opens it + assert.equal((await deadline(30_000, "vm.stop", runtime.vm.stop("shop"))).state, "stopped"); + assert.equal((await deadline(30_000, "status after stop", runtime.vm.status("shop"))).state, "stopped"); + const again = await deadline(60_000, "worker.start after stop", runtime.worker.start("shop")); + assert.deepEqual([again.vm.state, again.worker.state], ["running", "running"]); + } finally { + await runtime.close(); + restore(); + restoreWorker(work); + } + } else if (selected === "cleanup") { + // H39v's two findings: a provider create that fails must leave nothing + // behind — no Store volume for Apple Container (the CLI makes one before it + // fails on a busy port), and no hidden sandbox directory for Microsandbox. + const containerState = join(work, "container"); + mkdirSync(containerState); + const restoreContainer = standIn("container", CONTAINER, containerState); + try { + const container = (await import("../src/platform/vm/apple-container.js")).default; + await container.ensureStore("lfs-vm-test-clean-store", "20G"); + process.env.FAKE_CONTAINER_FAIL_RUN = "port"; + await assert.rejects(container.create({ + name: "lfs-vm-test-clean", image: "node:22-trixie-slim", cpus: 2, memory: "2G", + storeName: "lfs-vm-test-clean-store", storeSize: "20G", home, hostPort: 18080, workerPort: 7070, + }), /port is already allocated/); + assert.equal(existsSync(join(containerState, "volumes", "lfs-vm-test-clean-store.json")), false, "the Store volume this create made is gone"); + assert.equal(existsSync(join(containerState, "vms", "lfs-vm-test-clean.json")), false, "and no VM record"); + } finally { + delete process.env.FAKE_CONTAINER_FAIL_RUN; + restoreContainer(); + } + const msbState = join(work, "msb"); + mkdirSync(msbState); + const restoreMsb = standIn("msb", MSB, msbState); + const hidden = join(tmpdir(), ".microsandbox-clean-check"); + try { + const msb = (await import("../src/platform/vm/microsandbox.js")).default; + const name = `lfs-vm-test-hidden-${process.pid}`; + const dir = join(process.env.HOME ?? join(tmpdir(), "home"), ".microsandbox", "sandboxes", name); + process.env.FAKE_MSB_FAIL_CREATE = "mount"; + await assert.rejects(msb.create({ + name, image: "node:22-trixie-slim", cpus: 2, memory: "2G", storeName: "lfs-vm-test-hidden-store", + storeSize: "20G", home, hostPort: 18081, workerPort: 7070, + }), /host path for one of the mounts/); + assert.equal(existsSync(dir), false, `the hidden sandbox directory ${dir} the failed create made is gone`); + } finally { + delete process.env.FAKE_MSB_FAIL_CREATE; + restoreMsb(); + rmSync(hidden, { recursive: true, force: true }); + } + } else { + throw new Error("select settings, flow, two, bundle, delete, hosts, apple-container, ssh or cleanup"); + } +} finally { + rmSync(work, { recursive: true, force: true }); +} +console.log(`PASS vm_placement/${selected}`); +// The run is over: exit explicitly so a stand-in child that outlived its case +// cannot hold this process open. `restoreWorker` already ended them. +process.exit(0); + +/** The stand-in's worker outlives its CLI: end it from the pid the fixture wrote. */ +function restoreWorker(folder) { + for (const provider of ["msb", "container", "ssh"]) { + const guests = join(folder, provider, "guest"); + if (existsSync(guests)) { + for (const vm of readdirSync(guests)) { + for (const file of [join(guests, vm, "layerfs/state/worker.pid"), join(guests, "layerfs/state/worker.pid")]) { + if (!existsSync(file)) continue; + try { process.kill(Number(readFileSync(file, "utf8").trim())); } catch { /* already gone */ } + } + } + } + const sessions = join(folder, provider, "sessions"); + if (existsSync(sessions)) { + for (const file of readdirSync(sessions)) { + try { + const session = JSON.parse(readFileSync(join(sessions, file), "utf8")); + if (session.forwardPid) process.kill(session.forwardPid); + } catch {} + } + } + } +} diff --git a/core/docs/architecture/nfs-native-lifecycle.md b/core/docs/architecture/nfs-native-lifecycle.md new file mode 100644 index 0000000000..775159664e --- /dev/null +++ b/core/docs/architecture/nfs-native-lifecycle.md @@ -0,0 +1,142 @@ +# Native NFS helper and local Sandbox teardown + +Status: implemented on the macOS native adaptation branch. This describes the +NFS route in `layerfs-daemon`, not the upstream FUSE force-drain protocol in +[forced teardown](80-forced-teardown.md). No persistence profile, wire format, +mount option or Store interface changes here. + +## Helper ownership and deadlines + +The helper budget remains 60 seconds plus five seconds for post-KILL exit +observation. A 15-second cutoff is not qualified: a normal slow writeback may +exceed it, and interrupting a helper cannot prove its kernel effect was undone. +Removing Sandbox ownership from the wait resolves the serialization problem +without trading it for earlier unknown outcomes. + +[`control/export_helper.rs`](../../crates/layerfs-daemon/src/control/export_helper.rs) +owns each NFS mount, unmount, force and optional flush helper attempt. The +public `Helper` and `HELPER_WAIT` exports remain available from `control`. +One attempt gets 60 seconds of exit observation. At expiry or an observation +error it sends one KILL and polls `try_wait` for at most five more seconds. +It never calls an unconditional blocking `wait` after KILL: macOS can leave an +unmount helper in kernel wait where a signal does not establish process exit. + +An unobserved exit is an error with PID, KILL result and retained-child detail. +The process-wide export serving owner retains the `Child`; a later helper call +can reap an exit already observed with `try_wait`, but never signals or reruns +that attempt. No claim is made that the kernel released a mount or that KILL +rolled back a helper's effects. At daemon exit, process ownership passes to the +operating system as for any remaining child. + +Output pipes are nonblocking, drained in bounded chunks during observation and +capped at 2048 retained bytes. An exited helper's descendant can keep a pipe +open, so EOF is not a completion requirement. Output beyond the cap is discarded; +continuous output cannot postpone the exit deadline. A normal nonzero helper +answer containing `busy` keeps the existing reversible Busy disposition. An +interrupted attempt is never classified Busy from partial diagnostic output. + +The control layer keeps a timed-out unmount's export and engine mount at +`Retained/Detach`; subsequent requests report the same custody without replay. +A timed-out **mount** may already have reached the kernel, so Attach also retains +the serving export, engine mount and directory rather than revoking or deleting +through an unknown client mount. No Ready acknowledgement is issued for it. + +## Discovery ownership wait + +The Node binding registers a Workspace-scoped unmount intent under Sandbox +`ownership.lock`, then drops Sandbox ownership before waiting for the daemon. +The existing NAPI worker waits synchronously on one control connection: no +new background helper or client process is introduced. The daemon still owns +its helper, reap deadline and export custody. Another Workspace can use another +pooled control connection; Sandbox discovery reads also proceed during the wait. + +`api/binding/src/unmount.rs` owns a private `workspace-.unmount` result +record and an exclusive nonblocking `workspace-.unmount-lock` flock. The +lease spans registration, the lock-free exchange and publication. It is explicitly +unlocked on drop, including when another fork briefly inherited its descriptor. +Lock order is Sandbox guard → nonblocking Workspace lease at admission; on +completion it publishes the known result using only its Workspace lease, then +reacquires the Sandbox guard for shared-record cleanup. A cleanup failure adds +`cleanupError` to a known successful response rather than replacing success with +Busy. Close checks the lease even for a published result, so it cannot delete or +reuse the Sandbox directory while that cleanup is in progress. Observers use +nonblocking shared flock probes, so they neither wait on the Workspace lease nor +mistake another observer for the exclusive operation owner. +The shared discovery readiness deadline is back to 60 seconds; it no longer +needs to cover the 60+5-second helper wait. Other long discovery operations or +control-pool exhaustion can still cause waiting; this is not a global latency +promise for all calls. + +State transitions: + +- No intent → Unmounting (path and start time saved before sending a request). +- Unmounting → Unmounted after known daemon success (including an explicit + Missing answer); shared-record cleanup follows. Repeated unmount reads the + saved result without sending another request. +- Busy/Dirty → no intent, preserving their reversible semantics. +- A control connection failure before the unmount exchange removes the intent; + no operation was sent. Other failure → Retained with the original refusal + saved, never replayed. +- Unmounting with no lease holder → Unknown/Retained, including client crash + before send or after daemon completion. Observation cannot decide whether an + effect occurred. It neither removes the Workspace nor repeats the request. + +The public method signatures and daemon protocol are unchanged. During active +unmount, existing `Busy` refusals carry `detail.state = "Unmounting"` and +`elapsedMs`; owner loss uses existing `Unknown` with `state = "Retained"`. +Workspace token/directory consumers and backend reads check this state before +sending a call; Sandbox close refuses an active intent. A new mount creates a +different Workspace and remains admissible. This detail shape and completed +unmount replay behavior need maintainer alignment before upstream publication. + +Terminal result records remain until Sandbox deletion; they are bounded by +that Sandbox's lifetime rather than silently evicted. Known token lookups use a +direct filename, not a scan of completed records; unknown-path recovery and +Sandbox close scan the saved records. An explicit Sandbox close can take over an +orphaned or terminal Retained unmount while the daemon is still healthy, provided +no unrelated live Workspace remains. It performs the existing Stop/Delete, +never a replay of the uncertain Workspace operation. If the daemon dies, the +existing stopped-Sandbox reclaim path owns physical detach and record removal. +Stop still detaches before daemon termination, and Delete retains its checked +detach fence. A failed result-file write is reported as cleanupError on known +success; durable recovery of that lost result is not claimed. + +## Stop before daemon termination + +[`backend/native/sandbox.rs`](../../crates/layerfs-sandbox/src/backend/native/sandbox.rs) +attempts detach below that Sandbox's own directory before TERM/KILL, while its +NFS server can still answer client writeback. A detach failure is logged and +does not prevent the existing daemon and process-group stop. Stop remains crash +teardown, with no Commit or graceful daemon-wide drain. It does not delete the +Sandbox directory. Delete retains its checked detach before directory removal; +failed detach never authorizes removal through a mount. + +Linux continues using `MNT_DETACH`, deepest mount first; macOS uses one +`umount -f` per mount, deepest first. The macOS force helper likewise gets a +60-second observation and a five-second bounded post-KILL reap. Unreaped children +are retained, and failure includes the exact PID and observation rather than +claiming a successful detach. The mount table is checked after ordinary helper +completion. No global NFS operation or cleanup of another Sandbox is performed. + +## Evidence and limits + +- `layerfs-nfs/tests/helper_deadline.rs` drives the real control Service with + process helpers: descendant-held pipe, verbose Busy answer, stopped helper + crossing the real 60-second deadline and repeated Retained requests. +- `layerfs-sandbox/tests/local_stop_mount.rs` is an opt-in macOS test using an + explicitly supplied disposable Sandbox. It proves Stop removes the mount + before Delete and leaves the directory. It never selects an existing user + Sandbox automatically. The normal suite lists this test as ignored. +- `api/binding/tests/unmount-concurrency.mjs` requires explicitly supplied + disposable paths and binaries. It runs real NFS with a paused helper, checks + another Workspace and discovery read below one second, checks same-Workspace + refusals and cross-process progress, reversible Busy/Dirty, saved failures, + publication while discovery cleanup is blocked, and explicit close takeover + after client-owner crash or retained failure. +- These tests cannot simulate an unkillable kernel U-state process: userspace + cannot ignore SIGKILL. Bounded observation of an unreaped U-state helper is + source-checked; the original long kernel hang has not been reproduced. +- A retained helper is reaped opportunistically on a later helper invocation, + not by a background reaper. Helper cleanup and kernel resource recovery remain + distinct facts. Stop may take the sum of deadlines over several mounts; + there is no promise that a multi-mount stop completes in one helper budget. diff --git a/engine/crates/layerfs-api/PROVIDERS.md b/engine/crates/layerfs-api/PROVIDERS.md new file mode 100644 index 0000000000..16bd8b3cf9 --- /dev/null +++ b/engine/crates/layerfs-api/PROVIDERS.md @@ -0,0 +1,13 @@ +# Sandbox runtimes in the SDK + +Status: Current general guide. Native macOS is implemented and host-verified; Linux native is implemented, pending Linux host qualification; the Docker runtime is the default. `layerfs-sandbox` selects between them with the concrete `Runtime` enum; there are no provider traits. + +`layerfs-sandbox::backend` exposes `Runtime` (a deployment's choice: `Docker`, `MacosNative`, `LinuxNative`, `Windows`) and `Sandbox` (the launched owner of one Sandbox). `layerfs_sandbox::{Runtime, Sandbox}` re-export both; `layerfs_sandbox::provider` keeps the inputs and records they share. `ExecRequest` lives in the shared types layer; its Docker re-export remains compatible. `SandboxId` aliases the existing `ContainerId`, preserving encoded identities and records. The runtime layer has no daemon control dependency. + +`Runtime::MacosNative(..)` and `Runtime::LinuxNative(..)` wrap `NativeMacosProvider::new(root)` and `NativeLinuxProvider::new(root)`, the two aliases of the same native implementation whose instances share the existing local runtime owner. Each refuses the wrong platform before root access. macOS uses Darwin process facts, the `mount` table and `umount -f`; Linux uses the retained #402 `/proc` PID/start/boot checks, process groups and lazy `MNT_DETACH`. Linux requires a root daemon and a root directory traversable by command users; ordinary commands keep #402's nonroot identity (Node selects 501:20) and fixed container PATH. macOS keeps the caller identity, inherited PATH and loopback NFS. The provider explicitly selects `LAYERFS_KERNEL=fuse` on Linux and `nfs` on macOS and rejects conflicting launch overrides. Launch preserves the original single-attempt process lifecycle, native identity, inherited PATH, NFS environment and partial owner on failure. `inspect` reports exact process liveness; unknown exit/storage facts stay absent. `Runtime::Windows` refuses every operation with `RuntimeError::Unsupported` before any effect; no Windows host runtime exists. Records remain the Node binding's responsibility; runtimes neither scan directories nor infer project ownership. B8 list aggregation and doctor remain future work. + +`SandboxApi::with_runtime(Arc)` and `create_provider_file(ProviderCreate, store, manifest)` (or `create(ProviderCreate, project)`) launch through the deployment's runtime and return `ProviderSandbox`, whose `runtime` field is the `Sandbox` handle. Both Docker (via `Runtime::Docker(DockerProvider)`) and native runtimes route through this single unified API path. `WorkspaceApi::exec` delegates ordinary execution through `sandbox.runtime.run(...)` with streaming stdout/stderr buffers, and `api.close` drains and removes instances via `sandbox.runtime.stop(1)` and `sandbox.runtime.delete()`. Before launch, API validates authentication identities, selected Store, command identity and local resource ownership: overlay and mount home must belong to the instance, Store must live outside it, and native listen uses dynamic loopback. API encodes daemon configuration, then uses the shared TCP/Noise/Hello and Store-install pipeline in `sandbox/connection.rs`. Failures retain their phase, original cause, runtime, control, observed status, installation acknowledgement and handshake work. No automatic retry or cleanup occurs in API. + +`layerfs-node` builds a `Runtime` from `deploy.sandbox` (`docker` by default, `native` on Linux and macOS), keeps the launched `Sandbox` in its Sandbox state, and reads records, image, scratch directory and background-process tracking through it. Legacy deprecated forwarders (`SandboxApi::new(Docker)`, `SandboxCreate`, `ManagedSandbox`, `from_managed`, `internal::split/exec/control_managed`) remain removed. + +The Store remains explicitly Disposable/WAL/OFF. Native macOS keeps #427's direct Store access without a worker; this refactor does not change persistence authorization or filesystem semantics. diff --git a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs index c2e8f73671..cf2fd9aeed 100644 --- a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs +++ b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs @@ -4,8 +4,8 @@ //! registered with the daemon, read the complete root and mutate it through //! plain syscalls. use layerfs_api::{ - InitRequest, ManagedSandbox, MountedWorkspace, OperationCause, ProjectApi, SandboxApi, - SandboxCreate, WorkspaceApi, + InitRequest, MountedWorkspace, OperationCause, ProjectApi, ProviderCreate, ProviderSandbox, + SandboxApi, WorkspaceApi, }; use layerfs_bridge::{ control::{ControlCode, DaemonPhase, MountPhase, Reply, Request}, @@ -14,7 +14,8 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ @@ -22,6 +23,7 @@ use std::{ fs, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, + sync::Arc, time::{Duration, Instant}, }; @@ -100,59 +102,46 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -type Running = ( - layerfs_sandbox::backend::docker::RuntimeOutput, - layerfs_sandbox::backend::docker::ExecHandle, -); enum Directory<'a> { Mounted(&'a MountedWorkspace), Kernel(&'a str), } -/// Mounted commands use SDK exec; outside-mount commands retain ordinary Sandbox execution. -fn start( - owner: &ManagedSandbox, +fn run_dir( + runtime: &layerfs_sandbox::Sandbox, + identity: layerfs_sandbox::CommandIdentity, script: &str, - directory: Directory<'_>, -) -> Result> { - let arguments = vec!["bash".into(), "-c".into(), script.into()]; - let created = match directory { - Directory::Mounted(mounted) => WorkspaceApi::exec(owner, mounted, arguments, vec![], false), - Directory::Kernel(directory) => layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - arguments, - vec![], - directory.into(), - false, - ), - } - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - Ok((output, handle)) -} -fn finish((output, handle): Running) -> Result, Box> { - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - output - .copy_to(&mut stdout, &mut stderr) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - if inspected.known_root_exit() != Some(0) { + directory: &str, +) -> Result, String> { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = runtime + .run( + &layerfs_sandbox::ExecRequest { + identity, + arguments: vec!["bash".into(), "-c".into(), script.into()], + environment: vec![], + directory: directory.into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("exec {f:?}"))?; + if exit.code != 0 { return Err(format!( - "command {inspected:?}: {} {}", + "command exit {exit:?}: {} {}", String::from_utf8_lossy(&stdout), String::from_utf8_lossy(&stderr) - ) - .into()); + )); } Ok(stdout) } -fn run( - owner: &ManagedSandbox, - script: &str, - directory: Directory<'_>, -) -> Result, Box> { - finish(start(owner, script, directory)?) +fn run(owner: &ProviderSandbox, script: &str, directory: Directory<'_>) -> Result, String> { + let dir = match directory { + Directory::Mounted(mounted) => &mounted.ready.directory, + Directory::Kernel(dir) => dir, + }; + run_dir(owner.runtime.as_ref(), owner.command_identity, script, dir) } fn main() -> Result<(), Box> { let began = Instant::now(); @@ -191,22 +180,30 @@ fn main() -> Result<(), Box> { fs::remove_dir_all(directory.join("source"))?; let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; - let api = SandboxApi::new(Docker::new(&args[1], Duration::from_secs(3))?); + let docker = Docker::new(&args[1], Duration::from_secs(3))?; + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); let mut owner = api .create( - SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "kernel-mount-test".into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup: setup(daemon.private, keys.public), - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }, &project, ) @@ -323,45 +320,54 @@ fn main() -> Result<(), Box> { // Reversible normal unmount: a running command's working directory keeps // the kernel busy. The filesystem neither kills nor waits for the command. - let holder = start( - &owner, - "sleep 1.2", - Directory::Kernel(&format!("{root}/node_modules")), - )?; - run( - &owner, - &format!( - "until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{root}/node_modules'; do sleep 0.02; done" - ), - Directory::Kernel("/"), - )?; - let busy = WorkspaceApi::new(&mut owner) - .unmount(mounted.bound.token, false) - .unwrap_err(); - match &busy.cause { - OperationCause::Remote(refusal) => { - assert_eq!( - (refusal.code, refusal.phase.as_str()), - (ControlCode::Busy, "unmount:kernel") - ); + let node_modules_dir = format!("{root}/node_modules"); + let identity = owner.command_identity; + let (runtime, control) = layerfs_api::internal::provider_parts(&mut owner); + std::thread::scope(|s| { + let holder = s.spawn(|| run_dir(runtime, identity, "sleep 1.2", &node_modules_dir)); + run_dir( + runtime, + identity, + &format!("until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{node_modules_dir}'; do sleep 0.02; done"), + "/", + ) + .unwrap(); + let busy = WorkspaceApi::from_control(control) + .unmount(mounted.bound.token, false) + .unwrap_err(); + match &busy.cause { + OperationCause::Remote(refusal) => { + assert_eq!( + (refusal.code, refusal.phase.as_str()), + (ControlCode::Busy, "unmount:kernel") + ); + } + other => panic!("{other:?}"), } - other => panic!("{other:?}"), - } - let status = WorkspaceApi::new(&mut owner).status(mounted.bound.token)?; - let kernel = status.kernel.expect("kernel status block"); - assert_eq!(kernel.phase, MountPhase::Ready); - assert_eq!( - kernel.ready.as_ref(), - Some(&mounted.ready), - "same connection" - ); - assert_eq!( - run(&owner, "cat .git/index", Directory::Mounted(&mounted))?, - FILES[1].1 - ); - finish(holder)?; - println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); - mark("busy"); + let status = WorkspaceApi::from_control(control) + .status(mounted.bound.token) + .unwrap(); + let kernel = status.kernel.expect("kernel status block"); + assert_eq!(kernel.phase, MountPhase::Ready); + assert_eq!( + kernel.ready.as_ref(), + Some(&mounted.ready), + "same connection" + ); + assert_eq!( + run_dir( + runtime, + identity, + "cat .git/index", + &mounted.ready.directory + ) + .unwrap(), + FILES[1].1 + ); + holder.join().unwrap().unwrap(); + println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); + mark("busy"); + }); // Terminal unmount: detach, loop joins, drain, revocation and Close. WorkspaceApi::new(&mut owner) diff --git a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs index f54c059ff8..f6958361ab 100644 --- a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs +++ b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs @@ -1,6 +1,6 @@ //! Positive-path host Init -> protected owned Sandbox -> direct shared Store control proof. //! Typed failure custody is covered separately by public adapter tests; this example reports failures in stderr. -use layerfs_api::{InitRequest, ManagedSandbox, ProjectApi, SandboxApi, SandboxCreate}; +use layerfs_api::{InitRequest, ProjectApi, ProviderCreate, ProviderSandbox, SandboxApi}; use layerfs_bridge::{ control::{DaemonPhase, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, @@ -8,13 +8,15 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ fs, io::Write, path::PathBuf, + sync::Arc, time::{Duration, Instant}, }; fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { @@ -39,25 +41,26 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { - let command="set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; - let created = layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - vec!["bash".into(), "-c".into(), command.into()], - vec![], - "/".into(), - false, - ) - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - let progress = output - .copy_to(&mut std::io::stdout(), &mut std::io::stderr()) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - println!("ORDINARY_RESULT {:?} {:?}", progress, inspected); - assert_eq!(inspected.known_root_exit(), Some(0)); +fn ordinary(owner: &ProviderSandbox) -> Result<(), Box> { + let command = "set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; + let mut stdout = std::io::stdout(); + let mut stderr = std::io::stderr(); + let exit = owner + .runtime + .run( + &layerfs_sandbox::ExecRequest { + identity: layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), command.into()], + environment: vec![], + directory: "/".into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("run {f:?}"))?; + println!("ORDINARY_RESULT {:?}", exit); + assert_eq!(exit.code, 0); Ok(()) } fn main() -> Result<(), Box> { @@ -96,23 +99,30 @@ fn main() -> Result<(), Box> { let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; let docker = Docker::new(&args[1], Duration::from_secs(3))?; - let api = SandboxApi::new(docker); - let make = |setup: DaemonSetup| SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); + let make = |name: &str, setup: DaemonSetup| ProviderCreate { + launch: LaunchRequest { + name: name.into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup, - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }; let mut first = api - .create(make(setup(daemon.private, keys.public)), &project) + .create(make("first", setup(daemon.private, keys.public)), &project) .map_err(|f| format!("first {f:?}"))?; println!( "OWNED_FIRST_CONTAINER {} phase={:?}", @@ -131,7 +141,7 @@ fn main() -> Result<(), Box> { let mut second_setup = setup(daemon.private, keys.public); second_setup.existing_store = Some(installed.manifest.clone()); let second = api - .create(make(second_setup), &project) + .create(make("second", second_setup), &project) .map_err(|f| format!("second {f:?}"))?; println!( "OWNED_SECOND_CONTAINER {} phase={:?}", @@ -161,7 +171,7 @@ fn main() -> Result<(), Box> { let mut reopened_setup = setup(daemon.private, keys.public); reopened_setup.existing_store = Some(installed.manifest.clone()); let mut reopened = api - .create(make(reopened_setup), &project) + .create(make("reopened", reopened_setup), &project) .map_err(|f| format!("reopen {f:?}"))?; println!( "OWNED_REOPEN_CONTAINER {} phase={:?}", diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 7bffa705c0..108b2f0251 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -98,59 +98,40 @@ pub fn into_connection(control: Control) -> Connection { /// Original authenticated control owner, without a reconnect or reset. #[cfg(unix)] -pub fn control(sandbox: &mut crate::ManagedSandbox) -> &mut Control { +pub fn control(sandbox: &mut crate::sandbox::ProviderSandbox) -> &mut Control { &mut sandbox.control } -/// The runtime, control owner and command identity of one Sandbox, taken -/// apart for an owner that serves several Workspaces at once: commands run on -/// the runtime while the control owner answers one request at a time. +/// Transfers provider runtime, original channel and admitted command identity. #[cfg(unix)] -pub fn split( - sandbox: crate::ManagedSandbox, +pub fn split_provider( + sandbox: crate::sandbox::ProviderSandbox, ) -> ( - layerfs_sandbox::backend::docker::Sandbox, + Box, Control, layerfs_sandbox::CommandIdentity, ) { (sandbox.runtime, sandbox.control, sandbox.command_identity) } +/// Borrows the same provider instance and original channel for concurrent execution. +#[cfg(unix)] +pub fn provider_parts( + sandbox: &mut crate::sandbox::ProviderSandbox, +) -> (&layerfs_sandbox::Sandbox, &mut Control) { + (sandbox.runtime.as_ref(), &mut sandbox.control) +} /// Original one-attempt startup used by existing-Store deployments and lifecycle proofs. #[cfg(unix)] pub fn start( api: &crate::SandboxApi, - request: crate::SandboxCreate, -) -> Result> { - crate::sandbox::create::create(&api.docker, request) + request: crate::sandbox::ProviderCreate, +) -> Result> { + api.start(request) } /// Existing bounded startup observation for control-channel qualification. #[cfg(unix)] pub fn sandbox_status( - sandbox: &mut crate::ManagedSandbox, + sandbox: &mut crate::sandbox::ProviderSandbox, wait_for_store: bool, ) -> Result> { sandbox.status(wait_for_store) } - -/// Disjoint original runtime/control borrows for concurrent Workspace orchestration. -#[cfg(unix)] -pub fn parts( - sandbox: &mut crate::ManagedSandbox, -) -> (&layerfs_sandbox::backend::docker::Sandbox, &mut Control) { - (&sandbox.runtime, &mut sandbox.control) -} - -/// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. -#[cfg(unix)] -pub fn exec( - sandbox: &layerfs_sandbox::backend::docker::Sandbox, - identity: layerfs_sandbox::CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result< - layerfs_sandbox::backend::docker::ExecCreated, - Box, -> { - crate::sandbox::exec::launch(sandbox, identity, arguments, environment, directory, stdin) -} diff --git a/engine/crates/layerfs-api/src/lib.rs b/engine/crates/layerfs-api/src/lib.rs index e652d32640..1641980cd8 100644 --- a/engine/crates/layerfs-api/src/lib.rs +++ b/engine/crates/layerfs-api/src/lib.rs @@ -27,7 +27,7 @@ pub use layerfs_project::store::{ pub use layerfs_project::{InitCounters, InitCounts, InitStep, ProjectError, SourceRefusal}; #[cfg(unix)] pub use sandbox::{ - connect_address, ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, + connect_address, ProviderCreate, ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, }; pub use sandbox::{BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi}; diff --git a/engine/crates/layerfs-api/src/sandbox/close.rs b/engine/crates/layerfs-api/src/sandbox/close.rs index e0a57ec08a..5f55414a9e 100644 --- a/engine/crates/layerfs-api/src/sandbox/close.rs +++ b/engine/crates/layerfs-api/src/sandbox/close.rs @@ -1,10 +1,10 @@ //! Explicit stop then delete of the exact acknowledged Sandbox runtime owner. -use super::{ManagedSandbox, SandboxApi, SandboxCause, SandboxFailure, SandboxPhase}; +use super::{ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase}; impl SandboxApi { /// Stops once, then deletes once after acknowledged stop. No Workspace drain is inferred. /// A failed attempt retains the original runtime, control and deployment inputs; no replay. - pub fn close(&self, mut sandbox: ManagedSandbox) -> Result<(), Box> { + pub fn close(&self, mut sandbox: ProviderSandbox) -> Result<(), Box> { if let Err(cause) = sandbox.runtime.stop(1) { return Err(failure( sandbox, @@ -23,14 +23,14 @@ impl SandboxApi { } } fn failure( - sandbox: ManagedSandbox, + sandbox: ProviderSandbox, phase: SandboxPhase, cause: SandboxCause, -) -> Box { - Box::new(SandboxFailure { +) -> Box { + Box::new(ProviderFailure { request: sandbox.request, phase, - sandbox: Some(sandbox.runtime), + runtime: Some(sandbox.runtime), control: Some(sandbox.control), handshake: sandbox.handshake, observed: Some(sandbox.startup), diff --git a/engine/crates/layerfs-api/src/sandbox/connection.rs b/engine/crates/layerfs-api/src/sandbox/connection.rs new file mode 100644 index 0000000000..ba3a75df2d --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/connection.rs @@ -0,0 +1,108 @@ +//! One shared TCP/Noise/Hello pipeline, retaining progress through every failure. +use super::create::{hello, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest}, + duplex::{initiate_observed, ChannelWork}, +}; +use std::{ + net::{SocketAddr, TcpStream}, + time::Duration, +}; + +pub(super) struct Startup { + pub phase: SandboxPhase, + pub control: Option, + pub handshake: ChannelWork, + pub observed: Option, +} +impl Startup { + pub fn new() -> Self { + Self { + phase: SandboxPhase::Validate, + control: None, + handshake: ChannelWork::default(), + observed: None, + } + } + pub fn connect( + &mut self, + endpoint: SocketAddr, + host: Option<&str>, + wait: Duration, + secret: &[u8; 32], + peer: [u8; 32], + existing: bool, + ) -> Result { + self.phase = SandboxPhase::Connect; + let address = super::create::connect_address(endpoint, host).map_err(SandboxCause::Io)?; + let stream = TcpStream::connect_timeout(&address, wait).map_err(SandboxCause::Io)?; + stream + .set_read_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + stream + .set_write_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + self.phase = SandboxPhase::Authenticate; + let start = initiate_observed(stream, secret, peer); + self.handshake = start.work; + self.control = Some(Control::new(start.result.map_err(SandboxCause::Channel)?)); + self.phase = SandboxPhase::Policy; + self.control + .as_ref() + .expect("authenticated") + .clear_io_waits() + .map_err(SandboxCause::Channel)?; + self.phase = SandboxPhase::Hello; + let status = hello( + self.control.as_mut().expect("authenticated"), + HelloRequest { + expected_instance: None, + wait_for_store: existing, + }, + ) + .map_err(SandboxCause::Control)?; + self.observed = Some(status.clone()); + let expected = if existing { + DaemonPhase::ControlReady + } else { + DaemonPhase::InstallPending + }; + if status.phase != expected { + return Err(SandboxCause::Setup(ControlError("original startup phase"))); + } + Ok(status) + } +} +/// Installation and its Ready observation are identical for every runtime. +pub(super) fn install( + control: &mut Control, + startup: &mut DaemonStatus, + installed: &mut Option, + phase: &mut SandboxPhase, + store: &layerfs_persistence::SealedStore, + manifest: &layerfs_bridge::provision::StoreManifest, +) -> Result<(), SandboxCause> { + *phase = SandboxPhase::Install; + *installed = Some( + control + .install_file(store, manifest) + .map_err(SandboxCause::Install)?, + ); + *phase = SandboxPhase::Hello; + let status = hello( + control, + HelloRequest { + expected_instance: Some(startup.instance), + wait_for_store: true, + }, + ) + .map_err(SandboxCause::Control)?; + *startup = status; + if startup.phase != DaemonPhase::ControlReady { + return Err(SandboxCause::Setup(ControlError( + "original installed startup phase", + ))); + } + Ok(()) +} diff --git a/engine/crates/layerfs-api/src/sandbox/create.rs b/engine/crates/layerfs-api/src/sandbox/create.rs index 2146783582..61837c6693 100644 --- a/engine/crates/layerfs-api/src/sandbox/create.rs +++ b/engine/crates/layerfs-api/src/sandbox/create.rs @@ -1,48 +1,10 @@ //! Ordinary Sandbox startup, original deployment inputs and failure custody. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::{ - control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest, Reply, Request}, - daemon_setup::DaemonSetup, - duplex::{initiate_observed, public_key, ChannelError, ChannelWork}, -}; -use layerfs_sandbox::backend::docker::{ - Docker, ListenerFailure, Sandbox, SandboxCreateFailure, SandboxRequest, UploadFailure, -}; -use std::{fmt, fs::File, io, net::TcpStream, path::PathBuf, time::Duration}; -/// One actual stopped-container deployment, private archive and authenticated startup. -#[derive(Clone)] -pub struct SandboxCreate { - /// Exact immutable image and borrowed shared named volume. - pub deployment: SandboxRequest, - /// Explicit private daemon resources and Disposable install/open selection. - pub setup: DaemonSetup, - /// Sealed Linux executable supplied by the caller; streamed once, never read as Store data. - pub executable: PathBuf, - /// Controller Noise secret; redacted from Debug and never sent in env/argv. - pub controller_private: [u8; 32], - /// Expected actual daemon identity, checked against private deployment config. - pub daemon_peer: [u8; 32], - /// Explicit listener event wait; does not apply to ordinary commands. - pub startup_wait: Duration, - /// Host to dial for the daemon's loopback-published port, when this - /// process does not share the Docker host's loopback (a client in a - /// container: `host.docker.internal`). `None` dials the observed - /// 127.0.0.1 endpoint. The publish and its check are unchanged. - pub connect_host: Option, -} -impl fmt::Debug for SandboxCreate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxCreate") - .field("deployment", &self.deployment) - .field("setup", &self.setup) - .field("executable", &self.executable) - .field("controller_private", &"[redacted]") - .field("daemon_peer", &self.daemon_peer) - .field("startup_wait", &self.startup_wait) - .field("connect_host", &self.connect_host) - .finish() - } -} +use layerfs_bridge::control::{ControlError, DaemonStatus, HelloRequest, Reply, Request}; +use layerfs_bridge::duplex::ChannelError; +use layerfs_sandbox::Runtime; +use std::{io, sync::Arc}; + /// The address to dial for a daemon `endpoint` observed on the Docker host's /// loopback: the endpoint itself, or the port on `host` (one resolution, its /// first address; nothing is tried twice). @@ -62,6 +24,7 @@ pub fn connect_address( ) }) } + /// Concrete boundary reached by one original lifecycle operation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SandboxPhase { @@ -94,6 +57,7 @@ pub enum SandboxPhase { /// One delete attempt after acknowledged stop. Delete, } + /// Exact original failure, independent of owned resource disposition. #[derive(Debug)] pub enum SandboxCause { @@ -101,14 +65,10 @@ pub enum SandboxCause { Io(io::Error), /// Local config or correlated startup validation. Setup(ControlError), - /// Actual original runtime Create including partial decoded ID. - Create(Box), - /// Actual incremental archive extraction failure and work. - Upload(Box), - /// Original listener wait/progress/fence failure. - Listener(Box), /// Original Engine transfer, no replay. Runtime(Box), + /// Original provider launch cause; partial runtime ownership is retained separately. + Provider(Box), /// Original authentication failure; handshake work remains in the receipt. Channel(ChannelError), /// Original correlated daemon control exchange/refusal. @@ -116,181 +76,13 @@ pub enum SandboxCause { /// Original streaming installation failure and exact publication knowledge. Install(Box), } -/// Caller retains exact acknowledged owner, connection and original deployment inputs. -pub struct SandboxFailure { - /// Original inputs with secrets retained privately. - pub request: SandboxCreate, - /// Original reached boundary. - pub phase: SandboxPhase, - /// Exact acknowledged owned container; no automatically adopted partial ID. - pub sandbox: Option, - /// Original authenticated control owner, if established. - pub control: Option, - /// Actual original handshake work, including failure. - pub handshake: ChannelWork, - /// Original correlated Hello, even if its phase failed selection validation. - pub observed: Option, - /// Installation acknowledgement, retained if the following Ready observation fails. - pub installed: Option, - /// Exact original cause. - pub cause: SandboxCause, -} -impl fmt::Debug for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxFailure") - .field("request", &self.request) - .field("phase", &self.phase) - .field("sandbox", &self.sandbox) - .field("has_control", &self.control.is_some()) - .field("handshake", &self.handshake) - .field("observed", &self.observed) - .field("cause", &self.cause) - .finish() - } -} -impl fmt::Display for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) - } -} -impl std::error::Error for SandboxFailure {} + /// Actual local Engine and authenticated daemon lifecycle organization. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SandboxApi { - pub(crate) docker: Docker, -} -/// Exact acknowledged container and authenticated control owner. -pub struct ManagedSandbox { - /// Owned runtime container; explicit stop/delete only. - pub runtime: Sandbox, - /// Existing sequential control owner; correlation and failed state are never reset. - pub(crate) control: Control, - /// Original acknowledged installation, when this create installed the Project. - pub installed: Option, - pub(crate) request: SandboxCreate, - /// Original correlated startup status; this is not kernel Workspace Ready. - pub startup: DaemonStatus, - /// Actual original duplex handshake work. - pub handshake: ChannelWork, - pub(crate) command_identity: layerfs_sandbox::CommandIdentity, -} -impl SandboxApi { - /// Uses one explicit local Engine endpoint and metadata wait policy. - pub fn new(docker: Docker) -> Self { - Self { docker } - } - /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. - pub fn create( - &self, - request: SandboxCreate, - project: &crate::SealedProject, - ) -> Result> { - self.create_file(request, &project.store, &project.manifest) - } - /// Creates a new daemon using exact persisted sealed-file and manifest facts. - pub fn create_file( - &self, - request: SandboxCreate, - store: &layerfs_persistence::SealedStore, - manifest: &layerfs_bridge::provision::StoreManifest, - ) -> Result> { - let existing = request.setup.existing_store.is_some(); - if let Some(selected) = &request.setup.existing_store { - let mut expected = manifest.clone(); - expected.daemon_sqlite = selected.daemon_sqlite.clone(); - if *selected != expected { - return Err(Box::new(SandboxFailure { - request, - phase: SandboxPhase::Validate, - sandbox: None, - control: None, - handshake: ChannelWork::default(), - observed: None, - installed: None, - cause: SandboxCause::Setup(ControlError("selected existing Project")), - })); - } - } - let mut owner = create(&self.docker, request)?; - if existing { - // The original startup already selected and waited for this Store. - // Installing again would change the existing one-attempt lifecycle. - return Ok(owner); - } - let installed = owner - .control - .install_file(store, manifest) - .map_err(|cause| { - Box::new(SandboxFailure { - request: owner.request.clone(), - phase: SandboxPhase::Install, - sandbox: None, - control: None, - handshake: owner.handshake, - observed: Some(owner.startup.clone()), - installed: None, - cause: SandboxCause::Install(cause), - }) - }); - let installed = match installed { - Ok(installed) => installed, - Err(mut failure) => { - failure.sandbox = Some(owner.runtime); - failure.control = Some(owner.control); - return Err(failure); - } - }; - owner.installed = Some(installed); - match hello( - &mut owner.control, - HelloRequest { - expected_instance: Some(owner.startup.instance), - wait_for_store: true, - }, - ) { - Ok(status) if status.phase == DaemonPhase::ControlReady => { - owner.startup = status; - Ok(owner) - } - Ok(status) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(status), - installed: owner.installed, - cause: SandboxCause::Setup(ControlError("original installed startup phase")), - })), - Err(cause) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(owner.startup), - installed: owner.installed, - cause: SandboxCause::Control(cause), - })), - } - } -} -impl ManagedSandbox { - /// Explicit startup observation/wait on the same incarnation and Control owner. - /// A later observation never resolves an earlier unknown operation. - pub(crate) fn status( - &mut self, - wait_for_store: bool, - ) -> Result> { - hello( - &mut self.control, - HelloRequest { - expected_instance: Some(self.startup.instance), - wait_for_store, - }, - ) - } + pub(crate) runtime: Arc, } + pub(super) fn hello( control: &mut Control, request: HelloRequest, @@ -301,118 +93,3 @@ pub(super) fn hello( reply => Err(OperationFailure::unexpected(request, reply)), } } -pub(crate) fn create( - docker: &Docker, - request: SandboxCreate, -) -> Result> { - let mut phase = SandboxPhase::Validate; - let mut sandbox = None; - let mut control = None; - let mut handshake = ChannelWork::default(); - let mut observed = None; - let result = (|| { - let config = request.setup.encode().map_err(SandboxCause::Setup)?; - if request.setup.listen != format!("0.0.0.0:{}", request.deployment.port) - || request.setup.store != "/layerfs-store/global/store.sqlite" - || request.setup.overlay != "/layerfs-local/overlay/overlay.sqlite" - || request.setup.mounts != "/workspaces" - || request.startup_wait.is_zero() - || public_key(&request.controller_private).map_err(SandboxCause::Channel)? - != request.setup.control_peer - || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? - != request.daemon_peer - { - return Err(SandboxCause::Setup(ControlError( - "concrete deployment topology/identity", - ))); - } - phase = SandboxPhase::Executable; - let mut file = File::open(&request.executable).map_err(SandboxCause::Io)?; - phase = SandboxPhase::Create; - sandbox = Some( - docker - .create_sandbox(request.deployment.clone()) - .map_err(SandboxCause::Create)?, - ); - let owner = sandbox.as_mut().expect("acknowledged container"); - phase = SandboxPhase::Upload; - owner - .upload(&mut file, &config) - .map_err(SandboxCause::Upload)?; - phase = SandboxPhase::Start; - owner.start().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Listener; - owner - .wait_listener(request.startup_wait) - .map_err(SandboxCause::Listener)?; - phase = SandboxPhase::Endpoint; - let endpoint = owner.endpoint().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Connect; - let address = - connect_address(endpoint, request.connect_host.as_deref()).map_err(SandboxCause::Io)?; - let stream = - TcpStream::connect_timeout(&address, request.startup_wait).map_err(SandboxCause::Io)?; - stream - .set_read_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - stream - .set_write_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - phase = SandboxPhase::Authenticate; - let start = initiate_observed(stream, &request.controller_private, request.daemon_peer); - handshake = start.work; - let connection = start.result.map_err(SandboxCause::Channel)?; - // The startup handshake waits are cleared before control/product operations. - // Bridge owns cloned duplex directions; clear policy through that actual owner. - control = Some(Control::new(connection)); - phase = SandboxPhase::Policy; - control - .as_ref() - .expect("authenticated") - .clear_io_waits() - .map_err(SandboxCause::Channel)?; - phase = SandboxPhase::Hello; - let status = hello( - control.as_mut().expect("authenticated"), - HelloRequest { - expected_instance: None, - wait_for_store: request.setup.existing_store.is_some(), - }, - ) - .map_err(SandboxCause::Control)?; - let expected = if request.setup.existing_store.is_some() { - DaemonPhase::ControlReady - } else { - DaemonPhase::InstallPending - }; - observed = Some(status.clone()); - if status.phase != expected { - return Err(SandboxCause::Setup(ControlError("original startup phase"))); - } - Ok(status) - })(); - match result { - Ok(startup) => Ok(ManagedSandbox { - request: request.clone(), - installed: None, - runtime: sandbox.take().expect("owned"), - control: control.take().expect("authenticated"), - startup, - handshake, - command_identity: layerfs_sandbox::CommandIdentity { - uid: request.setup.command_uid, - gid: request.setup.command_gid, - }, - }), - Err(cause) => Err(Box::new(SandboxFailure { - request, - phase, - sandbox, - control, - handshake, - observed, - installed: None, - cause, - })), - } -} diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index c0db8c8e5b..40e0a5e449 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -1,44 +1,30 @@ //! Optional ordinary Sandbox launch at the acknowledged mounted directory. -use crate::{ManagedSandbox, MountedWorkspace, WorkspaceApi}; -use layerfs_sandbox::{ - backend::docker::{ExecCreateFailure, ExecCreated, ExecRequest, Sandbox}, - CommandIdentity, -}; +use super::{MountedWorkspace, ProviderSandbox, WorkspaceApi}; +use layerfs_sandbox::{provider::ExecExit, ExecRequest, WireFailure}; +use std::io::Write; impl WorkspaceApi<'_> { - /// Creates one ordinary command at this mount using the deployment's admitted identity. - /// The returned Sandbox runtime owner owns Start, streams, exit status and cancellation. + /// Runs one ordinary command at this mount using the deployment's admitted identity. + /// The runtime provider owns execution, streams, exit status and cancellation. /// This sends no daemon request, registers no Exec and performs no Commit or unmount. pub fn exec( - sandbox: &ManagedSandbox, + sandbox: &ProviderSandbox, workspace: &MountedWorkspace, arguments: Vec, environment: Vec, - stdin: bool, - ) -> Result> { - launch( - &sandbox.runtime, - sandbox.command_identity, - arguments, - environment, - workspace.ready.directory.clone(), - stdin, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + sandbox.runtime.run( + &ExecRequest { + identity: sandbox.command_identity, + arguments, + environment, + directory: workspace.ready.directory.clone(), + stdin: false, + }, + stdout, + stderr, ) } } -pub(crate) fn launch( - sandbox: &Sandbox, - identity: CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result> { - sandbox.create_exec(ExecRequest { - identity, - arguments, - environment, - directory, - stdin, - }) -} diff --git a/engine/crates/layerfs-api/src/sandbox/mod.rs b/engine/crates/layerfs-api/src/sandbox/mod.rs index 6983a84746..1bf10e271d 100644 --- a/engine/crates/layerfs-api/src/sandbox/mod.rs +++ b/engine/crates/layerfs-api/src/sandbox/mod.rs @@ -1,6 +1,12 @@ //! Actual ordinary Sandbox lifecycle and its Workspaces, with exact logical and kernel readiness. #[cfg(unix)] mod close; +#[cfg(unix)] +mod connection; +#[cfg(unix)] +mod provider; +#[cfg(unix)] +pub use provider::{ProviderCreate, ProviderFailure, ProviderSandbox}; mod commit; #[cfg(unix)] pub(crate) mod create; @@ -12,9 +18,9 @@ mod unmount; pub mod values; pub use commit::CommitReply; #[cfg(unix)] -pub use create::{connect_address, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase}; +pub use create::SandboxApi; #[cfg(unix)] -pub use create::{ManagedSandbox, SandboxApi}; +pub use create::{connect_address, SandboxCause, SandboxPhase}; pub use mount::BoundWorkspace; pub use mount::WorkspaceApi; pub use mount::{MountFailure, MountedWorkspace}; diff --git a/engine/crates/layerfs-api/src/sandbox/mount.rs b/engine/crates/layerfs-api/src/sandbox/mount.rs index 15d2e70aab..4d0414cebc 100644 --- a/engine/crates/layerfs-api/src/sandbox/mount.rs +++ b/engine/crates/layerfs-api/src/sandbox/mount.rs @@ -12,9 +12,14 @@ pub struct WorkspaceApi<'a> { impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. #[cfg(unix)] - pub fn new(sandbox: &'a mut crate::ManagedSandbox) -> Self { + pub fn new(sandbox: &'a mut super::ProviderSandbox) -> Self { + Self::from_control(&mut sandbox.control) + } + /// Borrows an authenticated control channel directly. + #[cfg(unix)] + pub fn from_control(control: &'a mut Control) -> Self { Self { - control: &mut sandbox.control, + control, observation_scope: None, } } diff --git a/engine/crates/layerfs-api/src/sandbox/provider.rs b/engine/crates/layerfs-api/src/sandbox/provider.rs new file mode 100644 index 0000000000..098f66dc8b --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/provider.rs @@ -0,0 +1,373 @@ +//! Runtime startup uses the same authenticated connection and Store installation as Docker. +use super::{connection::Startup, SandboxApi, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonStatus}, + daemon_setup::DaemonSetup, + duplex::{public_key, ChannelWork}, + provision::StoreManifest, +}; +use layerfs_sandbox::{ + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, Sandbox, +}; +use std::{fmt, sync::Arc}; + +/// Runtime launch and private daemon authentication inputs. +#[derive(Clone)] +pub struct ProviderCreate { + /// Runtime launch; configuration is encoded from setup after validation. + pub launch: LaunchRequest, + /// Explicit Disposable daemon topology and Store selection. + pub setup: DaemonSetup, + /// Controller secret; never printed or sent in argv/environment. + pub controller_private: [u8; 32], + /// Expected daemon public identity. + pub daemon_peer: [u8; 32], +} +impl fmt::Debug for ProviderCreate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderCreate") + .field("launch", &self.launch) + .field("setup", &self.setup) + .field("daemon_peer", &self.daemon_peer) + .finish_non_exhaustive() + } +} +/// Exact startup failure and all acknowledged ownership, without automatic cleanup. +pub struct ProviderFailure { + /// Original private request. + pub request: ProviderCreate, + /// Last attempted stage. + pub phase: SandboxPhase, + /// Acknowledged runtime, including a partially launched instance. + pub runtime: Option>, + /// Authenticated channel when established. + pub control: Option, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Last correlated startup observation. + pub observed: Option, + /// Installation acknowledgement even when the following Hello fails. + pub installed: Option, + /// Original deciding cause. + pub cause: SandboxCause, +} +impl fmt::Debug for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderFailure") + .field("phase", &self.phase) + .field("runtime", &self.runtime.as_ref().map(|r| r.identity())) + .field("cause", &self.cause) + .finish_non_exhaustive() + } +} +impl fmt::Display for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) + } +} +impl std::error::Error for ProviderFailure {} +/// Runtime owner and its original authenticated daemon channel. +pub struct ProviderSandbox { + /// Original private launch and daemon configuration inputs. + pub request: ProviderCreate, + /// Explicit runtime lifecycle owner. + pub runtime: Box, + /// Original correlated startup observation. + pub startup: DaemonStatus, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Store installation acknowledgement, if installed by this create. + pub installed: Option, + pub(crate) control: Control, + /// Command identity admitted for this Sandbox deployment. + pub command_identity: CommandIdentity, +} +impl ProviderSandbox { + /// Explicit startup observation/wait on the same incarnation and Control owner. + pub(crate) fn status( + &mut self, + wait_for_store: bool, + ) -> Result> { + super::create::hello( + &mut self.control, + layerfs_bridge::control::HelloRequest { + expected_instance: Some(self.startup.instance), + wait_for_store, + }, + ) + } +} +impl SandboxApi { + /// Uses the deployment's selected runtime. + pub fn with_runtime(runtime: Arc) -> Self { + Self { runtime } + } + /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. + pub fn create( + &self, + request: ProviderCreate, + project: &crate::SealedProject, + ) -> Result> { + self.create_provider_file(request, &project.store, &project.manifest) + } + /// Launches once and authenticates to Hello without Store installation (internal proof only). + pub(crate) fn start( + &self, + request: ProviderCreate, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + ) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed: None, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed: None, + cause, + })), + } + } + /// Launches once, authenticates, selects or installs the Store, then observes Ready. + pub fn create_provider_file( + &self, + request: ProviderCreate, + store: &layerfs_persistence::SealedStore, + manifest: &StoreManifest, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let mut installed = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if request + .setup + .existing_store + .as_ref() + .is_some_and(|m| m.locator != request.setup.store) + || launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + if let Some(selected) = &request.setup.existing_store { + let mut expected = manifest.clone(); + expected.daemon_sqlite = selected.daemon_sqlite.clone(); + if *selected != expected { + return Err(SandboxCause::Setup(ControlError( + "selected existing Project", + ))); + } + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + let mut status = progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + )?; + if request.setup.existing_store.is_none() { + let mut phase = SandboxPhase::Install; + let result = super::connection::install( + progress.control.as_mut().expect("authenticated"), + &mut status, + &mut installed, + &mut phase, + store, + manifest, + ); + progress.phase = phase; + progress.observed = Some(status.clone()); + result?; + } + Ok(status) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed, + cause, + })), + } + } +} diff --git a/engine/crates/layerfs-api/tests/local_install.rs b/engine/crates/layerfs-api/tests/local_install.rs index 2bb5ff0c76..15ff14753a 100644 --- a/engine/crates/layerfs-api/tests/local_install.rs +++ b/engine/crates/layerfs-api/tests/local_install.rs @@ -55,7 +55,7 @@ fn root() -> bool { std::fs::metadata("/proc/self").is_ok_and(|m| m.uid() == 0) } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn import_installs_the_store_in_process() { use std::os::unix::fs::PermissionsExt; @@ -64,6 +64,7 @@ fn import_installs_the_store_in_process() { let destination = directory.join("volume/global/store.sqlite"); let sealed_bytes = std::fs::read(&project.store.path).unwrap(); let outcome = install_local(&project.store.path, &project.manifest, &destination); + #[cfg(target_os = "linux")] if !root() { let failure = outcome.unwrap_err(); assert!(failure.cause.contains("must run as user 0"), "{failure}"); @@ -117,9 +118,9 @@ fn an_installed_manifest_is_refused_before_anything_is_written() { std::fs::remove_dir_all(directory).unwrap(); } -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] -fn local_install_is_unsupported_off_linux() { +fn local_install_is_unsupported_on_other_platforms() { let directory = scratch("host"); let project = sealed(&directory); let destination = directory.join("volume/global/store.sqlite"); diff --git a/engine/crates/layerfs-api/tests/provider_ownership.rs b/engine/crates/layerfs-api/tests/provider_ownership.rs new file mode 100644 index 0000000000..3d57397ab7 --- /dev/null +++ b/engine/crates/layerfs-api/tests/provider_ownership.rs @@ -0,0 +1,143 @@ +//! Invalid native topology must refuse before spawning or touching another instance. +#![cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; +static NEXT: AtomicUsize = AtomicUsize::new(0); +struct ProjectFixture { + directory: PathBuf, + project: layerfs_api::SealedProject, +} +impl ProjectFixture { + fn new() -> Self { + use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId}; + use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; + let directory = std::env::temp_dir().join(format!( + "sdk-existing-project-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&directory).unwrap(); + let source = directory.join("source"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("payload"), b"original complete project").unwrap(); + let request = layerfs_api::InitRequest { + source, + store: PersistenceConfig::sqlite(directory.join("store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + locator: "/layerfs-store/global/store.sqlite".into(), + policy: layerfs_storage::StoragePolicy::frozen_default(), + catalog: HistoryCatalogConfig { + binding_key: b"sdk-existing-project".to_vec(), + incarnation: 1, + cursor_key: [7; 32], + }, + stack: LayerStackId::from_authority([3; 16]), + stack_name: HistoryName::new("project").unwrap(), + branch: BranchId::from_authority([4; 16]), + branch_name: HistoryName::new("main").unwrap(), + scope_seed: [5; 32], + deadline: Instant::now() + Duration::from_secs(10), + }; + let project = layerfs_telemetry::timer::Timing::disabled("sdk.existing.project", |timer| { + layerfs_api::ProjectApi::new().init(request, timer) + }) + .0 + .unwrap(); + Self { directory, project } + } +} +impl Drop for ProjectFixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +#[test] +fn native_resource_paths_are_validated_before_launch() { + use layerfs_api::{sandbox::ProviderCreate, SandboxApi, SandboxPhase}; + use layerfs_bridge::{ + daemon_setup::{DaemonLimits, DaemonSetup}, + duplex, + }; + use layerfs_sandbox::{ + backend::native::NativeMacosProvider, + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, + }; + use std::{ + os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, + }; + let fixture = ProjectFixture::new(); + let root = fixture.directory.join("runtime"); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = NativeMacosProvider::new(&root).unwrap(); + let paths = provider.paths("one").unwrap(); + let metadata = fs::metadata(&root).unwrap(); + let identity = CommandIdentity { + uid: metadata.uid(), + gid: metadata.gid(), + }; + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [32; 32], + control_peer: duplex::public_key(&[31; 32]).unwrap(), + store: fixture.project.store.path.display().to_string(), + overlay: paths.overlay.display().to_string(), + mounts: paths.mounts.display().to_string(), + command_uid: identity.uid, + command_gid: identity.gid, + existing_store: None, + limits: DaemonLimits { + connections: 2, + handshake_ms: 1000, + cache_bytes: 65536, + namespaces: 2, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + }; + let request = ProviderCreate { + setup, + controller_private: [31; 32], + daemon_peer: duplex::public_key(&[32; 32]).unwrap(), + launch: LaunchRequest { + name: "one".into(), + daemon_executable: "/bin/false".into(), + daemon_config: vec![], + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: identity, + environment: vec![], + }, + }, + }; + let api = SandboxApi::with_runtime(Arc::new(Runtime::MacosNative(provider))); + for field in ["overlay", "mounts", "store", "listen"] { + let mut invalid = request.clone(); + match field { + "overlay" => { + invalid.setup.overlay = root.join("other/overlay.sqlite").display().to_string() + } + "mounts" => invalid.setup.mounts = root.join("other/workspaces").display().to_string(), + "store" => { + invalid.setup.store = paths.directory.join("store.sqlite").display().to_string() + } + _ => invalid.setup.listen = "0.0.0.0:30421".into(), + } + let failure = api + .create_provider_file(invalid, &fixture.project.store, &fixture.project.manifest) + .err() + .unwrap(); + assert_eq!(failure.phase, SandboxPhase::Validate); + assert!(failure.runtime.is_none() && failure.control.is_none()); + assert!(!paths.directory.exists()); + } +} diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 64859816e2..507f177de5 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -3,20 +3,26 @@ #[allow(dead_code)] #[path = "../../layerfs-sandbox/tests/support/engine.rs"] mod engine; -use engine::{fixed, frame, Peer}; +use engine::{fixed, frame, upgrade, Peer}; use layerfs_api::{ - ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxPhase, WorkspaceApi, + ProviderCreate, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, WorkspaceApi, }; use layerfs_bridge::{ control::{Answer, Call, DaemonPhase, DaemonStatus, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, duplex, }; +use layerfs_sandbox::backend::docker::DockerProvider; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::{ fs, net::TcpListener, path::PathBuf, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, thread::{self, JoinHandle}, time::{Duration, Instant}, }; @@ -52,7 +58,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { struct Fixture { api: SandboxApi, - owner: Option, + owner: Option, peer: Option, daemon: Option>, binary: PathBuf, @@ -124,7 +130,7 @@ impl Fixture { ]; responses.extend(tail); let (docker, peer) = Peer::new(responses, 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let binary = std::env::temp_dir().join(format!( "sdk-operations-{}-{}", std::process::id(), @@ -137,19 +143,23 @@ impl Fixture { manifest.daemon_sqlite = Some("3.51.0".into()); deployment_setup.existing_store = Some(manifest); } - let request = SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + let request = ProviderCreate { + launch: LaunchRequest { + name: "sdk-operations".into(), + daemon_executable: binary.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: binary.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }; // No Store is constructed or opened. This is the original InstallPending // startup used by component proofs, not a fabricated SDK owner. @@ -201,10 +211,8 @@ fn failed_stop_preserves_original_owner_channel_inputs_and_attempt_without_delet let handshake = owner.handshake; let failure = fixture.api.close(owner).unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Stop); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(!runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -227,10 +235,8 @@ fn failed_delete_retains_acknowledged_stop_and_exact_delete_failure() { .close(fixture.owner.take().unwrap()) .unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Delete); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -299,39 +305,44 @@ fn workspace_exec_returns_original_runtime_owner_at_the_mount_with_no_daemon_reg const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; let mut fixture = Fixture::new(vec![ fixed(201, format!("{{\"Id\":\"{EID}\"}}").as_bytes()), + upgrade(&frame(1, b"")), + fixed( + 200, + format!( + "{{\"ID\":\"{EID}\",\"ContainerID\":\"{CID}\",\"Running\":false,\"ExitCode\":0,\"Pid\":42}}" + ) + .as_bytes(), + ), fixed(204, b""), fixed(204, b""), ]); let before = layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(); - let created = WorkspaceApi::exec( + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec!["SDK_TEST=1".into()], - false, + &mut stdout, + &mut stderr, ) .unwrap(); - assert_eq!(created.identity().0.to_string(), CID); - assert_eq!(created.identity().1.to_string(), EID); - assert_eq!(created.request().directory, "/workspaces/selected-mount"); - assert_eq!( - created.request().identity, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 } - ); + assert_eq!(exit.code, 0); + assert_eq!(exit.pid, Some(42)); assert_eq!( layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(), before ); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); let calls = fixture.calls(); - assert_eq!(calls.len(), 9); + assert_eq!(calls.len(), 11); let request = std::str::from_utf8(&calls[6]).unwrap(); assert!(request.starts_with(&format!("POST /v1.54/containers/{CID}/exec "))); assert!(request.contains(r#""WorkingDir":"/workspaces/selected-mount""#)); assert!(request.contains(r#""User":"501:20""#)); - assert!(!calls - .iter() - .any(|call| call.windows(12).any(|bytes| bytes == b"/exec/start "))); + assert!(calls[7].starts_with(format!("POST /v1.54/exec/{EID}/start ").as_bytes())); + assert!(calls[8].starts_with(format!("GET /v1.54/exec/{EID}/json ").as_bytes())); } #[test] fn workspace_exec_failure_keeps_exact_original_request_and_container() { @@ -340,19 +351,22 @@ fn workspace_exec_failure_keeps_exact_original_request_and_container() { fixed(204, b""), fixed(204, b""), ]); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); let failure = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec![], - true, + &mut stdout, + &mut stderr, ) .unwrap_err(); - assert_eq!(failure.container.to_string(), CID); - assert_eq!(failure.request.directory, "/workspaces/selected-mount"); - assert_eq!(failure.request.arguments, ["/bin/pwd"]); - assert!(failure.request.stdin); - assert!(failure.transfer.attempted); + assert_eq!( + failure.requested_container, + Some(layerfs_sandbox::ContainerId::parse(CID).unwrap()) + ); + assert!(failure.attempted); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); assert_eq!(fixture.calls().len(), 9); } @@ -429,25 +443,29 @@ fn create_matching_existing_project_keeps_one_original_hello_and_does_not_reinst fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_effect() { let project = ProjectFixture::new(); let (docker, peer) = Peer::new(vec![], 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let mut deployment_setup = setup([32; 32], duplex::public_key(&[31; 32]).unwrap()); let mut manifest = project.project.manifest.clone(); manifest.root[0] ^= 1; deployment_setup.existing_store = Some(manifest); let result = api.create( - SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "mismatched-sandbox".into(), + daemon_executable: project.project.store.path.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: project.project.store.path.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }, &project.project, ); @@ -455,6 +473,6 @@ fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_e panic!("mismatched Project must refuse before effects") }; assert_eq!(failure.phase, SandboxPhase::Validate); - assert!(failure.sandbox.is_none() && failure.control.is_none() && failure.installed.is_none()); + assert!(failure.runtime.is_none() && failure.control.is_none() && failure.installed.is_none()); assert!(peer.finish().is_empty()); } diff --git a/engine/crates/layerfs-daemon/Cargo.toml b/engine/crates/layerfs-daemon/Cargo.toml index ea15738653..b53132243f 100644 --- a/engine/crates/layerfs-daemon/Cargo.toml +++ b/engine/crates/layerfs-daemon/Cargo.toml @@ -11,6 +11,8 @@ layerfs-overlay = { path = "../layerfs-overlay" } layerfs-store-service = { path = "../layerfs-store-service" } layerfs-workspace = { path = "../layerfs-workspace" } layerfs-fuse = { path = "../layerfs-fuse" } +# NFS kernel serving: the native route's mount entry beside FUSE. +layerfs-nfs = { path = "../layerfs-nfs" } layerfs-bridge = { path = "../layerfs-bridge" } layerfs-content = { path = "../../../core/crates/layerfs-content" } layerfs-storage = { path = "../../../core/crates/layerfs-storage" } @@ -19,7 +21,7 @@ layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } # Concrete Store opening stays in application bootstrap, outside store/. layerfs-persistence = { path = "../layerfs-persistence" } -nix = { version = "=0.31.3", features = ["fs", "user", "process"] } +nix = { version = "=0.31.3", features = ["fs", "user", "process", "resource"] } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/engine/crates/layerfs-daemon/src/application/config.rs b/engine/crates/layerfs-daemon/src/application/config.rs index e3c404c8a0..a2269fba99 100644 --- a/engine/crates/layerfs-daemon/src/application/config.rs +++ b/engine/crates/layerfs-daemon/src/application/config.rs @@ -29,7 +29,10 @@ pub(super) fn read(path: &Path) -> io::Result { return Err(invalid("private config record size")); } let config = DaemonSetup::decode(&bytes).map_err(|_| invalid("private config fields"))?; - if config.command_uid == uid { + // The NFS route opens no FUSE device and the native route runs its + // commands as the daemon's own user, unconfined (no isolation); every + // other route keeps the two identities apart. + if config.command_uid == uid && !super::filesystem::nfs_route() { return Err(invalid("command and daemon identities must differ")); } // One early reservation restores any value below the refill window; a @@ -108,11 +111,21 @@ fn secure_process() -> io::Result<()> { } Ok(()) } -#[cfg(not(target_os = "linux"))] +/// macOS (the native NFS route): files the daemon creates are its own +/// alone, and it never writes a core file holding its keys or Store pages. +/// There is no dumpable flag; a debugger of the same user is not refused. +#[cfg(target_os = "macos")] +fn secure_process() -> io::Result<()> { + use nix::sys::resource::{setrlimit, Resource}; + nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o077)); + setrlimit(Resource::RLIMIT_CORE, 0, 0).map_err(io::Error::from)?; + Ok(()) +} +#[cfg(not(any(target_os = "linux", target_os = "macos")))] fn secure_process() -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Unsupported, - "production daemon requires Linux process protection", + "production daemon requires Linux or macOS process protection", )) } fn invalid(message: &'static str) -> io::Error { diff --git a/engine/crates/layerfs-daemon/src/application/filesystem.rs b/engine/crates/layerfs-daemon/src/application/filesystem.rs index a71a5e82b9..21bfcec597 100644 --- a/engine/crates/layerfs-daemon/src/application/filesystem.rs +++ b/engine/crates/layerfs-daemon/src/application/filesystem.rs @@ -1,10 +1,72 @@ //! Kernel serving assembly over the daemon's one Store and one overlay owner. use crate::{ - control::{KernelServing, Service}, + control::{ExportServing, ExportServingConfig, Helper, KernelServing, Mounter, Service}, store::Store, Owner, }; -use std::sync::Arc; +use layerfs_bridge::daemon_setup::DaemonSetup; +use std::{path::Path, sync::Arc}; + +/// Selects the kernel route: `nfs` serves every mount over loopback NFSv3 +/// (the native route); anything else, or unset, is FUSE. +pub const KERNEL_ROUTE: &str = "LAYERFS_KERNEL"; +/// Optional helper command lines (`{port}`, `{dir}` replaced) overriding the +/// host client's defaults for the NFS route. +pub const NFS_MOUNT: &str = "LAYERFS_NFS_MOUNT"; +pub const NFS_UNMOUNT: &str = "LAYERFS_NFS_UNMOUNT"; +pub const NFS_FORCE: &str = "LAYERFS_NFS_FORCE"; +pub const NFS_FLUSH: &str = "LAYERFS_NFS_FLUSH"; + +/// Whether this daemon serves its mounts over NFS. +/// macOS has no other route: there it is always NFS. +pub(super) fn nfs_route() -> bool { + cfg!(target_os = "macos") || std::env::var(KERNEL_ROUTE).is_ok_and(|route| route == "nfs") +} +/// NFS serving for the native route, from the protected setup and the +/// declared helper overrides. Nothing is mounted until an explicit Attach. +pub(super) fn assemble_export(setup: &DaemonSetup) -> std::io::Result>> { + if !nfs_route() { + return Ok(None); + } + let mounts = Path::new(&setup.mounts); + if !mounts.is_absolute() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "mount home must be absolute", + )); + } + if !mounts.is_dir() { + std::fs::create_dir(mounts)?; + } + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper(NFS_MOUNT) { + mounter.mount = mount; + } + if let Some(unmount) = helper(NFS_UNMOUNT) { + mounter.unmount = unmount; + } + if let Some(force) = helper(NFS_FORCE) { + mounter.force = force; + } + mounter.flush = helper(NFS_FLUSH); + let serving = ExportServing::start(ExportServingConfig::new( + mounts.to_owned(), + setup.command_uid, + setup.command_gid, + mounter, + )); + // A daemon of this mount home that ended without unmounting left its + // client mounts; they are forced off before this one serves. + for line in serving.reclaim() { + eprintln!("LAYERFS_NFS_RECLAIM {line}"); + } + Ok(Some(Arc::new(serving))) +} /// Control and kernel serving share the same Store handle and overlay owner; /// the Service's registry stays the sole authority for every mount. The @@ -13,19 +75,19 @@ pub(super) fn service( store: Arc, owner: &Owner, kernel: Option<&Arc>, + export: Option<&Arc>, serial_low_water: u64, ) -> Arc { store.set_serial_low_water(serial_low_water); - Arc::new(match kernel { - Some(kernel) => Service::with_kernel(store, owner, kernel.clone()), - None => Service::new(store, owner), + Arc::new(match (export, kernel) { + (Some(export), _) => Service::with_export(store, owner, export.clone()), + (None, Some(kernel)) => Service::with_kernel(store, owner, kernel.clone()), + (None, None) => Service::new(store, owner), }) } /// Kernel serving is a Linux capability; elsewhere control alone is assembled. #[cfg(not(target_os = "linux"))] -pub(super) fn assemble( - _setup: &layerfs_bridge::daemon_setup::DaemonSetup, -) -> std::io::Result>> { +pub(super) fn assemble(_setup: &DaemonSetup) -> std::io::Result>> { Ok(None) } #[cfg(target_os = "linux")] @@ -58,6 +120,10 @@ mod linux { pub(in crate::application) fn assemble( setup: &DaemonSetup, ) -> io::Result>> { + // The NFS route opens no FUSE device. + if super::nfs_route() { + return Ok(None); + } let mounts = Path::new(&setup.mounts); device()?; home(mounts)?; diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index 9206dee1ac..c15d6833d8 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -44,6 +44,8 @@ pub struct Application { pub(super) owner: Owner, /// Fixed shared kernel workers, entered before control readiness. pub(super) kernel: Option>, + /// NFS serving of the native route, when selected. + pub(super) export: Option>, pub(super) state: Mutex, pub(super) changed: Condvar, pub(super) workers: Mutex>>>, @@ -120,11 +122,21 @@ impl Application { }) } }; + let export = match super::filesystem::assemble_export(&setup) { + Ok(export) => export, + Err(cause) => { + return Err(ApplicationError::AfterOverlay { + owner: Box::new(owner), + cause: Box::new(ApplicationError::Io(cause)), + }) + } + }; let service = existing.as_ref().map(|opened| { super::filesystem::service( opened.store.clone(), &owner, kernel.as_ref(), + export.as_ref(), l.serial_low_water, ) }); @@ -139,6 +151,7 @@ impl Application { instance, owner, kernel, + export, state: Mutex::new(State { startup: Startup { phase, diff --git a/engine/crates/layerfs-daemon/src/application/serve.rs b/engine/crates/layerfs-daemon/src/application/serve.rs index f88db6a0e6..645571dd32 100644 --- a/engine/crates/layerfs-daemon/src/application/serve.rs +++ b/engine/crates/layerfs-daemon/src/application/serve.rs @@ -96,6 +96,7 @@ impl Application { installed.opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.installed = Some(installed); @@ -119,6 +120,7 @@ impl Application { opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.phase = DaemonPhase::ControlReady; diff --git a/engine/crates/layerfs-daemon/src/control/attach.rs b/engine/crates/layerfs-daemon/src/control/attach.rs index 0478136630..5531daa8f2 100644 --- a/engine/crates/layerfs-daemon/src/control/attach.rs +++ b/engine/crates/layerfs-daemon/src/control/attach.rs @@ -35,6 +35,11 @@ impl Service { token: WorkspaceToken, timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return crate::telemetry::phase(timing, "kernel.attach", || { + self.attach_export(&export, token) + }); + } let Some(serving) = self.kernel.clone() else { return Err(Failure::Rejected( ControlCode::Invalid, diff --git a/engine/crates/layerfs-daemon/src/control/detach.rs b/engine/crates/layerfs-daemon/src/control/detach.rs index 0bd63f90a5..ce692cc176 100644 --- a/engine/crates/layerfs-daemon/src/control/detach.rs +++ b/engine/crates/layerfs-daemon/src/control/detach.rs @@ -51,7 +51,12 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + Kernel::Unattached + | Kernel::Attaching + | Kernel::Retained(_) + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => { unreachable!("handled by terminal admission") } } @@ -179,16 +184,6 @@ impl Service { } } } - /// Poison recovery stores this exact owner only; a poisoned registry never - /// authorizes another product operation. - pub(super) fn settle(&self, token: WorkspaceToken, kernel: Kernel, activity: Activity) { - let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); - if let Ok(binding) = bound_mut(&mut entries, token) { - binding.kernel = kernel; - binding.activity = activity; - binding.epoch = binding.epoch.saturating_add(1); - } - } /// Stops a terminal operation after effects, keeping its owner in the entry. /// A forced teardown's facts are stored with it, as they stood at the stop. pub(super) fn retain( diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs new file mode 100644 index 0000000000..592b203e50 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -0,0 +1,928 @@ +//! NFS kernel serving: the native route's mount entry, beside FUSE. +//! +//! Each Attach serves the Workspace's engine mount as one loopback NFSv3 +//! export (`layerfs-nfs`) on a fresh port, then has the host's own NFS client +//! mount it on the Workspace's mount directory with the configured helper +//! (`mount_nfs` on macOS). The registry keeps the export as the entry's +//! kernel half; the Workspace API, its control records and every engine and +//! Store step are the ones the FUSE route uses: +//! +//! - Ready carries the same receipt record; an NFS receipt is recognized by +//! `abi_major == 0`, and its `mount_id` is the export's TCP port. +//! - Commit first has the client write back what it buffered (the flush +//! helper, `sync(2)` by default) and closes the export's implicit file +//! descriptors, then runs the unchanged Commit. A Commit that installed +//! another root closes the export's descriptors again; the client +//! revalidates attributes on its own schedule (close-to-open, `actimeo`). +//! - Unmount asks the helper to unmount (a busy answer is reversible and the +//! export keeps serving), then stops the export (every connection joined, +//! every reference and descriptor given back), revokes the engine mount +//! and runs the one logical Close. A step that is not established keeps +//! the export and its engine mount in the entry, as FUSE custody is kept. +//! - ForceUnmount runs the forced helper once, then the same teardown. +use super::{ + export_helper::{Helpers, Ran}, + kernel::{bounded, Kernel}, + registry::{bound_mut, idle}, + Failure, Service, Success, +}; +use crate::store::BoundWorkspace; +use layerfs_bridge::control::{ + AbortDisposition, Activity, ControlCode, DetachDisposition, ForceUnmounted, ForcedCleanup, + ForcedFacts, ForcedOutcome, MountPhase, MountReceipt, MountStatus, MountWork, ReadyMount, + Reply, TeardownCustody, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{CleanupState, KernelMount}; +#[cfg(target_os = "macos")] +use std::process::{Command, Stdio}; +use std::{ + fmt, + net::{IpAddr, Ipv4Addr, TcpListener}, + path::PathBuf, + sync::Arc, +}; + +/// Largest write and read the export negotiates (FSINFO wtmax/rtmax). +const WINDOW: u32 = 128 * 1024; +pub use super::export_helper::{Helper, HELPER_WAIT}; + +/// Stale inodes one install refreshes through the mount point at most. +pub const REFRESHED: usize = 4096; +/// `0` turns the post-install refresh off (see [`Service::installed_export`]). +pub const REFRESH: &str = "LAYERFS_NFS_REFRESH"; + +/// One mode-preserving chmod of each path below `root`, through the host's +/// NFS client; a symbolic link (chmod would follow it) and a path that is +/// gone are skipped. The export answers each without a change. +fn refresh(root: &std::path::Path, paths: &[PathBuf]) { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + for path in paths { + let path = root.join(path); + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + if !metadata.file_type().is_symlink() { + let mode = std::fs::Permissions::from_mode(metadata.mode() & 0o7777); + let _ = std::fs::set_permissions(&path, mode); + } + } +} +/// How the host's NFS client mounts, flushes and unmounts an export. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Mounter { + pub mount: Helper, + pub unmount: Helper, + pub force: Helper, + /// Writes back what the client buffered before a Commit; `sync(2)` of + /// the daemon itself when absent. + pub flush: Option, +} +impl Mounter { + /// The host's own client: `mount_nfs` on macOS (no root needed), the + /// Linux NFS client elsewhere (root, as the daemon is in its container). + /// The macOS mount is soft (an export that ended fails calls after the + /// retries instead of hanging them) and hidden from Finder (`nobrowse`). + /// `dumbtimer,timeo=300` keeps a slow answer from failing as a timeout: + /// against a loopback server the round-trip estimator settles near its + /// floor, and a soft call then gave up within seconds. `mutejukebox` + /// keeps an `NFS3ERR_JUKEBOX` answer from raising the "server not + /// responding" dialog. A dead export still fails calls (connection + /// refused), within seconds. + pub fn host() -> Self { + let helper = |words: &[&str]| Helper(words.iter().map(|w| (*w).to_owned()).collect()); + if cfg!(target_os = "macos") { + Self { + mount: helper(&[ + "/sbin/mount_nfs", + "-o", + "vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["/sbin/umount", "{dir}"]), + force: helper(&["/sbin/umount", "-f", "{dir}"]), + flush: None, + } + } else { + Self { + mount: helper(&[ + "mount", + "-t", + "nfs", + "-o", + "vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["umount", "{dir}"]), + force: helper(&["umount", "-f", "-l", "{dir}"]), + flush: None, + } + } + } +} +/// Declared NFS serving inputs, assembled once by the application. +#[derive(Clone, Debug)] +pub struct ExportServingConfig { + /// Existing daemon-owned directory; each mount is a new child of it. + pub mounts: PathBuf, + /// Runtime command identity presented as every inode's owner. + pub command_uid: u32, + pub command_gid: u32, + /// Address each export listens on, port chosen by the system. + pub listen: IpAddr, + pub mounter: Mounter, + /// Refuse creating macOS AppleDouble `._*` names. + pub refuse_apple_double: bool, + /// Lookup references one export keeps before collecting. + pub lookups_held: usize, +} +impl ExportServingConfig { + pub fn new(mounts: PathBuf, command_uid: u32, command_gid: u32, mounter: Mounter) -> Self { + Self { + mounts, + command_uid, + command_gid, + listen: IpAddr::V4(Ipv4Addr::LOCALHOST), + mounter, + refuse_apple_double: true, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } +} +/// Process-wide NFS serving: configuration only; each Attach starts its own +/// export and each terminal operation stops it. +#[derive(Debug)] +pub struct ExportServing { + config: ExportServingConfig, + helpers: Helpers, +} +impl ExportServing { + pub fn start(config: ExportServingConfig) -> Self { + Self { + config, + helpers: Helpers::default(), + } + } + pub fn config(&self) -> &ExportServingConfig { + &self.config + } + fn directory(&self, token: WorkspaceToken) -> PathBuf { + self.config.mounts.join(token.namespace.to_string()) + } + /// At startup, before any Attach: every mount the host still has at or + /// under the mount home is one a daemon that ended without unmounting + /// left (its export is gone, so the client would hang on it). Each is + /// forced off once with the force helper; one line per mount says what + /// happened. Nothing is retried. + pub fn reclaim(&self) -> Vec { + let points = match mounted_under(&self.config.mounts) { + Ok(points) => points, + Err(error) => return vec![format!("mount table unreadable: {error}")], + }; + (points.into_iter()) + .map(|point| { + let text = point.display().to_string(); + match self.helpers.run(&self.config.mounter.force, 0, &text) { + Ran::Done => format!("forced off {text}"), + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + format!("left {text}: {detail}") + } + } + }) + .collect() + } +} +/// Mount points at or under `home`, deepest first, from the host's table: +/// `mount`'s ` on ()` lines on macOS, +/// `/proc/self/mountinfo` (field 5, octal escapes) on Linux. +fn mounted_under(home: &std::path::Path) -> std::io::Result> { + #[cfg(target_os = "macos")] + let points: Vec = { + let out = Command::new("/sbin/mount") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + (String::from_utf8_lossy(&out.stdout).lines()) + .filter_map(|line| { + Some(PathBuf::from( + line.split_once(" on ")?.1.rsplit_once(" (")?.0, + )) + }) + .collect() + }; + #[cfg(not(target_os = "macos"))] + let points: Vec = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) + .filter_map(|line| line.split(' ').nth(4)) + .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) + .collect(); + let mut found: Vec = (points.into_iter()) + .filter(|point| point.starts_with(home)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} +/// A serving export and the mount the host client holds of it. +pub(super) struct Exported { + pub server: Arc, + pub mount: KernelMount, + pub ready: ReadyMount, +} +/// A terminal operation owns the export; status still observes it. +pub(super) struct ExportLeaving { + pub ready: ReadyMount, + pub phase: MountPhase, + pub work: MountWork, +} +/// Export custody a terminal or attach step could not finish disposing: +/// kept exactly, never retried or settled by a later observation. +pub(super) struct ExportKept { + pub ready: Option, + pub stage: TeardownStage, + pub detail: String, + pub work: Option, + pub evidence: Box, + pub forced: Option, +} +impl ExportKept { + pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { + TeardownCustody { + token, + stage: self.stage, + detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), + work: self.work, + detail: bounded(&self.detail), + forced: self.forced.clone(), + } + } +} +impl fmt::Debug for ExportKept { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExportKept") + .field("ready", &self.ready) + .field("stage", &self.stage) + .field("detail", &self.detail) + .field("work", &self.work) + .field("evidence", &self.evidence) + .field("forced", &self.forced) + .finish() + } +} +impl fmt::Debug for Exported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Exported") + .field("address", &self.server.local_addr()) + .field("mount", &self.mount) + .field("held", &self.server.held()) + .finish() + } +} +/// The maintained counters of a serving export. +pub(super) fn work(server: &Server) -> MountWork { + let held = server.held(); + let served = server.counters().iter().map(|count| count.calls).sum(); + MountWork { + loops_configured: 1, + loops_entered: 1, + loops_exited: 0, + loops_joined: 0, + active: 0, + retained: u32::try_from(held.retained).unwrap_or(u32::MAX), + completed: served, + served, + inline: 0, + refused: 0, + terminal: 0, + forget_units: held.collected, + } +} +/// The status block of an export-owning entry. +pub(super) fn block(kernel: &Kernel) -> Option { + Some(match kernel { + Kernel::Exported(exported) => MountStatus { + phase: MountPhase::Ready, + ready: Some(exported.ready.clone()), + detached: false, + work: Some(work(&exported.server)), + }, + Kernel::ExportLeaving(leaving) => MountStatus { + phase: leaving.phase, + ready: Some(leaving.ready.clone()), + detached: leaving.phase == MountPhase::Draining, + work: Some(leaving.work), + }, + Kernel::ExportKept(kept) => MountStatus { + phase: MountPhase::Retained, + ready: kept.ready.clone(), + detached: matches!(kept.stage, TeardownStage::Revoke | TeardownStage::Close), + work: kept.work, + }, + _ => return None, + }) +} +fn receipt(token: WorkspaceToken, directory: String, mount: KernelMount, port: u16) -> ReadyMount { + ReadyMount { + token, + directory, + receipt: MountReceipt { + mount: mount.owner_id(), + root: mount.root_serial(), + mount_id: u64::from(port), + device_major: 0, + device_minor: 0, + abi_major: 0, + abi_minor: 0, + offered: 0, + selected: 0, + max_write: WINDOW, + max_readahead: WINDOW, + max_background: 0, + congestion_threshold: 0, + page_size: 4096, + loops: 1, + abort_bound: true, + }, + } +} +fn kernel_failure(code: ControlCode, phase: &'static str, detail: impl Into) -> Failure { + Failure::Kernel(Box::new(super::types::KernelFailure { + code, + phase, + detail: detail.into(), + evidence: None, + })) +} +/// Where a failed export Attach leaves the entry. +enum Stop { + Unattached(Failure), + Unknown(Failure), + Retained(Box), +} +impl Service { + /// Attach over NFS: Ready is acknowledged only after the export serves + /// and the host client's mount helper reported the mount. + pub(super) fn attach_export( + &self, + serving: &ExportServing, + token: WorkspaceToken, + ) -> Result { + let workspace = self.admit_export(token)?; + match self.export_admitted(serving, token, &workspace) { + Ok(exported) => { + let reply = Reply::Ready(Box::new(exported.ready.clone())); + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + Ok(Success::reply(reply)) + } + Err(Stop::Unattached(failure)) => { + self.settle(token, Kernel::Unattached, Activity::Idle); + Err(failure) + } + Err(Stop::Unknown(failure)) => { + self.settle(token, Kernel::Attaching, Activity::Uncertain); + Err(failure) + } + Err(Stop::Retained(kept)) => { + let custody = kept.custody(token); + self.settle(token, Kernel::ExportKept(kept), Activity::Uncertain); + Err(Failure::Retained(Box::new(custody))) + } + } + } + fn admit_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } + idle(value)?; + if !matches!(value.kernel, Kernel::Unattached) { + return Err(kernel_failure( + ControlCode::Invalid, + "attach:admission", + "Workspace already owns a kernel connection", + )); + } + value.kernel = Kernel::Attaching; + value.activity = Activity::Attaching; + value.epoch = value.epoch.saturating_add(1); + Ok(value.workspace.clone()) + } + fn export_admitted( + &self, + serving: &ExportServing, + token: WorkspaceToken, + workspace: &Arc, + ) -> Result { + let config = &serving.config; + let directory = serving.directory(token); + let Some(text) = directory + .to_str() + .filter(|text| text.len() <= MOUNT_DIRECTORY_LIMIT) + .map(str::to_owned) + else { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Invalid, + "attach:directory", + "mount directory exceeds the control record bound", + ))); + }; + let unknown = |failure: Failure| { + if failure.uncertain() { + Stop::Unknown(failure) + } else { + Stop::Unattached(failure) + } + }; + // The same preparation as the FUSE Attach: the root serial, then the + // initial authoritative serial range before any request can create. + let operation = workspace + .operation() + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + let root = operation + .workspace() + .base() + .map(|base| base.root().root_inode().serial()) + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + if let Err(error) = operation.workspace().prepare_serials(operation.ports()) { + let failure = match operation.ports().failure() { + Ok(Some(cause)) => match cause.as_ref() { + crate::store::PortError::History(error) => Failure::History(error.clone()), + _ => Failure::Workspace(error), + }, + Ok(None) | Err(_) => Failure::Workspace(error), + }; + return Err(unknown(failure)); + } + drop(operation); + if let Err(error) = std::fs::create_dir(&directory) { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Failed, + "attach:directory", + format!("mount directory {text}: {error}"), + ))); + } + let mount = match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_mount(route, root) + }) { + Ok(mount) => mount, + Err(failure) => { + drop(std::fs::remove_dir(&directory)); + return Err(unknown(failure)); + } + }; + let export = ExportConfig { + path: "/".into(), + uid: config.command_uid, + gid: config.command_gid, + read_only: false, + refuse_apple_double: config.refuse_apple_double, + lookups_held: config.lookups_held, + }; + let started = TcpListener::bind((config.listen, 0)).and_then(|listener| { + Server::start( + listener, + Arc::new(crate::service::Mounted(workspace.clone())), + mount, + export, + ) + }); + let server = match started { + Ok(server) => server, + Err(error) => { + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + None, + kernel_failure(ControlCode::Failed, "attach:export", error.to_string()), + )); + } + }; + let port = server.local_addr().port(); + match serving.helpers.run(&config.mounter.mount, port, &text) { + Ran::Done => {} + Ran::Interrupted(detail) => { + // The mount helper may have reached the kernel before its + // deadline. Keep the export alive and do not revoke/delete + // through a possibly mounted directory. + return Err(Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Detach, + detail, + work: Some(work(&server)), + evidence: Box::new(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }), + forced: None, + }))); + } + Ran::Busy(detail) | Ran::Failed(detail) => { + let released = server.stop(); + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + Some(released), + kernel_failure(ControlCode::Failed, "attach:mount", detail), + )); + } + } + Ok(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }) + } + /// No client mount and no export of this attempt remains: revoke its + /// engine mount so the entry returns to Unattached. + fn unwind_export( + &self, + workspace: &BoundWorkspace, + mount: KernelMount, + released: Option, + failure: Failure, + ) -> Stop { + match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + Ok(()) => Stop::Unattached(failure), + Err(revoke) => Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Revoke, + detail: revoke.wire().detail, + work: None, + evidence: Box::new((failure, mount, released, revoke)), + forced: None, + })), + } + } + /// Before a Commit: the client writes back what it buffered, and the + /// export closes its implicit file descriptors. Runs outside the + /// registry lock; a Workspace without an export has nothing to flush. + pub(super) fn flush_export(&self, token: WorkspaceToken) -> Result<(), Failure> { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(()), + } + }; + let Some(serving) = self.export.as_ref() else { + return Ok(()); + }; + match &serving.config.mounter.flush { + None => nix::unistd::sync(), + Some(helper) => { + match serving + .helpers + .run(helper, server.local_addr().port(), &directory) + { + Ran::Done => {} + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) + } + } + } + } + server.close_files(); + Ok(()) + } + /// After a Commit installed another root: the export's descriptors of the + /// replaced tree are closed, and the inodes the install made stale that + /// the client may cache are refreshed through the mount point (see + /// `layerfs_nfs::refresh`): a mode-preserving chmod of each, at most + /// [`REFRESHED`], on a thread of its own after this returns, so the + /// Commit reply never waits on the client. False without an export. + pub(super) fn installed_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + shown: layerfs_content::ObjectId, + installed: layerfs_content::ObjectId, + ) -> Result { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(false), + } + }; + server.installed(); + // `LAYERFS_NFS_REFRESH=0` leaves the client to its own revalidation + // (`actimeo`, close-to-open): the comparison without the refresh. + if std::env::var(REFRESH).is_ok_and(|value| value == "0") { + return Ok(true); + } + let mut stale = std::collections::BTreeSet::new(); + // A walk that stops (at the bound, or failing) refreshes what it found; + // the rest is revalidated on the client's own schedule. + let _ = workspace.stale_after_install(shown, installed, |found| { + stale.insert(match found { + crate::store::Stale::Inode(serial) => serial, + crate::store::Stale::Entry { parent, .. } => parent, + }); + if stale.len() >= REFRESHED { + return Err(std::io::Error::other("refresh bound")); + } + Ok(()) + }); + let paths = server.refresh(stale); + if !paths.is_empty() { + let root = PathBuf::from(directory); + let _ = std::thread::Builder::new() + .name("layerfs-nfs-refresh".into()) + .spawn(move || refresh(&root, &paths)); + } + Ok(true) + } + /// Normal terminal unmount of an export-owning entry, admitted by + /// [`take`]. A busy client mount is the reversible answer. + pub(super) fn unmount_export( + &self, + token: WorkspaceToken, + workspace: Arc, + exported: Exported, + ) -> Result { + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.unmount.clone()); + let port = exported.server.local_addr().port(); + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => {} + Ran::Busy(detail) => { + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + return Err(kernel_failure( + ControlCode::Busy, + "unmount:kernel", + format!("client reported the mount busy; no terminal effect: {detail}"), + )); + } + Ran::Failed(detail) | Ran::Interrupted(detail) => { + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("client unmount: {detail}"), + exported, + None, + )) + } + } + self.teardown_export(token, &workspace, exported, None)?; + self.close(token, &workspace, None) + } + /// After the client mount is gone: stop the export, revoke the engine + /// mount, remove the directory and leave the entry Unattached for Close. + fn teardown_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + exported: Exported, + forced: Option, + ) -> Result<(), Failure> { + self.export_phase(token, MountPhase::Draining, Some(work(&exported.server))); + let Exported { + server, + mount, + ready, + } = exported; + let released = match Arc::try_unwrap(server) { + Ok(server) => server.stop(), + Err(server) => { + return Err(self.keep_export( + token, + TeardownStage::Requests, + "export still shared by a running control step".into(), + Exported { + server, + mount, + ready, + }, + forced, + )) + } + }; + if let Err(failure) = self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + let custody = ExportKept { + ready: Some(ready), + stage: TeardownStage::Revoke, + detail: failure.wire().detail, + work: None, + evidence: Box::new((failure, mount, released)), + forced, + }; + let reply = custody.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(custody)), + Activity::Uncertain, + ); + return Err(Failure::Retained(Box::new(reply))); + } + drop(std::fs::remove_dir(&ready.directory)); + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = Kernel::Unattached; + } + } + self.released + .lock() + .map(|mut last| *last = Some((token, released))) + .ok(); + Ok(()) + } + fn keep_export( + &self, + token: WorkspaceToken, + stage: TeardownStage, + detail: String, + exported: Exported, + forced: Option, + ) -> Failure { + let kept = ExportKept { + ready: Some(exported.ready.clone()), + stage, + detail, + work: Some(work(&exported.server)), + evidence: Box::new(exported), + forced, + }; + let custody = kept.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(kept)), + Activity::Uncertain, + ); + Failure::Retained(Box::new(custody)) + } + fn export_phase(&self, token: WorkspaceToken, phase: MountPhase, work: Option) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + if let Kernel::ExportLeaving(leaving) = &mut binding.kernel { + leaving.phase = phase; + if let Some(work) = work { + leaving.work = work; + } + } + } + } + } + /// Describes retained export custody for an operator. Observation only. + pub fn retained_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + Ok(match &bound_mut(&mut entries, token)?.kernel { + Kernel::ExportKept(kept) => Some(format!("{kept:?}")), + _ => None, + }) + } + /// What the last stopped export of `token` gave back (diagnostics). + pub fn export_released(&self, token: WorkspaceToken) -> Option { + let last = self.released.lock().ok()?; + last.filter(|(seen, _)| *seen == token) + .map(|(_, released)| released) + } + /// The current holdings of a serving export (diagnostics). + pub fn export_held(&self, token: WorkspaceToken) -> Option { + let entries = self.entries.lock().ok()?; + match &super::registry::bound(&entries, token).ok()?.kernel { + Kernel::Exported(exported) => Some(exported.server.held()), + _ => None, + } + } + /// Forced terminal unmount of an export-owning entry: the same + /// before-effect guard as FUSE, one forced client unmount, then the + /// normal teardown and Close. + pub(super) fn force_export( + &self, + token: WorkspaceToken, + relinquish_unknown: bool, + ) -> Result { + let (workspace, exported, commit) = { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + let commit = super::force::guard(value, token, relinquish_unknown)?; + let exported = take(&mut value.kernel, MountPhase::Stopping)?; + value.activity = Activity::Closing; + value.epoch = value.epoch.saturating_add(1); + (value.workspace.clone(), exported, commit) + }; + let mut facts = ForcedFacts { + abort: AbortDisposition::Written, + detach: DetachDisposition::NotAttempted, + commit, + fenced: 0, + }; + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.force.clone()); + let port = exported.server.local_addr().port(); + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => facts.detach = DetachDisposition::Detached, + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + facts.detach = DetachDisposition::Failed; + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("forced client unmount: {detail}"), + exported, + Some(facts), + )); + } + } + let work = work(&exported.server); + self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; + let kernel = self.closed(token, &workspace, None, Some(facts.clone()))?; + let (cleanup, observation_failure) = match self + .owner + .observe_cleanup(None, token.namespace, token.workspace.to_bytes()) + .map_err(Failure::Owner) + { + Ok(CleanupState::Held) => (ForcedCleanup::Held, None), + Ok(CleanupState::Queued) => (ForcedCleanup::Queued, None), + Ok(CleanupState::Gone) => (ForcedCleanup::Gone, None), + Ok(CleanupState::Live) => ( + ForcedCleanup::Unobserved, + Some(Box::new(Failure::Rejected( + ControlCode::Unknown, + "closed Workspace remains live", + ))), + ), + Err(error) => (ForcedCleanup::Unobserved, Some(Box::new(error))), + }; + self.depart( + token, + Success { + reply: Reply::ForceUnmounted(Box::new(ForceUnmounted { + token, + outcome: ForcedOutcome { + facts, + work, + cleanup, + }, + })), + commit: None, + observation_failure, + kernel, + }, + ) + } +} +/// Terminal admission over an export-owning entry; the entry observes the +/// export as leaving until the operation settles it. +pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + match kernel { + Kernel::Exported(_) => {} + Kernel::ExportLeaving(_) => { + return Err(kernel_failure( + ControlCode::Busy, + "unmount:admission", + "terminal unmount already admitted", + )) + } + _ => { + return Err(kernel_failure( + ControlCode::Invalid, + "unmount:admission", + "Workspace owns no export", + )) + } + } + let Kernel::Exported(exported) = std::mem::replace(kernel, Kernel::Unattached) else { + unreachable!("checked export") + }; + *kernel = Kernel::ExportLeaving(Box::new(ExportLeaving { + ready: exported.ready.clone(), + phase, + work: work(&exported.server), + })); + Ok(*exported) +} diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs new file mode 100644 index 0000000000..a4796ab725 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -0,0 +1,155 @@ +//! One bounded NFS client helper attempt, including post-KILL observation. +use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use std::{ + io::{self, Read}, + process::{Child, Command, Stdio}, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// How long one helper run may take before it is killed and reported. +pub const HELPER_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const PAUSE: Duration = Duration::from_millis(5); +const OUTPUT: usize = 2048; + +/// One helper command; `{port}` and `{dir}` in its words are replaced. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Helper(pub Vec); +impl Helper { + /// Splits a configured command line on whitespace. + pub fn words(line: &str) -> Option { + let words: Vec = line.split_whitespace().map(str::to_owned).collect(); + (!words.is_empty()).then_some(Self(words)) + } + fn command(&self, port: u16, directory: &str) -> Option { + let mut words = self.0.iter().map(|word| { + word.replace("{port}", &port.to_string()) + .replace("{dir}", directory) + }); + let mut command = Command::new(words.next()?); + command.args(words); + Some(command) + } +} + +pub(super) enum Ran { + Done, + Busy(String), + Failed(String), + /// Timeout or observation failure after spawn: effects are not known. + Interrupted(String), +} + +/// Retained children are never waited on synchronously or signalled twice. +/// A later helper attempt may only reap their observed exits, not replay them. +#[derive(Debug, Default)] +pub(super) struct Helpers(Mutex>); +impl Helpers { + pub fn run(&self, helper: &Helper, port: u16, directory: &str) -> Ran { + { + let mut held = self.0.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let Some(mut command) = helper.command(port, directory) else { + return Ran::Failed("empty helper command".into()); + }; + let mut child = match command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(error) => return Ran::Failed(format!("{:?}: {error}", helper.0.first())), + }; + let mut pipes: Vec> = Vec::new(); + let mut pipe_error = None; + // Even an exited helper can leave a descendant holding its pipe open. + // Never wait for EOF, and drain while it runs so verbose helpers progress. + for pipe in [ + child.stderr.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + child.stdout.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + ] + .into_iter() + .flatten() + { + match pipe.1 { + Ok(_) => pipes.push(pipe.0), + Err(error) => pipe_error = Some(format!("helper output: {error}")), + } + } + let mut output = Vec::new(); + let deadline = Instant::now() + HELPER_WAIT; + let status = loop { + drain(&mut pipes, &mut output); + if let Some(error) = pipe_error.take() { + break Err(error); + } + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) => break Err(format!("helper did not finish in {HELPER_WAIT:?}")), + Err(error) => break Err(format!("helper wait: {error}")), + } + }; + let status = match status { + Ok(status) => Ok(status), + Err(detail) => { + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break false, + } + }; + let pid = child.id(); + let disposition = if reaped { + "exit reaped".to_owned() + } else { + self.0 + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; helper may remain in kernel wait; child custody retained".into() + }; + Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) + } + }; + drain(&mut pipes, &mut output); + let output = String::from_utf8_lossy(&output); + match status { + Ok(status) if status.success() => Ran::Done, + Ok(status) => { + let detail = format!("{status}: {}", output.trim()); + if output.to_ascii_lowercase().contains("busy") { + Ran::Busy(detail) + } else { + Ran::Failed(detail) + } + } + Err(detail) => Ran::Interrupted(format!("{detail}: {}", output.trim())), + } + } +} + +fn drain(pipes: &mut [Box], output: &mut Vec) { + for pipe in pipes { + let mut bytes = [0; OUTPUT]; + // One bounded read per pipe and observation: even continuous output + // cannot defer the helper's deadline. Excess bytes are discarded. + match pipe.read(&mut bytes) { + Ok(count) => output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(_) => {} + } + } +} diff --git a/engine/crates/layerfs-daemon/src/control/force.rs b/engine/crates/layerfs-daemon/src/control/force.rs index 6eb780c616..1699cdc211 100644 --- a/engine/crates/layerfs-daemon/src/control/force.rs +++ b/engine/crates/layerfs-daemon/src/control/force.rs @@ -45,7 +45,7 @@ fn refused(code: ControlCode, phase: &'static str, detail: &'static str) -> Fail /// The whole before-effect decision, made under the registry lock and in this /// order. A refusal changes nothing. Admitted, it returns the Commit knowledge /// the receipt will carry: copied here, never settled by a later read. -fn guard( +pub(super) fn guard( value: &Binding, token: WorkspaceToken, relinquish_unknown: bool, @@ -53,6 +53,9 @@ fn guard( // Stored custody answers first: no second abort or detach ever follows // it. The one exception is a normal unmount stalled at Join, admitted // idle with no Commit custody, whose abort control is still unused. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { if kept.forceable() { @@ -60,8 +63,6 @@ fn guard( } return Err(Failure::Retained(Box::new(kept.custody(token)))); } - #[cfg(not(target_os = "linux"))] - let _ = token; if matches!( value.activity, Activity::Committing | Activity::Attaching | Activity::Closing @@ -113,7 +114,6 @@ fn connection(kernel: &Kernel) -> Result { ADMISSION, "Workspace owns no kernel connection", )), - #[cfg(target_os = "linux")] Kernel::Attaching => Err(refused( ControlCode::Unknown, ADMISSION, @@ -129,6 +129,13 @@ fn connection(kernel: &Kernel) -> Result { )), #[cfg(target_os = "linux")] Kernel::Ready(attached) => Ok(attached.session.abort_bound()), + // The forced client unmount is the export's abort. + Kernel::Exported(_) => Ok(true), + Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => Err(refused( + ControlCode::Busy, + ADMISSION, + "terminal unmount already admitted", + )), } } #[cfg(not(target_os = "linux"))] @@ -139,6 +146,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; guard(bound_mut(&mut entries, token)?, token, relinquish_unknown)?; Err(refused( @@ -199,6 +209,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let admitted = self.admit_force(token, relinquish_unknown)?; self.force_admitted(token, admitted) } diff --git a/engine/crates/layerfs-daemon/src/control/invalidate.rs b/engine/crates/layerfs-daemon/src/control/invalidate.rs index 7a7b0dc8b5..19f668df8a 100644 --- a/engine/crates/layerfs-daemon/src/control/invalidate.rs +++ b/engine/crates/layerfs-daemon/src/control/invalidate.rs @@ -17,6 +17,12 @@ impl Service { shown: ObjectId, installed: ObjectId, ) -> Result<(), Failure> { + // An NFS client holds no invalidation channel: the export closes its + // descriptors of the replaced tree and refreshes the stale inodes the + // client may cache through the mount point. + if self.installed_export(token, workspace, shown, installed)? { + return Ok(()); + } #[cfg(target_os = "linux")] { use super::{kernel::Kernel, registry::bound}; diff --git a/engine/crates/layerfs-daemon/src/control/kernel.rs b/engine/crates/layerfs-daemon/src/control/kernel.rs index 92c07ef476..dc37ac187c 100644 --- a/engine/crates/layerfs-daemon/src/control/kernel.rs +++ b/engine/crates/layerfs-daemon/src/control/kernel.rs @@ -31,7 +31,6 @@ pub use super::serving::KernelServing; pub(super) enum Kernel { Unattached, /// One Attach is admitted, or its engine outcome is unknown. - #[cfg(target_os = "linux")] Attaching, #[cfg(target_os = "linux")] Ready(Box), @@ -40,6 +39,12 @@ pub(super) enum Kernel { Leaving(Box), #[cfg(target_os = "linux")] Retained(Box), + /// NFS route: a serving export the host client has mounted. + Exported(Box), + /// NFS route: a terminal operation owns the export. + ExportLeaving(Box), + /// NFS route: export custody a teardown could not finish disposing. + ExportKept(Box), } impl Kernel { /// Maintained observations only: no syscall, SQL job or kernel request. @@ -52,8 +57,10 @@ impl Kernel { }; match self { Self::Unattached => plain(MountPhase::Unattached), - #[cfg(target_os = "linux")] Self::Attaching => plain(MountPhase::Attaching), + Self::Exported(_) | Self::ExportLeaving(_) | Self::ExportKept(_) => { + super::export::block(self).unwrap_or_else(|| plain(MountPhase::Unattached)) + } #[cfg(target_os = "linux")] Self::Ready(attached) => { let facts = attached.session.facts(); diff --git a/engine/crates/layerfs-daemon/src/control/mod.rs b/engine/crates/layerfs-daemon/src/control/mod.rs index 9d42da651e..75abdf4e74 100644 --- a/engine/crates/layerfs-daemon/src/control/mod.rs +++ b/engine/crates/layerfs-daemon/src/control/mod.rs @@ -5,6 +5,8 @@ mod cleanup; #[cfg(target_os = "linux")] mod detach; mod dirty; +mod export; +mod export_helper; mod failure; mod force; mod inspect; @@ -19,6 +21,7 @@ mod serving; mod status; mod types; mod unmount; +pub use export::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; pub use kernel::{KernelConfig, KernelServing}; pub(crate) use observation::{reply_token, request_token}; pub use registry::Service; diff --git a/engine/crates/layerfs-daemon/src/control/observation.rs b/engine/crates/layerfs-daemon/src/control/observation.rs index b8f0edda08..e403c58c59 100644 --- a/engine/crates/layerfs-daemon/src/control/observation.rs +++ b/engine/crates/layerfs-daemon/src/control/observation.rs @@ -57,7 +57,11 @@ impl Service { Kernel::Ready(attached) => Some(attached.session.observer()), Kernel::Leaving(leaving) => Some(leaving.observer.clone()), Kernel::Retained(kept) => kept.observer.clone(), - Kernel::Unattached | Kernel::Attaching => None, + Kernel::Unattached + | Kernel::Attaching + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => None, }) } } diff --git a/engine/crates/layerfs-daemon/src/control/operations.rs b/engine/crates/layerfs-daemon/src/control/operations.rs index 3b44d81058..260b5da1a0 100644 --- a/engine/crates/layerfs-daemon/src/control/operations.rs +++ b/engine/crates/layerfs-daemon/src/control/operations.rs @@ -217,6 +217,12 @@ impl Service { run: impl FnOnce(&BoundWorkspace) -> Result>, ) -> Result { let workspace = self.admit(token, Activity::Committing)?; + // NFS route: what the client buffered is written back first, as a + // FUSE write has already reached the Workspace when it returned. + if let Err(failure) = self.flush_export(token) { + self.settle_activity(token, Activity::Idle); + return Err(failure); + } let result = run(&workspace); // An install that replaced what the Workspace showed is invalidated // in the kernel before the answer. @@ -313,6 +319,31 @@ impl Service { }, } } + /// Poison recovery stores this exact owner only; a poisoned registry never + /// authorizes another product operation. + pub(super) fn settle( + &self, + token: WorkspaceToken, + kernel: super::kernel::Kernel, + activity: Activity, + ) { + let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = kernel; + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + /// Returns an admitted entry to `activity` when its operation stopped + /// before any effect. + fn settle_activity(&self, token: WorkspaceToken, activity: Activity) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + } pub(super) fn job( &self, route: layerfs_workspace::Route, diff --git a/engine/crates/layerfs-daemon/src/control/registry.rs b/engine/crates/layerfs-daemon/src/control/registry.rs index f21131caa2..2bf5e2e910 100644 --- a/engine/crates/layerfs-daemon/src/control/registry.rs +++ b/engine/crates/layerfs-daemon/src/control/registry.rs @@ -33,6 +33,10 @@ pub struct Service { pub(super) entries: Mutex>, /// Absent for a control-only Service: Attach is then refused before effects. pub(super) kernel: Option>, + /// NFS serving (native route); when present, Attach serves NFS. + pub(super) export: Option>, + /// What the last stopped export gave back, for diagnostics. + pub(super) released: Mutex>, } impl Service { /// Binds control to the already initialized owner and its configured capacity. @@ -43,8 +47,26 @@ impl Service { capacity: owner.configuration().namespaces, entries: Mutex::new(BTreeMap::new()), kernel: None, + export: None, + released: Mutex::new(None), } } + /// Binds control and NFS serving (the native route's mount entry) to the + /// same Store and overlay owner. + pub fn with_export( + store: Arc, + owner: &Owner, + export: Arc, + ) -> Self { + Self { + export: Some(export), + ..Self::new(store, owner) + } + } + /// Whether Attach serves NFS rather than FUSE. + pub fn serves_nfs(&self) -> bool { + self.export.is_some() + } /// Binds control and kernel serving to the same Store and overlay owner. pub fn with_kernel(store: Arc, owner: &Owner, kernel: Arc) -> Self { Self { diff --git a/engine/crates/layerfs-daemon/src/control/status.rs b/engine/crates/layerfs-daemon/src/control/status.rs index 9b86d52495..43a4a1a30c 100644 --- a/engine/crates/layerfs-daemon/src/control/status.rs +++ b/engine/crates/layerfs-daemon/src/control/status.rs @@ -44,7 +44,7 @@ impl Service { value.epoch, value.published.clone(), // Copied with the control fields: maintained counters only. - self.kernel.as_ref().map(|_| value.kernel.block()), + (self.kernel.is_some() || self.export.is_some()).then(|| value.kernel.block()), ) }; let observed = self.job( diff --git a/engine/crates/layerfs-daemon/src/control/types.rs b/engine/crates/layerfs-daemon/src/control/types.rs index 320b574fd1..8d4df91f7e 100644 --- a/engine/crates/layerfs-daemon/src/control/types.rs +++ b/engine/crates/layerfs-daemon/src/control/types.rs @@ -65,7 +65,6 @@ impl Success { } } impl Failure { - #[cfg(target_os = "linux")] pub(super) fn kernel( code: ControlCode, phase: &'static str, diff --git a/engine/crates/layerfs-daemon/src/control/unmount.rs b/engine/crates/layerfs-daemon/src/control/unmount.rs index 062b26a19f..5d90daa83d 100644 --- a/engine/crates/layerfs-daemon/src/control/unmount.rs +++ b/engine/crates/layerfs-daemon/src/control/unmount.rs @@ -17,6 +17,8 @@ pub(super) enum Departure { Unattached, #[cfg(target_os = "linux")] Session(Box), + /// NFS route: the export and the client mount of it. + Export(Box), } impl Service { /// Refuses uncommitted changes with their paths unless `discard`. @@ -30,6 +32,7 @@ impl Service { let (workspace, departure) = self.admit_unmount(token)?; match departure { Departure::Unattached => {} + Departure::Export(exported) => return self.unmount_export(token, workspace, *exported), #[cfg(target_os = "linux")] Departure::Session(attached) => { return self.unmount_kernel(token, workspace, *attached) @@ -46,6 +49,9 @@ impl Service { let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; let value = bound_mut(&mut entries, token)?; // Retained custody answers before ordinary activity admission. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { return Err(Failure::Retained(Box::new(kept.custody(token)))); @@ -53,7 +59,12 @@ impl Service { idle(value)?; let departure = match value.kernel { Kernel::Unattached => Departure::Unattached, - #[cfg(target_os = "linux")] + Kernel::Exported(_) | Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => { + Departure::Export(Box::new(super::export::take( + &mut value.kernel, + layerfs_bridge::control::MountPhase::Probing, + )?)) + } Kernel::Attaching => { return Err(Failure::kernel( layerfs_bridge::control::ControlCode::Unknown, @@ -175,13 +186,16 @@ impl Service { } failure } - /// Attach is a Linux capability; elsewhere it is refused before effects. + /// FUSE Attach is a Linux capability; elsewhere only NFS serving attaches. #[cfg(not(target_os = "linux"))] pub(super) fn attach( &self, - _token: WorkspaceToken, + token: WorkspaceToken, _timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return self.attach_export(&export, token); + } Err(Failure::Rejected( layerfs_bridge::control::ControlCode::Invalid, "kernel serving requires Linux", diff --git a/engine/crates/layerfs-nfs/Cargo.toml b/engine/crates/layerfs-nfs/Cargo.toml new file mode 100644 index 0000000000..13828bf3ae --- /dev/null +++ b/engine/crates/layerfs-nfs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "layerfs-nfs" +description = "Loopback NFSv3 service of one mounted Workspace over the kernel request ports" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +layerfs-content = { path = "../../../core/crates/layerfs-content" } +layerfs-overlay = { path = "../layerfs-overlay" } +layerfs-workspace = { path = "../layerfs-workspace" } +layerfs-fuse = { path = "../layerfs-fuse" } + +[dev-dependencies] +layerfs-bridge = { path = "../layerfs-bridge" } +layerfs-daemon = { path = "../layerfs-daemon" } +layerfs-history = { path = "../../../core/crates/layerfs-history" } +layerfs-persistence = { path = "../layerfs-persistence" } +layerfs-project = { path = "../layerfs-project" } +layerfs-storage = { path = "../../../core/crates/layerfs-storage" } +layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } diff --git a/engine/crates/layerfs-nfs/examples/daemon_nfs.rs b/engine/crates/layerfs-nfs/examples/daemon_nfs.rs new file mode 100644 index 0000000000..2ce0ea9dd1 --- /dev/null +++ b/engine/crates/layerfs-nfs/examples/daemon_nfs.rs @@ -0,0 +1,172 @@ +//! The daemon's control Service with NFS serving, driven by control requests +//! (A3 kernel-client runs). +//! +//! usage: daemon_nfs +//! +//! Initializes a Store from `` (Branch `main`), then runs the +//! Workspace control requests the daemon's control connection answers: +//! Mount and Attach (`BOUND`, `READY `); the client mount, flush +//! and unmount are the helpers named by `LAYERFS_NFS_MOUNT`, +//! `LAYERFS_NFS_FLUSH`, `LAYERFS_NFS_UNMOUNT` and `LAYERFS_NFS_FORCE` +//! (`{port}`, `{dir}` replaced), as in the daemon. `LAYERFS_NFS_LOOKUPS` sets +//! the export's reference bound. Then, whenever a trigger file appears: +//! `.status` prints the Status kernel block and the export's +//! holdings (`STATUS …`); `.commit` runs a control Commit +//! (`COMMITTED …`) and serves an independent view of the committed Branch +//! read-only on `` (`VIEW …`); `.unmount` runs a +//! control Unmount without discard (`UNMOUNT …`), and on success stops. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use layerfs_bridge::control::{CommitControl, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter}; +use layerfs_history::WorkspaceId; +use std::{net::TcpListener, path::PathBuf, sync::Arc, time::Duration}; + +fn main() { + let arguments: Vec = std::env::args().collect(); + if arguments.len() != 7 { + eprintln!("usage: daemon_nfs "); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let mounts = PathBuf::from(&arguments[3]); + let view_port: u16 = arguments[4].parse().expect("view port"); + let uid: u32 = arguments[5].parse().expect("uid"); + let gid: u32 = arguments[6].parse().expect("gid"); + let trigger = |name: &str| PathBuf::from(format!("{}.{name}", work.display())); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!("INIT entries={}", rig.entries); + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper("LAYERFS_NFS_MOUNT") { + mounter.mount = mount; + } + if let Some(unmount) = helper("LAYERFS_NFS_UNMOUNT") { + mounter.unmount = unmount; + } + if let Some(force) = helper("LAYERFS_NFS_FORCE") { + mounter.force = force; + } + mounter.flush = helper("LAYERFS_NFS_FLUSH"); + let mut config = ExportServingConfig::new(mounts, uid, gid, mounter); + if let Some(bound) = std::env::var("LAYERFS_NFS_LOOKUPS") + .ok() + .and_then(|value| value.parse().ok()) + { + config.lookups_held = bound; + } + let service = rig.service(Arc::new(ExportServing::start(config))); + let workspace = WorkspaceId::from_authority([41; 32]).unwrap(); + let token = match service + .execute_control(&Request::Mount { + workspace, + branch: rig.branch, + }) + .expect("Mount") + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + println!("BOUND {token:?}"); + let ready = match service.execute_control(&Request::Attach(token)) { + Ok(success) => match success.reply { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }, + Err(failure) => { + println!("ATTACH_FAILED {failure:?}"); + std::process::exit(1); + } + }; + println!("READY {} {}", ready.receipt.mount_id, ready.directory); + let mut views = Vec::new(); + loop { + if trigger("status").exists() { + match service + .execute_control(&Request::Status(token)) + .map(|s| s.reply) + { + Ok(Reply::Status(status)) => println!( + "STATUS phase={:?} work={:?} dirty_inodes={:?} held={:?}", + status.kernel.as_ref().map(|k| k.phase), + status.kernel.as_ref().and_then(|k| k.work), + status.local.as_ref().map(|l| l.dirty_inodes), + service.export_held(token) + ), + other => println!("STATUS {other:?}"), + } + drop(std::fs::remove_file(trigger("status"))); + } + if trigger("commit").exists() { + let at = std::time::Instant::now(); + let done = service.execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("a3 over nfs".into()), + })); + let ms = at.elapsed().as_millis(); + match done { + Ok(success) => { + let kind = match &success.reply { + Reply::Committed(_) => "Committed", + Reply::MergeCommitted(_) => "MergeCommitted", + Reply::Conflict(_) => "Conflict", + _ => "Other", + }; + println!("COMMITTED {kind} ms={ms}"); + let mut authority = [90; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let port = view_port + views.len() as u16; + let viewer = rig.serve_view( + TcpListener::bind(("127.0.0.1", port)).expect("bind view"), + &view, + uid, + gid, + ); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + } + Err(failure) => println!("COMMIT_FAILED ms={ms} {failure:?}"), + } + drop(std::fs::remove_file(trigger("commit"))); + } + if trigger("unmount").exists() { + drop(std::fs::remove_file(trigger("unmount"))); + let held = service.export_held(token); + match service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: false, + })) { + Ok(success) => { + println!("UNMOUNT {:?} held_before={held:?}", success.reply); + if matches!(success.reply, Reply::Unmounted(_)) { + break; + } + } + Err(failure) => println!("UNMOUNT_FAILED {failure:?}"), + } + } + std::thread::sleep(Duration::from_millis(200)); + } + let released = service.export_released(token).unwrap_or_default(); + println!("RELEASED {released:?}"); + println!( + "DIRECTORY_GONE {}", + !std::path::Path::new(&ready.directory).exists() + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + drop(service); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); +} diff --git a/engine/crates/layerfs-nfs/examples/serve_tree.rs b/engine/crates/layerfs-nfs/examples/serve_tree.rs new file mode 100644 index 0000000000..676d063b45 --- /dev/null +++ b/engine/crates/layerfs-nfs/examples/serve_tree.rs @@ -0,0 +1,101 @@ +//! Serves a host tree, committed as a Branch, over loopback NFSv3. +//! +//! usage: serve_tree [uid gid] [write] +//! +//! Prints `READY
` once the export accepts calls. With `write` the +//! export is writable, and whenever `.commit` appears the export +//! closes its implicit file descriptors, the Workspace is committed with the +//! product Commit (`COMMITTED …`), and a second, independent Workspace bound +//! to the committed Branch is served read-only on `+1` (`VIEW …`); +//! the trigger file is then removed. It serves until `.stop` +//! exists; it then prints per-procedure call counts and time, gives back +//! every engine reference, revokes the engine mounts and removes +//! ``. Mount on Linux with, for example: +//! `mount -t nfs -o vers=3,proto=tcp,port=P,mountport=P,mountproto=tcp,nolock HOST:/ DIR`. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use std::{net::TcpListener, path::PathBuf, time::Duration}; + +fn main() { + let mut arguments: Vec = std::env::args().collect(); + let writable = arguments.last().is_some_and(|last| last == "write"); + if writable { + arguments.pop(); + } + if arguments.len() != 4 && arguments.len() != 6 { + eprintln!("usage: serve_tree [uid gid] [write]"); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let port: u16 = arguments[3].parse().expect("port"); + let (uid, gid) = if arguments.len() == 6 { + (arguments[4].parse().unwrap(), arguments[5].parse().unwrap()) + } else { + (501, 20) + }; + let stop = PathBuf::from(format!("{}.stop", work.display())); + let commit = PathBuf::from(format!("{}.commit", work.display())); + let started = std::time::Instant::now(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!( + "INIT entries={} seconds={:.2}", + rig.entries, + started.elapsed().as_secs_f64() + ); + let listener = TcpListener::bind(("127.0.0.1", port)).expect("bind"); + let server = rig.export(listener, uid, gid, !writable); + println!("READY {} writable={writable}", server.local_addr()); + let mut views = Vec::new(); + while !stop.exists() { + if writable && commit.exists() { + let closed = server.close_files(); + let at = std::time::Instant::now(); + let done = rig.commit(); + println!( + "COMMITTED closed_files={closed} ms={} history={:?} replaced={:?}", + at.elapsed().as_millis(), + done.history, + done.replaced + ); + drop(done); + let mut authority = [80; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let listener = + TcpListener::bind(("127.0.0.1", port + 1 + views.len() as u16)).expect("bind"); + let viewer = rig.serve_view(listener, &view, uid, gid); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + drop(std::fs::remove_file(&commit)); + } + std::thread::sleep(Duration::from_millis(200)); + } + let mut total = (0, 0); + for count in server.counters() { + total.0 += count.calls; + total.1 += count.nanoseconds; + println!( + "PROC {:<12} calls={:<8} total_ms={:<10.1} avg_us={:.1}", + count.procedure, + count.calls, + count.nanoseconds as f64 / 1e6, + count.nanoseconds as f64 / 1e3 / count.calls as f64 + ); + } + println!( + "TOTAL calls={} server_ms={:.1}", + total.0, + total.1 as f64 / 1e6 + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + let released = server.stop(); + println!("RELEASED {released:?}"); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); + drop(std::fs::remove_file(stop)); +} diff --git a/engine/crates/layerfs-nfs/src/apple_double.rs b/engine/crates/layerfs-nfs/src/apple_double.rs new file mode 100644 index 0000000000..b64bba7ab6 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/apple_double.rs @@ -0,0 +1,326 @@ +//! AppleDouble companions held by the export alone. +//! +//! On NFSv3 the macOS client keeps a file's extended attributes and resource +//! fork in a `._` companion file (see [`crate::names`]). When the export +//! keeps such names out of the Workspace, a CREATE of one makes a file that +//! lives only here, in memory, under its own serial range: LOOKUP, GETATTR, +//! ACCESS, READ, WRITE, SETATTR, COMMIT, REMOVE and a RENAME to another +//! AppleDouble name work on it, READDIR does not list it, and no mutation of +//! it ever reaches the engine, so the Workspace stays clean and no Commit can +//! capture it. Everything is dropped when the export is released. +//! +//! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] +//! each and [`TOTAL_BYTES`] in all. Beyond the count a creation is refused +//! (`NFS3ERR_ACCES`, as before this module), beyond the bytes a write +//! (`NFS3ERR_FBIG` per file, `NFS3ERR_NOSPC` in all). +use crate::{ + attributes::Status, + change::{How, Sattr, Tickets}, + kernel::Kernel, +}; +use layerfs_content::{filesystem::attributes::PortableMetadata, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::ViewStat; +use std::{ + collections::HashMap, + sync::atomic::Ordering, + time::{SystemTime, UNIX_EPOCH}, +}; + +/// The first serial of the export's own companions, far above any serial the +/// engine allocates. +pub const SERIALS: u64 = 1 << 62; +/// Companions held at once. +pub const HELD: usize = 4096; +/// Largest companion kept. +pub const FILE_BYTES: u64 = 1 << 20; +/// Bytes kept over all companions. +pub const TOTAL_BYTES: usize = 64 << 20; +/// Mode of a companion created without one. +const MODE: u32 = 0o644; + +struct Companion { + named: (u64, Vec), + data: Vec, + metadata: PortableMetadata, +} +/// The companions of one export. +#[derive(Default)] +pub(crate) struct Companions { + next: u64, + files: HashMap, + names: HashMap<(u64, Vec), u64>, + bytes: usize, +} +impl Companions { + pub(crate) fn len(&self) -> usize { + self.files.len() + } + fn stat(serial: u64, file: &Companion) -> ViewStat { + ViewStat { + serial, + kind: InodeKind::RegularFile, + metadata: file.metadata, + namespace_refs: 1, + logical_len: file.data.len() as u64, + subdirs: 0, + } + } + fn drop_serial(&mut self, serial: u64) { + if let Some(file) = self.files.remove(&serial) { + self.names.remove(&file.named); + self.bytes -= file.data.len(); + } + } + /// Resizes `serial` to `size`, within the bounds. + fn resize(&mut self, serial: u64, size: u64) -> Result<(), Status> { + if size > FILE_BYTES { + return Err(Status::FBig); + } + let file = self.files.get_mut(&serial).ok_or(Status::Stale)?; + let old = file.data.len(); + let new = size as usize; + if new > old && self.bytes + (new - old) > TOTAL_BYTES { + return Err(Status::NoSpc); + } + file.data.resize(new, 0); + self.bytes = self.bytes + new - old; + Ok(()) + } +} +fn now() -> PortableMetadata { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + PortableMetadata { + mode: MODE, + mtime_seconds: i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX), + mtime_nanoseconds: elapsed.subsec_nanos(), + } +} +fn touch(metadata: &mut PortableMetadata) { + let fresh = now(); + metadata.mtime_seconds = fresh.mtime_seconds; + metadata.mtime_nanoseconds = fresh.mtime_nanoseconds; +} +/// A serial of the export's own companions. +pub fn companion(serial: u64) -> bool { + serial >= SERIALS +} + +impl Kernel { + fn companions(&self) -> std::sync::MutexGuard<'_, Companions> { + self.companions + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// GETATTR of a companion serial. + pub(crate) fn companion_stat(&self, serial: u64) -> Result { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + Ok(Companions::stat(serial, file)) + } + /// LOOKUP of a companion name, if one is held. + pub(crate) fn companion_named(&self, parent: u64, name: &[u8]) -> Option { + let companions = self.companions(); + let serial = *companions.names.get(&(parent, name.to_vec()))?; + companions + .files + .get(&serial) + .map(|file| Companions::stat(serial, file)) + } + /// CREATE of an AppleDouble name. + pub(crate) fn companion_create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + if let Some(stat) = self.companion_named(parent, name) { + // Opened again: UNCHECKED applies its size, as an open with + // truncation would; GUARDED and EXCLUSIVE find it taken. + let How::Unchecked(sattr) = how else { + return Err(Status::Exist); + }; + return match sattr.size { + Some(size) => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.companion_setattr(stat.serial, sized, None, tickets) + } + None => Ok(stat), + }; + } + // A directory must exist and a companion the tree itself carries + // stays the tree's: neither is ours to shadow. + let directory = self.getattr(parent)?; + if directory.kind != InodeKind::Directory { + return Err(Status::NotDir); + } + let taken = match self.lookup(parent, name) { + Ok(_) => true, + Err(Status::NoEnt) => false, + Err(status) => return Err(status), + }; + let mut companions = self.companions(); + if taken || companions.len() >= HELD { + drop(companions); + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + let mut metadata = now(); + if let How::Unchecked(sattr) | How::Guarded(sattr) = how { + if let Some(mode) = sattr.mode { + metadata.mode = mode & 0o777; + } + } + let serial = SERIALS + companions.next; + companions.next += 1; + let named = (parent, name.to_vec()); + companions.names.insert(named.clone(), serial); + companions.files.insert( + serial, + Companion { + named, + data: Vec::new(), + metadata, + }, + ); + let file = &companions.files[&serial]; + let stat = Companions::stat(serial, file); + drop(companions); + self.names_detached.fetch_add(1, Ordering::Relaxed); + Ok(stat) + } + /// REMOVE of a companion name; `false` when none is held. + pub(crate) fn companion_remove(&self, parent: u64, name: &[u8]) -> bool { + let mut companions = self.companions(); + match companions.names.get(&(parent, name.to_vec())).copied() { + Some(serial) => { + companions.drop_serial(serial); + true + } + None => false, + } + } + /// RENAME of a companion name: only to another AppleDouble name, which + /// it replaces when held. `None` when no companion has that name. + pub(crate) fn companion_rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + ) -> Option> { + let mut companions = self.companions(); + let serial = *companions.names.get(&(from, name.to_vec()))?; + if !crate::names::apple_double(new_name) || new_name.len() > 255 { + return Some(Err(Status::Access)); + } + let target = (to, new_name.to_vec()); + if let Some(replaced) = companions.names.get(&target).copied() { + if replaced != serial { + companions.drop_serial(replaced); + } + } + companions.names.remove(&(from, name.to_vec())); + companions.names.insert(target.clone(), serial); + if let Some(file) = companions.files.get_mut(&serial) { + file.named = target; + } + Some(Ok(())) + } + /// READ of a companion. + pub(crate) fn companion_read( + &self, + serial: u64, + offset: u64, + count: u32, + ) -> Result<(Vec, bool), Status> { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let length = file.data.len() as u64; + let start = offset.min(length) as usize; + let end = offset.saturating_add(u64::from(count)).min(length) as usize; + Ok((file.data[start..end].to_vec(), end as u64 >= length)) + } + /// WRITE to a companion: all of it or nothing. + pub(crate) fn companion_write( + &self, + serial: u64, + offset: u64, + data: &[u8], + ) -> Result<(u32, ViewStat), Status> { + let mut companions = self.companions(); + let end = offset.checked_add(data.len() as u64).ok_or(Status::FBig)?; + let length = companions + .files + .get(&serial) + .ok_or(Status::Stale)? + .data + .len() as u64; + if end > length { + companions.resize(serial, end)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + file.data[offset as usize..end as usize].copy_from_slice(data); + touch(&mut file.metadata); + let stat = Companions::stat(serial, file); + Ok((data.len() as u32, stat)) + } + /// SETATTR of a companion: mode, size and modification time; the + /// owner is the export's identity, as for every inode. + pub(crate) fn companion_setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + _tickets: &mut Tickets, + ) -> Result { + let identity = self.identity(); + let foreign = sattr.uid.is_some_and(|uid| uid != identity.uid) + || sattr.gid.is_some_and(|gid| gid != identity.gid); + if foreign { + return Err(Status::Perm); + } + let mut companions = self.companions(); + if let Some(expected) = guard { + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let current = Companions::stat(serial, file); + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if let Some(size) = sattr.size { + companions.resize(serial, size)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + if let Some(mode) = sattr.mode { + file.metadata.mode = mode & 0o777; + } + match sattr.mtime { + Some(Stamp::At(time)) => { + file.metadata.mtime_seconds = time.seconds; + file.metadata.mtime_nanoseconds = time.nanoseconds; + } + Some(Stamp::Processing) => touch(&mut file.metadata), + None if sattr.size.is_some() => touch(&mut file.metadata), + None => {} + } + Ok(Companions::stat(serial, file)) + } + /// Drops every companion; their count. + pub(crate) fn companions_release(&self) -> usize { + let mut companions = self.companions(); + let held = companions.len(); + *companions = Companions::default(); + held + } + /// Companions held now. + pub(crate) fn companions_held(&self) -> usize { + self.companions().len() + } +} diff --git a/engine/crates/layerfs-nfs/src/attributes.rs b/engine/crates/layerfs-nfs/src/attributes.rs new file mode 100644 index 0000000000..70fd740c1d --- /dev/null +++ b/engine/crates/layerfs-nfs/src/attributes.rs @@ -0,0 +1,204 @@ +//! NFSv3 (RFC 1813) status codes and the projection of Workspace inode facts. +use crate::xdr::Writer; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_workspace::{Refusal, ViewStat}; + +/// `nfsstat3` values this server returns. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum Status { + Ok = 0, + Perm = 1, + NoEnt = 2, + Io = 5, + Access = 13, + Exist = 17, + NotDir = 20, + IsDir = 21, + Inval = 22, + FBig = 27, + NoSpc = 28, + RoFs = 30, + MLink = 31, + NameTooLong = 63, + NotEmpty = 66, + Stale = 70, + BadHandle = 10001, + NotSync = 10002, + BadCookie = 10003, + NotSupp = 10004, + TooSmall = 10005, + ServerFault = 10006, + /// The request may succeed later; the client retries after a delay. + Jukebox = 10008, +} +impl Status { + pub const fn code(self) -> u32 { + self as u32 + } +} +/// A Workspace refusal as the status the client sees; the same errnos the +/// FUSE adapter replies with. +pub fn refusal(value: Refusal) -> Status { + match value { + Refusal::Exists => Status::Exist, + Refusal::Missing => Status::NoEnt, + Refusal::NotDirectory => Status::NotDir, + Refusal::IsDirectory => Status::IsDir, + Refusal::NotEmpty => Status::NotEmpty, + Refusal::NotPermitted => Status::Perm, + Refusal::Unsupported => Status::NotSupp, + Refusal::Invalid | Refusal::AncestryRequired | Refusal::AncestryMismatch => Status::Inval, + Refusal::TooManyLinks => Status::MLink, + Refusal::TooLarge => Status::FBig, + } +} + +/// `ftype3` of a Workspace inode kind. +pub const fn file_type(kind: InodeKind) -> u32 { + match kind { + InodeKind::RegularFile => 1, + InodeKind::Directory => 2, + InodeKind::Symlink => 5, + } +} + +/// The identity every inode is shown as, and the export's fixed facts. +#[derive(Clone, Copy, Debug)] +pub struct Identity { + /// Serial of the mount's root directory. + pub root: u64, + pub uid: u32, + pub gid: u32, + /// `fsid` of every attribute; one export is one filesystem. + pub fsid: u64, +} +impl Identity { + /// POSIX link count, as the FUSE adapter projects it: a directory's is + /// derived (0 once removed, else two plus its child directories); any + /// other kind's is its namespace reference count. + pub fn nlink(self, value: &ViewStat) -> u32 { + if value.kind == InodeKind::Directory { + if value.namespace_refs == 0 && value.serial != self.root { + 0 + } else { + u32::try_from(value.subdirs.saturating_add(2)).unwrap_or(u32::MAX) + } + } else { + u32::try_from(value.namespace_refs).unwrap_or(u32::MAX) + } + } + /// Encodes one `fattr3`. The Workspace keeps one timestamp; atime, mtime + /// and ctime all show it. `fileid` is the serial itself. + pub fn fattr3(self, out: &mut Writer, value: &ViewStat) { + let (seconds, nanoseconds) = time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ); + let used = value.logical_len.div_ceil(512).saturating_mul(512); + out.u32(file_type(value.kind)); + out.u32(value.metadata.mode & 0o7777); + out.u32(self.nlink(value)); + out.u32(self.uid).u32(self.gid); + out.u64(value.logical_len).u64(used); + out.u32(0).u32(0); // rdev + out.u64(self.fsid).u64(value.serial); + for _ in 0..3 { + out.u32(seconds).u32(nanoseconds); + } + } + /// `post_op_attr` carrying attributes. + pub fn post_op(self, out: &mut Writer, value: &ViewStat) { + out.bool(true); + self.fattr3(out, value); + } +} +/// The `ctime` a client sees for `value` (the one stored timestamp), as the +/// SETATTR guard compares it. +pub fn ctime(value: &ViewStat) -> (u32, u32) { + time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ) +} +/// `wcc_attr` (size, mtime, ctime) of a `pre_op_attr` carrying attributes. +pub fn pre_op(out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => { + let (seconds, nanoseconds) = ctime(value); + out.bool(true).u64(value.logical_len); + out.u32(seconds).u32(nanoseconds); + out.u32(seconds).u32(nanoseconds); + } + None => { + out.bool(false); + } + } +} +/// `post_op_attr`, present or absent. +pub fn post_op_maybe(identity: Identity, out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => identity.post_op(out, value), + None => { + out.bool(false); + } + } +} +/// `nfstime3` holds unsigned seconds; times before the epoch show as the epoch. +fn time(seconds: i64, nanoseconds: u32) -> (u32, u32) { + match u32::try_from(seconds) { + Ok(seconds) => (seconds, nanoseconds.min(999_999_999)), + Err(_) if seconds < 0 => (0, 0), + Err(_) => (u32::MAX, 999_999_999), + } +} + +/// ACCESS3 bits. +pub const ACCESS_READ: u32 = 0x01; +pub const ACCESS_LOOKUP: u32 = 0x02; +pub const ACCESS_MODIFY: u32 = 0x04; +pub const ACCESS_EXTEND: u32 = 0x08; +pub const ACCESS_DELETE: u32 = 0x10; +pub const ACCESS_EXECUTE: u32 = 0x20; + +/// The ACCESS answer for a caller. Every inode belongs to the one command +/// identity; a caller with that uid gets the owner bits, a caller in that +/// group the group bits, anyone else the other bits. uid 0 gets no +/// superuser override: LayerFS has no root semantics, and root inside a +/// sandbox must not bypass the mode. A read-only export grants no change. +pub fn access( + identity: Identity, + value: &ViewStat, + caller: Option<(u32, u32, &[u32])>, + requested: u32, + read_only: bool, +) -> u32 { + let mode = value.metadata.mode; + let bits = match caller { + Some((uid, _, _)) if uid == identity.uid || uid == 0 => (mode >> 6) & 7, + Some((_, gid, groups)) if gid == identity.gid || groups.contains(&identity.gid) => { + (mode >> 3) & 7 + } + _ => mode & 7, + }; + let (read, write, execute) = (bits & 4 != 0, bits & 2 != 0, bits & 1 != 0); + let directory = value.kind == InodeKind::Directory; + let mut granted = 0; + if read { + granted |= ACCESS_READ; + } + if write && !read_only { + granted |= ACCESS_MODIFY | ACCESS_EXTEND; + if directory { + granted |= ACCESS_DELETE; + } + } + if execute { + granted |= if directory { + ACCESS_LOOKUP + } else { + ACCESS_EXECUTE + }; + } + granted & requested +} diff --git a/engine/crates/layerfs-nfs/src/block.rs b/engine/crates/layerfs-nfs/src/block.rs new file mode 100644 index 0000000000..7449bf9d34 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/block.rs @@ -0,0 +1,32 @@ +//! Runs one port future to completion on the connection thread. +//! +//! The service ports return boxed futures; the daemon's implementations run +//! their owner job on the calling thread and are ready on first poll. A +//! future that is not ready parks this thread until its waker unparks it. +use std::{ + future::Future, + sync::Arc, + task::{Context, Poll, Wake, Waker}, + thread::{self, Thread}, +}; + +struct Unpark(Thread); +impl Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + fn wake_by_ref(self: &Arc) { + self.0.unpark(); + } +} +pub fn wait(future: impl Future) -> T { + let mut future = std::pin::pin!(future); + let waker = Waker::from(Arc::new(Unpark(thread::current()))); + let mut context = Context::from_waker(&waker); + loop { + if let Poll::Ready(value) = future.as_mut().poll(&mut context) { + return value; + } + thread::park(); + } +} diff --git a/engine/crates/layerfs-nfs/src/change.rs b/engine/crates/layerfs-nfs/src/change.rs new file mode 100644 index 0000000000..a1c5008d16 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/change.rs @@ -0,0 +1,549 @@ +//! The changing procedures over the kernel mutation ports: one decided plan +//! per engine visit, and every publication ticket released only after the +//! call's single reply attempt. +//! +//! Each procedure runs [`KernelMutation::perform`], the same orchestration +//! the FUSE connection uses: the serial of a creating operation is reserved +//! there (an exact allocator contention is the declined `Contended`, served +//! as `NFS3ERR_JUKEBOX` so the client retries later with nothing created), +//! need rounds read base facts, and the one publishing visit hands back a +//! [`KernelMutation`] that still owns its ticket. The procedure puts it in +//! the call's [`Tickets`]; the connection releases them with +//! [`Tickets::replied`] after `write_record` returned, whether or not the +//! reply reached the client. A failure that keeps custody is retained, never +//! released or undone on a guess; a fenced step or a failed base demand of +//! this request alone is relinquished, as the FUSE connection does. +use crate::{ + attributes::{refusal, Status}, + block::wait, + kernel::Kernel, + names, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::{ + create, link, remove, rename, + write::{self, AttributeChange, Stamp}, + Declined, KernelMutation, MutationFailure, MutationInput, MutationRequest, +}; +use layerfs_overlay::WRITE_WINDOW; +use layerfs_workspace::{Time, ViewStat}; +use std::{io, sync::atomic::Ordering, time::SystemTime}; + +/// The page a client reads a symlink target into; a longer target could not +/// be read back whole and is refused at creation, as on the FUSE side. +pub const LINK_PAGE: u32 = 4096; +/// Modes when a CREATE or MKDIR names none (clients normally send one). +const FILE_MODE: u32 = 0o644; +const DIRECTORY_MODE: u32 = 0o755; +/// EXCLUSIVE CREATE carries no attributes; the client sets them right after. +const EXCLUSIVE_MODE: u32 = 0o600; + +/// Mutations decided while serving one call, each owed exactly one release +/// of its publication ticket after that call's reply attempt. A declined or +/// unchanged mutation holds no ticket; its release issues no job. +#[derive(Default)] +#[must_use = "every decided mutation is owed its post-reply release"] +pub struct Tickets(Vec); +impl Tickets { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + /// Called only after the reply's single send attempt returned. A release + /// that fails is retained with its remaining custody. + pub fn replied(self, kernel: &Kernel) { + for mutation in self.0 { + if let Err(failure) = wait(mutation.replied()) { + kernel.retain(failure); + } + } + } +} + +/// The settable attributes of `sattr3`. Access time is accepted and not +/// stored (the Workspace keeps one timestamp; the mount is `noatime`). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Sattr { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + pub mtime: Option, +} +impl Sattr { + fn change(self) -> AttributeChange { + AttributeChange { + mode: self.mode, + owner: self.uid, + group: self.gid, + size: self.size, + mtime: self.mtime, + } + } + fn is_empty(self) -> bool { + self.change() == AttributeChange::default() + } +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Sattr), + Guarded(Sattr), + Exclusive([u8; 8]), +} +/// What a published result must carry for its reply to be composed. +#[derive(Clone, Copy)] +enum Expect { + Nothing, + /// The published inode, with this serial when given. + Stat(Option), +} +/// A declined mutation as the status the client sees. +pub fn declined(reason: Declined) -> Status { + match reason { + Declined::Refused(value) => refusal(value), + Declined::NoSpace => Status::NoSpc, + Declined::Contended => Status::Jukebox, + Declined::TargetTooLong => Status::NameTooLong, + } +} + +impl Kernel { + /// One kernel mutation. Its decided [`KernelMutation`] goes to `tickets` + /// whatever the answer; a failure is relinquished or retained. + fn mutate( + &self, + protected: u64, + handle: Option, + expect: Expect, + tickets: &mut Tickets, + build: impl FnOnce(Time) -> Result, + ) -> Result, Status> { + let now = write::time(SystemTime::now()).ok_or(Status::Io)?; + let input = build(now).map_err(declined)?; + let (services, request) = self.request()?; + let request = MutationRequest { + mount: self.mount(), + request, + protected, + handle, + input, + open: None, + now, + }; + let mutation = match wait(KernelMutation::perform(services, request)) { + Ok(mutation) => mutation, + Err(failure) => { + let status = if crate::kernel::failure_is_stale(&failure.reason) { + Status::Stale + } else { + Status::Io + }; + self.failed(failure); + return Err(status); + } + }; + let stat = match mutation.value() { + Err(reason) => { + tickets.0.push(mutation); + return Err(declined(reason)); + } + Ok(published) => published.stat.clone(), + }; + if let Expect::Stat(serial) = expect { + let composable = stat + .as_ref() + .is_some_and(|stat| serial.is_none_or(|serial| serial == stat.serial)); + if !composable { + // The reply cannot be composed from the published result: + // an error for the client and a retained mutation. + let reason = io::Error::other("NFS reply composition").into(); + self.retain(mutation.retain(reason)); + return Err(Status::Io); + } + } + tickets.0.push(mutation); + Ok(stat) + } + fn failed(&self, failure: MutationFailure) { + if failure.fenced() || failure.base_demand().is_some() { + if let Err(failure) = wait(failure.relinquish()) { + self.retain(failure); + } + } else { + self.retain(failure); + } + } + /// A name a changing procedure may bind (`creating`) or remove. + fn checked(&self, name: &[u8], creating: bool) -> Result { + match name { + b"" => return Err(Status::Inval), + b"." | b".." => { + return Err(if creating { + Status::Exist + } else { + Status::Inval + }) + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ if creating && self.refuses_apple_double() && names::apple_double(name) => { + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + _ => {} + } + PathName::from_bytes(name).map_err(|_| Status::Inval) + } + /// The entry a name binds now, or none. + fn existing(&self, parent: u64, name: &[u8]) -> Result, Status> { + match self.lookup(parent, name) { + Ok(stat) => Ok(Some(stat)), + Err(Status::NoEnt) => Ok(None), + Err(status) => Err(status), + } + } + /// CREATE of a regular file. + pub fn create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + if self.refuses_apple_double() && names::apple_double(name) && name.len() <= 255 { + return self.companion_create(parent, name, how, tickets); + } + let checked = self.checked(name, true)?; + let key = (parent, name.to_vec()); + if let How::Exclusive(verifier) = how { + // A retransmitted EXCLUSIVE CREATE finds its own file. + let known = self.table().verifiers.get(&key).copied(); + if let Some((stored, serial, _)) = known { + if stored == verifier { + if let Some(stat) = self.existing(parent, name)? { + if stat.serial == serial { + return Ok(stat); + } + } + } + } + } + let (mode, rest) = match how { + How::Unchecked(sattr) | How::Guarded(sattr) => ( + sattr.mode.unwrap_or(FILE_MODE), + Sattr { + mode: None, + ..sattr + }, + ), + How::Exclusive(_) => (EXCLUSIVE_MODE, Sattr::default()), + }; + let created = self.mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::create(parent, checked, mode))) + }); + let stat = match (created, how) { + (Ok(stat), _) => stat.ok_or(Status::ServerFault)?, + (Err(Status::Exist), How::Unchecked(sattr)) => { + // UNCHECKED over an existing regular file: only its size is + // applied, as an open with truncation would. + let stat = self.existing(parent, name)?.ok_or(Status::Exist)?; + if stat.kind != InodeKind::RegularFile { + return Err(Status::Exist); + } + return match sattr.size { + Some(size) if size != stat.logical_len => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.setattr(stat.serial, sized, None, tickets) + } + _ => Ok(stat), + }; + } + (Err(status), _) => return Err(status), + }; + self.entry_taken(&stat, parent, name); + if let How::Exclusive(verifier) = how { + let mut table = self.table(); + let tick = table.tick(); + if table.verifiers.len() >= crate::kernel::VERIFIERS_HELD { + let oldest = table + .verifiers + .iter() + .min_by_key(|(_, entry)| entry.2) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + table.verifiers.remove(&oldest); + } + } + table.verifiers.insert(key, (verifier, stat.serial, tick)); + } + self.settle(stat, rest, tickets) + } + /// Applies what a creating call's attributes ask beyond the mode. + fn settle( + &self, + stat: ViewStat, + rest: Sattr, + tickets: &mut Tickets, + ) -> Result { + let rest = Sattr { + size: rest.size.filter(|size| *size != stat.logical_len), + ..rest + }; + if rest.is_empty() { + return Ok(stat); + } + self.setattr(stat.serial, rest, None, tickets) + } + /// MKDIR. + pub fn mkdir( + &self, + parent: u64, + name: &[u8], + sattr: Sattr, + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let mode = sattr.mode.unwrap_or(DIRECTORY_MODE); + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::mkdir(parent, checked, mode))) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + let rest = Sattr { + mode: None, + size: None, + ..sattr + }; + self.settle(stat, rest, tickets) + } + /// SYMLINK. A symlink's mode is fixed; its attributes are not applied. + pub fn symlink( + &self, + parent: u64, + name: &[u8], + target: &[u8], + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + create::symlink(parent, checked, target, LINK_PAGE).map(MutationInput::Named) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// The silly-named serial `name` denotes in `parent`, if any. + fn silly_entry(&self, parent: u64, name: &[u8]) -> Option { + self.table().silly.get(&(parent, name.to_vec())).copied() + } + /// Drops one silly name; the inode goes back once no silly name keeps it. + fn drop_silly(&self, parent: u64, name: &[u8]) { + let serial = { + let mut table = self.table(); + let serial = table.silly.remove(&(parent, name.to_vec())); + serial.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = serial { + self.unlinked(serial); + } + } + /// REMOVE of a non-directory name. + pub fn remove(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + if self.companion_remove(parent, name) { + return Ok(()); + } + if self.silly_entry(parent, name).is_some() { + self.drop_silly(parent, name); + return Ok(()); + } + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(parent, checked))) + })?; + if target.namespace_refs <= 1 && !self.table().pinned(target.serial) { + self.unlinked(target.serial); + } + Ok(()) + } + /// RMDIR. + pub fn rmdir(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::rmdir(parent, checked))) + })?; + self.unlinked(target.serial); + Ok(()) + } + /// RENAME, replacing the destination. A rename onto a client silly name + /// is the source's unlink with the inode kept alive by the export. + pub fn rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + tickets: &mut Tickets, + ) -> Result<(), Status> { + if let Some(renamed) = self.companion_rename(from, name, to, new_name) { + return renamed; + } + if let Some(serial) = self.silly_entry(from, name) { + // Moving a silly name to another silly name keeps it ours; + // giving it a real name again would need a link of an unlinked + // inode, which the Workspace refuses. + if !names::silly(new_name) { + return Err(Status::Access); + } + let mut table = self.table(); + table.silly.remove(&(from, name.to_vec())); + table.silly.insert((to, new_name.to_vec()), serial); + return Ok(()); + } + let source_name = self.checked(name, false)?; + let destination_name = self.checked(new_name, true)?; + let source = self.existing(from, name)?.ok_or(Status::NoEnt)?; + if names::silly(new_name) && source.kind == InodeKind::RegularFile { + // Keep a writable descriptor before the last name may go. + self.open_file(source.serial, true)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(from, source_name))) + })?; + self.silly_renamed.fetch_add(1, Ordering::Relaxed); + let replaced = { + let mut table = self.table(); + let replaced = table.silly.insert((to, new_name.to_vec()), source.serial); + replaced.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = replaced { + self.unlinked(serial); + } + return Ok(()); + } + let destination = self.existing(to, new_name)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + rename::rename(from, source_name, to, destination_name, 0).map(MutationInput::Named) + })?; + if source.kind == InodeKind::Directory { + self.saw_directory(source.serial, to); + } + self.renamed(source.serial, to, new_name); + if let Some(replaced) = destination.filter(|stat| stat.serial != source.serial) { + let last = replaced.kind == InodeKind::Directory || replaced.namespace_refs <= 1; + if last && !self.table().pinned(replaced.serial) { + self.unlinked(replaced.serial); + } + } + Ok(()) + } + /// LINK: one more name for `serial`. + pub fn link( + &self, + serial: u64, + parent: u64, + name: &[u8], + tickets: &mut Tickets, + ) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::Access); + } + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(Some(serial)), tickets, |_| { + Ok(MutationInput::Named(link::link(serial, parent, checked))) + })? + .ok_or(Status::ServerFault)?; + // The engine took one more lookup reference; the export holds one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// SETATTR. `guard` is the client's expected ctime (seconds, + /// nanoseconds), compared with a fresh GETATTR first: the check is not + /// atomic with the change. + pub fn setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + tickets: &mut Tickets, + ) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_setattr(serial, sattr, guard, tickets); + } + if let Some(expected) = guard { + let current = self.getattr(serial)?; + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if sattr.is_empty() || self.refreshed(serial, &sattr) { + return self.getattr(serial); + } + let identity = (self.identity().uid, self.identity().gid); + let change = sattr.change(); + // A silly-named inode has no name left: a size change goes through + // its descriptor, as an ftruncate would. + let pinned = self.table().pinned(serial); + let handle = if pinned && change.size.is_some() { + Some(self.open_file(serial, true)?.handle) + } else { + None + }; + let stat = self + .mutate(serial, handle, Expect::Stat(Some(serial)), tickets, |now| { + write::set_attributes(serial, handle.is_some(), identity, change, now) + })? + .ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok(stat) + } + /// WRITE through the implicit writable descriptor, in engine windows of + /// at most `WRITE_WINDOW` bytes, each its own publication. A failure after + /// the first window is a short write of the windows already published. + pub fn write( + &self, + serial: u64, + offset: u64, + data: &[u8], + tickets: &mut Tickets, + ) -> Result<(u32, ViewStat), Status> { + if crate::apple_double::companion(serial) { + return self.companion_write(serial, offset, data); + } + let open = self.open_file(serial, true)?; + if data.is_empty() { + return Ok((0, self.getattr(serial)?)); + } + let mut written = 0usize; + let mut last = None; + for window in data.chunks(WRITE_WINDOW) { + let at = offset.checked_add(written as u64).ok_or(Status::FBig)?; + let published = self.mutate( + serial, + Some(open.handle), + Expect::Stat(Some(serial)), + tickets, + |_| write::write(at, window, false), + ); + match published { + Ok(stat) => { + written += window.len(); + last = stat; + } + Err(status) if written == 0 => return Err(status), + Err(_) => break, + } + } + let stat = last.ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok((u32::try_from(written).unwrap_or(u32::MAX), stat)) + } +} diff --git a/engine/crates/layerfs-nfs/src/changing.rs b/engine/crates/layerfs-nfs/src/changing.rs new file mode 100644 index 0000000000..408c914639 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/changing.rs @@ -0,0 +1,268 @@ +//! Arguments and results of the changing NFSv3 procedures (RFC 1813): +//! SETATTR, WRITE, CREATE, MKDIR, SYMLINK, MKNOD, REMOVE, RMDIR, RENAME, +//! LINK and COMMIT. +//! +//! Every directory a procedure changes is answered with `wcc_data`: its +//! attributes read just before the change and just after. The two reads are +//! separate visits, not atomic with the change; the connection serves one +//! call at a time, so they bracket it exactly unless another connection or +//! a Commit installs a change in between, in which case the client sees an +//! unexplained difference and drops its cache, which is safe. +//! +//! WRITE answers `FILE_SYNC`: every window was published to the Workspace +//! before the reply, as every FUSE write is. That is publication, not +//! durability; the write verifier changes with every export, so a client +//! that ever sent UNSTABLE data resends it to a restarted export. +use crate::{ + attributes::{post_op_maybe, pre_op, Status}, + change::{How, Sattr, Tickets}, + kernel::Kernel, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::{Time, ViewStat}; + +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Longest symlink target a SYMLINK may carry. +const PATH_LIMIT: usize = 64 * 1024; +/// Longest WRITE payload accepted in one call (two engine windows of slack +/// over what FSINFO declares). +const WRITE_LIMIT: usize = 1 << 20; +/// `stable_how` FILE_SYNC. +const FILE_SYNC: u32 = 2; + +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +fn time_how(arguments: &mut Reader<'_>) -> Result, Garbage> { + Ok(match arguments.u32()? { + 0 => None, + 1 => Some(Stamp::Processing), + 2 => { + let seconds = arguments.u32()?; + let nanoseconds = arguments.u32()?; + Some(Stamp::At(Time { + seconds: i64::from(seconds), + nanoseconds: nanoseconds.min(999_999_999), + })) + } + _ => return Err(Garbage), + }) +} +/// `sattr3`. The access time is decoded and dropped. +pub fn sattr(arguments: &mut Reader<'_>) -> Result { + let mode = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let uid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let gid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let size = arguments.bool()?.then(|| arguments.u64()).transpose()?; + let _atime = time_how(arguments)?; + let mtime = time_how(arguments)?; + Ok(Sattr { + mode, + uid, + gid, + size, + mtime, + }) +} +/// `wcc_data` of one object around a change. +fn wcc(kernel: &Kernel, out: &mut Writer, before: Option<&ViewStat>, after: Option<&ViewStat>) { + pre_op(out, before); + post_op_maybe(kernel.identity(), out, after); +} +/// Attributes of `serial` when the handle was accepted. +fn attributes(kernel: &Kernel, serial: &Result) -> Option { + serial + .as_ref() + .ok() + .and_then(|serial| kernel.getattr(*serial).ok()) +} +/// The results of CREATE, MKDIR and SYMLINK. +fn created( + kernel: &Kernel, + out: &mut Writer, + result: Result, + directory: &Result, + before: Option<&ViewStat>, +) { + let after = attributes(kernel, directory); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.bool(true).opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + wcc(kernel, out, before, after.as_ref()); +} +/// Decodes the arguments of changing `procedure`, runs it and appends its +/// results. Mutations it decides go to `tickets`, released by the caller +/// after the reply attempt. `Err` is a garbage-arguments refusal. +pub fn call( + kernel: &Kernel, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Garbage> { + match procedure { + 2 => { + let serial = handle(kernel, arguments)?; + let change = sattr(arguments)?; + let guard = if arguments.bool()? { + Some((arguments.u32()?, arguments.u32()?)) + } else { + None + }; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.setattr(serial, change, guard, tickets)); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 7 => { + let serial = handle(kernel, arguments)?; + let offset = arguments.u64()?; + let count = arguments.u32()? as usize; + let _stable = arguments.u32()?; + let data = arguments.opaque(WRITE_LIMIT)?; + let data = &data[..count.min(data.len())]; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.write(serial, offset, data, tickets)); + match result { + Ok((written, stat)) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + out.u32(written).u32(FILE_SYNC).fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 8..=10 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match procedure { + 8 => { + let how = match arguments.u32()? { + 0 => How::Unchecked(sattr(arguments)?), + 1 => How::Guarded(sattr(arguments)?), + 2 => { + let mut verifier = [0; 8]; + verifier.copy_from_slice(arguments.fixed(8)?); + How::Exclusive(verifier) + } + _ => return Err(Garbage), + }; + directory.and_then(|parent| kernel.create(parent, &name, how, tickets)) + } + 9 => { + let change = sattr(arguments)?; + directory.and_then(|parent| kernel.mkdir(parent, &name, change, tickets)) + } + _ => { + let _attributes = sattr(arguments)?; + let target = arguments.opaque(PATH_LIMIT)?.to_vec(); + directory.and_then(|parent| kernel.symlink(parent, &name, &target, tickets)) + } + }; + created(kernel, out, result, &directory, before.as_ref()); + } + 11 => { + // Only regular files, directories and symlinks exist in the + // canonical format: no device, socket or FIFO node. + let directory = handle(kernel, arguments)?; + let _name = arguments.opaque(NAME_LIMIT)?; + let before = attributes(kernel, &directory); + let status = directory.err().unwrap_or(Status::NotSupp); + created(kernel, out, Err(status), &directory, before.as_ref()); + } + 12 | 13 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = directory.and_then(|parent| { + if procedure == 12 { + kernel.remove(parent, &name, tickets) + } else { + kernel.rmdir(parent, &name, tickets) + } + }); + let after = attributes(kernel, &directory); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 14 => { + let from = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let to = handle(kernel, arguments)?; + let new_name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before_from = attributes(kernel, &from); + let before_to = attributes(kernel, &to); + let result = match (&from, &to) { + (Ok(from), Ok(to)) => kernel.rename(*from, &name, *to, &new_name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after_from = attributes(kernel, &from); + let after_to = attributes(kernel, &to); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before_from.as_ref(), after_from.as_ref()); + wcc(kernel, out, before_to.as_ref(), after_to.as_ref()); + } + 15 => { + let file = handle(kernel, arguments)?; + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match (&file, &directory) { + (Ok(file), Ok(parent)) => kernel.link(*file, *parent, &name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after = attributes(kernel, &directory); + let (status, object) = match result { + Ok(stat) => (Status::Ok, Some(stat)), + Err(status) => (status, attributes(kernel, &file)), + }; + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, object.as_ref()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 21 => { + // Every WRITE was answered FILE_SYNC, so nothing is uncommitted. + let serial = handle(kernel, arguments)?; + let _offset = arguments.u64()?; + let _count = arguments.u32()?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, None, Some(&stat)); + out.fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + wcc(kernel, out, None, None); + } + } + } + _ => return Err(Garbage), + } + Ok(()) +} diff --git a/engine/crates/layerfs-nfs/src/directory.rs b/engine/crates/layerfs-nfs/src/directory.rs new file mode 100644 index 0000000000..92a984bbb0 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/directory.rs @@ -0,0 +1,138 @@ +//! READDIR and READDIRPLUS over the engine's published directory cookies. +//! +//! The NFS cookie is the engine's kernel offset (1 for `.`, 2 for `..`, then +//! the offsets the engine publishes) on one implicit directory descriptor, +//! named to the client by `cookieverf`. Cookie 0 opens a fresh descriptor and +//! a fresh verifier. A verifier whose descriptor was closed, or an offset a +//! rewind retired, is `NFS3ERR_BAD_COOKIE`; the client then lists again from 0. +//! +//! One reply carries at most one engine window: exactly the names that fit +//! the client's byte budget are published, and the reply is not at EOF. The +//! next call resumes after the last accepted cookie; a call that finds no +//! names left replies EOF. READDIRPLUS entries carry no attributes and no +//! handle (both optional in RFC 1813), so the client looks each name up. +use crate::{ + attributes::Status, + block::wait, + kernel::{failure, Kernel}, + xdr::{opaque_size, Writer}, +}; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_fuse::operations::{DirectoryFailure, DirectoryStep, DirectoryStream}; + +/// One enumeration request. +pub struct Listing { + pub serial: u64, + pub cookie: u64, + pub verifier: u64, + /// The client's byte budget for the whole reply (`count` / `maxcount`). + pub budget: usize, + /// READDIRPLUS: each entry also carries two absent optional fields. + pub plus: bool, +} +/// Bytes of the reply around the entries: status, absent post_op_attr, +/// cookieverf, the list terminator and the EOF flag, plus the RPC reply header. +const FIXED: usize = 4 + 4 + 8 + 4 + 4 + 24; + +struct Entries { + out: Writer, + used: usize, + budget: usize, + plus: bool, + count: usize, +} +impl Entries { + fn size(&self, name: &[u8]) -> usize { + 4 + 8 + opaque_size(name.len()) + 8 + if self.plus { 8 } else { 0 } + } + /// Appends one entry when it fits the budget. + fn push(&mut self, fileid: u64, name: &[u8], cookie: u64) -> bool { + let size = self.size(name); + if self.used + size > self.budget { + return false; + } + self.used += size; + self.count += 1; + self.out.bool(true).u64(fileid).opaque(name).u64(cookie); + if self.plus { + self.out.bool(false).bool(false); + } + true + } +} + +/// The encoded `READDIR3resok`/`READDIRPLUS3resok` body after the status, +/// or the failure status. +pub fn list(kernel: &Kernel, request: Listing) -> Result, Status> { + let handle = if request.cookie == 0 { + kernel.open_directory(request.serial)? + } else { + kernel + .directory(request.serial, request.verifier) + .ok_or(Status::BadCookie)? + }; + let (services, id) = kernel.request()?; + let stream = wait(DirectoryStream::prepare( + services, + kernel.mount(), + id, + request.serial, + handle, + request.cookie, + )) + .map_err(directory_failure)?; + let mut entries = Entries { + out: Writer::with_capacity(request.budget.min(64 * 1024)), + used: FIXED, + budget: request.budget, + plus: request.plus, + count: 0, + }; + let dots: Vec<(&str, u64, u64)> = stream.dots().collect(); + for (name, serial, cookie) in dots { + if name == ".." && serial != request.serial { + kernel.saw_directory(request.serial, serial); + } + if !entries.push(serial, name.as_bytes(), cookie) { + return finish(entries, handle, false); + } + } + let eof = match wait(stream.next()).map_err(directory_failure)? { + DirectoryStep::End(_) => true, + DirectoryStep::Batch(batch) => { + let mut accepted = 0; + for (entry, cookie) in batch.entries() { + if !entries.push(entry.serial, &entry.name, cookie) { + break; + } + if entry.kind == InodeKind::Directory { + kernel.saw_directory(entry.serial, request.serial); + } + accepted += 1; + } + // Publishing ends this reply's window even when names remain. + drop(wait(batch.accept(accepted)).map_err(directory_failure)?); + false + } + }; + finish(entries, handle, eof) +} +fn finish(entries: Entries, verifier: u64, eof: bool) -> Result, Status> { + if entries.count == 0 && !eof { + return Err(Status::TooSmall); + } + let mut out = Writer::with_capacity(entries.out.len() + 24); + out.bool(false); // dir_attributes + out.u64(verifier); + out.fixed(entries.out.as_bytes()); + out.bool(false); // no further entry + out.bool(eof); + Ok(out.into_bytes()) +} +fn directory_failure(failure: DirectoryFailure) -> Status { + if failure.rewound() { + Status::BadCookie + } else { + self::failure(failure.reason) + } +} diff --git a/engine/crates/layerfs-nfs/src/handle.rs b/engine/crates/layerfs-nfs/src/handle.rs new file mode 100644 index 0000000000..1599a69205 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/handle.rs @@ -0,0 +1,69 @@ +//! Stable NFS file handles for one kernel mount. +//! +//! A handle is `[version][mount owner][serial][tag]`: 25 bytes, well inside +//! the 64-byte NFSv3 limit. The same serial always encodes to the same bytes +//! for the life of one mount, so a client may compare handles. The mount owner +//! is fresh for every engine mount; a handle from an earlier mount decodes as +//! stale. The tag is a keyed hash under a key drawn once per export, so a +//! local process cannot forge a handle for a serial it was never given. It is +//! not a cryptographic MAC (std SipHash, no third-party dependency); it raises +//! the cost of guessing, it does not authenticate the caller. +use std::{collections::hash_map::RandomState, hash::BuildHasher}; + +const VERSION: u8 = 1; +/// Encoded length of every handle this export issues. +pub const HANDLE_BYTES: usize = 25; + +/// Why a client's handle was not accepted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HandleError { + /// Not a handle this export ever issued (length, version or tag). + Bad, + /// Issued for an earlier mount of this export. + Stale, +} + +/// Encodes and checks the handles of one kernel mount. +pub struct Handles { + owner: u64, + key: RandomState, +} +impl Handles { + pub fn new(owner: u64) -> Self { + Self { + owner, + key: RandomState::new(), + } + } + fn tag(&self, owner: u64, serial: u64) -> u64 { + self.key.hash_one((VERSION, owner, serial)) + } + pub fn encode(&self, serial: u64) -> [u8; HANDLE_BYTES] { + let mut handle = [0; HANDLE_BYTES]; + handle[0] = VERSION; + handle[1..9].copy_from_slice(&self.owner.to_be_bytes()); + handle[9..17].copy_from_slice(&serial.to_be_bytes()); + handle[17..25].copy_from_slice(&self.tag(self.owner, serial).to_be_bytes()); + handle + } + /// The serial a handle names. A handle of another mount is stale even + /// when its tag cannot be checked, since its key is gone. + pub fn decode(&self, handle: &[u8]) -> Result { + if handle.len() != HANDLE_BYTES || handle[0] != VERSION { + return Err(HandleError::Bad); + } + let word = |range: std::ops::Range| { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&handle[range]); + u64::from_be_bytes(bytes) + }; + let (owner, serial, tag) = (word(1..9), word(9..17), word(17..25)); + if owner != self.owner { + return Err(HandleError::Stale); + } + if serial == 0 || tag != self.tag(owner, serial) { + return Err(HandleError::Bad); + } + Ok(serial) + } +} diff --git a/engine/crates/layerfs-nfs/src/kernel.rs b/engine/crates/layerfs-nfs/src/kernel.rs new file mode 100644 index 0000000000..77f0ae2aba --- /dev/null +++ b/engine/crates/layerfs-nfs/src/kernel.rs @@ -0,0 +1,847 @@ +//! The synthetic kernel: the lookup references, open descriptors and parent +//! index a FUSE kernel would hold, kept by the export for a stateless client. +//! +//! The engine fences every request on a kernel reference: the root's implicit +//! one, a lookup reference, or an open descriptor. An NFS client presents a +//! handle at any time and never sends FORGET, OPEN or RELEASE, so: +//! +//! - every serial the export hands out in a handle holds exactly one lookup +//! reference, taken by the LOOKUP visit or the creating mutation that +//! decided it; a later visit that adds one more for the same serial is +//! answered by forgetting the extra one at once; +//! - READ and WRITE use one implicit descriptor per file (read-only until the +//! first WRITE, which reopens it writable), and READDIR one implicit +//! directory descriptor per enumeration (`cookieverf`); both are bounded +//! and the least recently used is closed when the bound is reached; +//! - `..` is answered from the parent recorded when a directory was looked +//! up, created, moved or listed, since the engine resolves names only +//! downward; +//! - a serial whose last name a REMOVE, RMDIR or replacing RENAME of this +//! export took away gives its reference and descriptor back at once, so the +//! engine can reclaim it; its handle is stale afterwards, as on any server; +//! - a client's silly rename (`.nfs…`) is not a namespace change: the source +//! name is unlinked and the inode is kept alive by the export alone, under +//! a name only this export knows (see [`crate::names`]). +//! +//! - the references are bounded (reference collection, "plan A" of the A2 +//! design, with no engine interface added): every handed-out serial keeps +//! the name it was last reached by. When more than the configured number +//! of serials hold a reference, the least recently used quarter that holds +//! no descriptor and no silly name is forgotten, its name memory kept. A +//! later handle of a forgotten serial is resolved again by the ordinary +//! LOOKUP of that remembered name (its parent first, recursively): the +//! same serial continues; another serial or none is a stale handle, as for +//! a file renamed or removed elsewhere. +//! +//! Everything still held is given back by [`Kernel::release`], which runs +//! before the engine mount is revoked. +use crate::{ + attributes::{refusal, Identity, Status}, + block::wait, + handle::{HandleError, Handles}, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::{ + operations::{KernelData, KernelRead, MutationFailure, ReadDataInput, ReadFailure}, + ports::{Fence, MountServices, RequestServices, ServiceError}, +}; +use layerfs_overlay::{KernelMount, OverlayError, READ_WINDOW}; +use layerfs_workspace::{KernelReadOperation, ViewStat}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, MutexGuard, + }, +}; + +/// Implicit file descriptors kept open at once. +pub const FILES_HELD: usize = 256; +/// Implicit directory descriptors (enumerations) kept open at once. +pub const DIRECTORIES_HELD: usize = 256; +/// EXCLUSIVE CREATE verifiers remembered for retransmissions. +pub const VERIFIERS_HELD: usize = 1024; +/// Lookup references an export keeps by default before collecting. +pub const LOOKUPS_HELD: usize = 65_536; +/// Remembered names beyond which a handle of a forgotten serial is stale. +const NAMES_HELD: usize = 4 * 1024 * 1024; +/// Parents resolved again for one handle at most (directory depth). +const DEPTH: usize = 4096; +/// Serials used within this many most recent uses are never collected: a +/// call another connection is serving may be about to use them. +pub const RECENT_KEPT: u64 = 64; + +/// A name in one directory: (directory serial, name bytes). +pub(crate) type Named = (u64, Vec); +/// An EXCLUSIVE CREATE's verifier, the serial it created and its last use. +pub(crate) type Verified = ([u8; 8], u64, u64); + +/// One implicit file descriptor. +#[derive(Clone, Copy)] +pub(crate) struct Held { + pub handle: u64, + /// The size last published or observed through this export. + pub size: u64, + pub tick: u64, + pub writable: bool, +} + +#[derive(Default)] +pub(crate) struct Table { + /// Serials holding the export's one lookup reference (the root's is implicit). + pub lookups: HashSet, + /// Directory serial -> the parent it was last seen under. + pub parents: HashMap, + /// File serial -> its implicit descriptor. + pub files: HashMap, + /// Directory descriptor -> (directory serial, last use). + pub directories: HashMap, + /// Silly-renamed inodes: (directory, client name) -> serial. Their + /// descriptors are never evicted; their source name is gone. + pub silly: HashMap, + /// EXCLUSIVE CREATE: (directory, name) -> (verifier, serial, last use). + pub verifiers: HashMap, + /// Serial -> the name it was last reached by, kept after its reference + /// is collected so a later handle can be resolved again. + pub names: HashMap, + /// Serial -> last use, for the collection order. + pub used: HashMap, + /// Serials marked for a cache refresh, and when (see [`crate::refresh`]). + pub refreshing: HashMap, + pub tick: u64, +} +impl Table { + pub fn tick(&mut self) -> u64 { + self.tick += 1; + self.tick + } + /// Serials kept alive by a silly name. + pub fn pinned(&self, serial: u64) -> bool { + self.silly.values().any(|held| *held == serial) + } +} +/// One opened file as READ and WRITE need it. +#[derive(Clone, Copy)] +pub(crate) struct OpenFile { + pub handle: u64, + pub size: u64, +} + +/// One kernel mount served over NFS: its ports and its synthetic kernel state. +pub struct Kernel { + services: Arc, + fence: Fence, + mount: KernelMount, + identity: Identity, + handles: Handles, + table: Mutex, + requests: AtomicU64, + /// Mutations whose failure keeps engine custody (possibly a publication + /// ticket): kept, never released or undone on a guess, as the FUSE + /// connection retains them. + retained: Mutex>, + /// Keep macOS AppleDouble (`._*`) names out of the Workspace: a regular + /// file lives in the export alone (see [`crate::apple_double`]), any + /// other kind is refused. + refuse_apple_double: bool, + /// The AppleDouble companions held in memory. + pub(crate) companions: Mutex, + /// Lookup references kept before the least recently used are collected. + lookups_held: usize, + /// References collected, handles resolved again, and resolutions that + /// found another serial or none (stale), over the export's life. + collected: AtomicU64, + resolved: AtomicU64, + unresolved: AtomicU64, + /// Silly renames served as unlinks, and creations refused by name. + pub(crate) silly_renamed: AtomicU64, + pub(crate) names_refused: AtomicU64, + /// AppleDouble companions created in memory. + pub(crate) names_detached: AtomicU64, +} +impl Kernel { + pub fn new(services: Arc, mount: KernelMount, uid: u32, gid: u32) -> Self { + let root = mount.root_serial(); + Self { + services, + fence: Fence::default(), + mount, + identity: Identity { + root, + uid, + gid, + fsid: mount.owner_id(), + }, + handles: Handles::new(mount.owner_id()), + table: Mutex::default(), + requests: AtomicU64::new(1), + retained: Mutex::default(), + refuse_apple_double: false, + companions: Mutex::default(), + lookups_held: LOOKUPS_HELD, + collected: AtomicU64::new(0), + resolved: AtomicU64::new(0), + unresolved: AtomicU64::new(0), + silly_renamed: AtomicU64::new(0), + names_refused: AtomicU64::new(0), + names_detached: AtomicU64::new(0), + } + } + /// Whether creating `._*` names is refused (see [`crate::names`]). + pub fn refusing_apple_double(mut self, refuse: bool) -> Self { + self.refuse_apple_double = refuse; + self + } + pub fn refuses_apple_double(&self) -> bool { + self.refuse_apple_double + } + /// How many lookup references are kept before collection (at least 4). + pub fn holding(mut self, lookups: usize) -> Self { + self.lookups_held = lookups.max(4); + self + } + pub fn identity(&self) -> Identity { + self.identity + } + pub fn root(&self) -> u64 { + self.identity.root + } + pub fn mount(&self) -> KernelMount { + self.mount + } + pub fn encode(&self, serial: u64) -> Vec { + self.handles.encode(serial).to_vec() + } + pub(crate) fn table(&self) -> MutexGuard<'_, Table> { + self.table + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// Keeps a failed mutation's remaining custody. + pub(crate) fn retain(&self, failure: MutationFailure) { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(failure); + } + /// The serial a client handle names, holding its reference: a collected + /// one is resolved again by its remembered name. + pub fn decode(&self, handle: &[u8]) -> Result { + let serial = self.handles.decode(handle).map_err(|error| match error { + HandleError::Bad => Status::BadHandle, + HandleError::Stale => Status::Stale, + })?; + self.held_reference(serial)?; + Ok(serial) + } + /// Ensures the export holds `serial`'s reference, resolving a collected + /// serial again; marks it used. + pub(crate) fn held_reference(&self, serial: u64) -> Result<(), Status> { + if serial == self.root() { + return Ok(()); + } + if crate::apple_double::companion(serial) { + // The export's own companion holds no engine reference. + return self.companion_stat(serial).map(|_| ()); + } + { + let mut table = self.table(); + if table.lookups.contains(&serial) { + let tick = table.tick(); + table.used.insert(serial, tick); + return Ok(()); + } + } + self.resolve(serial, 0) + } + /// One collected serial resolved by the LOOKUP of its remembered name, + /// after its parent. Another serial, or none, there is a stale handle. + fn resolve(&self, serial: u64, depth: usize) -> Result<(), Status> { + let named = self.table().names.get(&serial).cloned(); + let Some((parent, name)) = named else { + return Err(Status::Stale); + }; + if depth >= DEPTH { + return Err(Status::Stale); + } + let parent_held = parent == self.root() || self.table().lookups.contains(&parent); + if !parent_held { + self.resolve(parent, depth + 1)?; + } + let found = PathName::from_bytes(&name) + .map_err(|_| Status::Stale) + .and_then(|path| { + self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + ) + }); + match found { + Ok((stat, _, _)) => { + self.entry_taken(&stat, parent, &name); + if stat.serial == serial { + self.resolved.fetch_add(1, Ordering::Relaxed); + return Ok(()); + } + } + Err(Status::NoEnt | Status::Stale | Status::NotDir) => {} + Err(status) => return Err(status), + } + let mut table = self.table(); + if table + .names + .get(&serial) + .is_some_and(|(p, n)| *p == parent && *n == name) + { + table.names.remove(&serial); + } + self.unresolved.fetch_add(1, Ordering::Relaxed); + Err(Status::Stale) + } + pub fn request(&self) -> Result<(Arc, u64), Status> { + let services = self.services.request(&self.fence).map_err(failure)?; + Ok((services, self.requests.fetch_add(1, Ordering::Relaxed))) + } + fn observe( + &self, + protected: u64, + handle: Option, + operation: KernelReadOperation, + ) -> Result<(ViewStat, Option, Option), Status> { + let (services, request) = self.request()?; + let found = wait(KernelRead::prepare( + services, self.mount, request, protected, handle, operation, + )) + .map_err(read_failure)?; + let answer = found.value().map(|value| { + ( + value.stat.clone(), + value.file.map(|file| file.owner_id()), + value.directory.map(|directory| directory.owner_id()), + ) + }); + wait(found.dispose()).map_err(read_failure)?; + answer.map_err(refusal) + } + /// GETATTR. An inode that lost its last name while a silly name keeps it + /// is observed through its descriptor; any other inode no name reaches + /// any more is stale. + pub fn getattr(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_stat(serial); + } + let handle = { + let table = self.table(); + if table.pinned(serial) { + table.files.get(&serial).map(|held| held.handle) + } else { + None + } + }; + let stat = self + .observe(serial, handle, KernelReadOperation::Getattr { serial }) + .map(|(stat, _, _)| stat) + .map_err(|status| { + if status == Status::NoEnt { + Status::Stale + } else { + status + } + })?; + self.observed_size(&stat); + Ok(stat) + } + /// LOOKUP of `name` in directory `parent`, including `.`, `..` and the + /// export's own silly names. + pub fn lookup(&self, parent: u64, name: &[u8]) -> Result { + match name { + b"" => return Err(Status::NoEnt), + b"." => return self.getattr(parent), + b".." => { + let up = if parent == self.root() { + parent + } else { + let known = self.table().parents.get(&parent).copied(); + known.ok_or(Status::NoEnt)? + }; + self.held_reference(up)?; + return self.getattr(up); + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ => {} + } + if let Some(stat) = self.companion_named(parent, name) { + return Ok(stat); + } + let silly = self.table().silly.get(&(parent, name.to_vec())).copied(); + if let Some(serial) = silly { + return self.getattr(serial); + } + let path = PathName::from_bytes(name).map_err(|_| Status::Inval)?; + let (stat, _, _) = self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + )?; + // The deciding visit took one engine reference; keep exactly one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// A visit or mutation that decided `stat` as the entry `name` of + /// `parent` took one engine lookup reference on it: the export keeps + /// exactly one per serial, remembers the name and records a directory's + /// parent. Beyond the bound, the least recently used are collected. + pub(crate) fn entry_taken(&self, stat: &ViewStat, parent: u64, name: &[u8]) { + let (extra, collect) = { + let mut table = self.table(); + if stat.kind == InodeKind::Directory { + table.parents.insert(stat.serial, parent); + } + let root = stat.serial == self.root(); + if !root { + let tick = table.tick(); + table.used.insert(stat.serial, tick); + if table.names.len() < NAMES_HELD || table.names.contains_key(&stat.serial) { + table.names.insert(stat.serial, (parent, name.to_vec())); + } + } + let extra = root || !table.lookups.insert(stat.serial); + let collect = if table.lookups.len() > self.lookups_held { + collectable(&mut table, self.lookups_held - self.lookups_held / 4) + } else { + Vec::new() + }; + (extra, collect) + }; + if extra { + self.forget(stat.serial, 1); + } + self.collected + .fetch_add(collect.len() as u64, Ordering::Relaxed); + for serial in collect { + self.forget(serial, 1); + } + self.observed_size(stat); + } + /// Records that `serial` is now reached as `name` in `parent` (RENAME). + pub(crate) fn renamed(&self, serial: u64, parent: u64, name: &[u8]) { + let mut table = self.table(); + if table.names.contains_key(&serial) { + table.names.insert(serial, (parent, name.to_vec())); + } + } + pub(crate) fn forget(&self, serial: u64, count: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.forget(self.mount, serial, count))); + } + } + /// Gives back everything the export holds for `serial`, whose last name + /// is gone: its descriptor, its reference and its parent record. Later + /// handles of it are stale. + pub(crate) fn unlinked(&self, serial: u64) { + if serial == self.root() { + return; + } + let (file, directories, held) = { + let mut table = self.table(); + table.parents.remove(&serial); + table.names.remove(&serial); + table.used.remove(&serial); + table.verifiers.retain(|_, entry| entry.1 != serial); + let directories: Vec = table + .directories + .iter() + .filter(|(_, entry)| entry.0 == serial) + .map(|(handle, _)| *handle) + .collect(); + for handle in &directories { + table.directories.remove(handle); + } + ( + table.files.remove(&serial), + directories, + table.lookups.remove(&serial), + ) + }; + if let Some(file) = file { + self.close_file(serial, file.handle); + } + for handle in directories { + self.close_directory(serial, handle); + } + if held { + self.forget(serial, 1); + } + } + /// Records the parent of a directory seen in a listing or moved by RENAME. + pub fn saw_directory(&self, serial: u64, parent: u64) { + self.table().parents.insert(serial, parent); + } + /// The size a published or observed stat reports, for READ's EOF. + pub(crate) fn observed_size(&self, stat: &ViewStat) { + if let Some(held) = self.table().files.get_mut(&stat.serial) { + held.size = stat.logical_len; + } + } + /// READLINK: the raw target bytes. + pub fn readlink(&self, serial: u64) -> Result, Status> { + if crate::apple_double::companion(serial) { + return Err(Status::Inval); + } + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + None, + ReadDataInput::Link, + )) + .map_err(read_failure)? + .map_err(refusal)?; + Ok(data.bytes().to_vec()) + } + /// The implicit descriptor of `serial`, opened (or reopened writable) + /// when needed. + pub(crate) fn open_file(&self, serial: u64, writable: bool) -> Result { + { + let mut table = self.table(); + let tick = table.tick(); + if let Some(held) = table.files.get_mut(&serial) { + if held.writable || !writable { + held.tick = tick; + return Ok(OpenFile { + handle: held.handle, + size: held.size, + }); + } + } + } + let operation = KernelReadOperation::Open { serial, writable }; + let (stat, file, _) = self.observe(serial, None, operation)?; + let handle = file.ok_or(Status::ServerFault)?; + let (evicted, closed, chosen) = { + let mut table = self.table(); + let tick = table.tick(); + let current = table.files.get(&serial).copied(); + match current { + // Another connection opened a good enough one meanwhile. + Some(theirs) if theirs.writable || !writable => { + (None, Some((serial, handle)), theirs) + } + _ => { + let evicted = if current.is_none() && table.files.len() >= FILES_HELD { + let pinned: HashSet = table.silly.values().copied().collect(); + oldest( + table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.tick)), + ) + .and_then(|serial| { + table + .files + .remove(&serial) + .map(|held| (serial, held.handle)) + }) + } else { + None + }; + let held = Held { + handle, + size: stat.logical_len, + tick, + writable, + }; + table.files.insert(serial, held); + (evicted, current.map(|old| (serial, old.handle)), held) + } + } + }; + if let Some((old, descriptor)) = evicted { + self.close_file(old, descriptor); + } + if let Some((old, descriptor)) = closed { + self.close_file(old, descriptor); + } + Ok(OpenFile { + handle: chosen.handle, + size: chosen.size, + }) + } + pub(crate) fn close_file(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_file(self.mount, serial, handle))); + } + } + /// READ of at most one engine window. `eof` is set when the reply reaches + /// the size last seen for the file, or comes back short. + pub fn read(&self, serial: u64, offset: u64, count: u32) -> Result<(Vec, bool), Status> { + let length = count.min(READ_WINDOW as u32); + if crate::apple_double::companion(serial) { + return self.companion_read(serial, offset, length); + } + let open = self.open_file(serial, false)?; + if offset >= open.size { + return Ok((Vec::new(), true)); + } + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + Some(open.handle), + ReadDataInput::File { offset, length }, + )) + .map_err(read_failure)? + .map_err(refusal)?; + let bytes = data.bytes().to_vec(); + let end = offset.saturating_add(bytes.len() as u64); + let eof = bytes.len() < length as usize || end >= open.size; + Ok((bytes, eof)) + } + /// A fresh implicit directory descriptor for an enumeration from cookie 0. + pub fn open_directory(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::NotDir); + } + let (_, _, directory) = + self.observe(serial, None, KernelReadOperation::Opendir { serial })?; + let handle = directory.ok_or(Status::ServerFault)?; + let evicted = { + let mut table = self.table(); + let tick = table.tick(); + let evicted = if table.directories.len() >= DIRECTORIES_HELD { + oldest( + table + .directories + .iter() + .map(|(handle, entry)| (*handle, entry.1)), + ) + .and_then(|handle| { + table + .directories + .remove(&handle) + .map(|entry| (entry.0, handle)) + }) + } else { + None + }; + table.directories.insert(handle, (serial, tick)); + evicted + }; + if let Some((old, descriptor)) = evicted { + self.close_directory(old, descriptor); + } + Ok(handle) + } + /// The descriptor of a continuing enumeration, if it is still open. + pub fn directory(&self, serial: u64, handle: u64) -> Option { + let mut table = self.table(); + let tick = table.tick(); + let entry = table.directories.get_mut(&handle)?; + if entry.0 != serial { + return None; + } + entry.1 = tick; + Some(handle) + } + fn close_directory(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_directory(self.mount, serial, handle))); + } + } + /// Gives back everything the synthetic kernel holds: descriptors first, + /// then lookup references. Run once, before the engine mount is revoked. + /// Retained mutation failures keep their custody and are only counted. + pub fn release(&self) -> Released { + let table = std::mem::take(&mut *self.table()); + let companions = self.companions_release(); + let released = Released { + companions, + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + }; + for (serial, held) in table.files { + self.close_file(serial, held.handle); + } + for (handle, (serial, _)) in table.directories { + self.close_directory(serial, handle); + } + for serial in table.lookups { + self.forget(serial, 1); + } + released + } + fn retained_count(&self) -> usize { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .len() + } + /// Current holdings, for diagnostics. + pub fn held(&self) -> Released { + let table = self.table(); + Released { + companions: self.companions_held(), + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + } + } + /// Closes every implicit file descriptor that no silly name needs, so + /// nothing the export opened is held across a Commit. Returns how many. + pub fn close_files(&self) -> usize { + let closing: Vec<(u64, u64)> = { + let mut table = self.table(); + let pinned: HashSet = table.silly.values().copied().collect(); + let closing: Vec<(u64, u64)> = table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.handle)) + .collect(); + for (serial, _) in &closing { + table.files.remove(serial); + } + closing + }; + for (serial, handle) in &closing { + self.close_file(*serial, *handle); + } + closing.len() + } + /// Closes every enumeration descriptor; a continuing READDIR then + /// answers `NFS3ERR_BAD_COOKIE` and the client lists again from the + /// start. Used after a Commit installed another root. Returns how many. + pub fn close_directories(&self) -> usize { + let closing: Vec<(u64, u64)> = std::mem::take(&mut self.table().directories) + .into_iter() + .map(|(handle, (serial, _))| (serial, handle)) + .collect(); + for (serial, handle) in &closing { + self.close_directory(*serial, *handle); + } + closing.len() + } +} +/// Takes the least recently used references out of the table until `keep` +/// remain, skipping recently used serials, serials that hold a descriptor +/// or a silly name, and +/// directories a held serial was reached through: the engine's own parent +/// index of a held inode never loses its link, so collection proceeds from +/// the leaves. Names and parent records stay for a later resolution. +fn collectable(table: &mut Table, keep: usize) -> Vec { + let held_parents = table.lookups.iter().filter_map(|serial| { + table + .names + .get(serial) + .map(|named| named.0) + .or_else(|| table.parents.get(serial).copied()) + }); + let busy: HashSet = table + .silly + .values() + .copied() + .chain(table.files.keys().copied()) + .chain(table.directories.values().map(|entry| entry.0)) + .chain(held_parents) + .collect(); + let recent = table.tick.saturating_sub(RECENT_KEPT); + let mut candidates: Vec<(u64, u64)> = table + .lookups + .iter() + .filter(|serial| !busy.contains(serial)) + .map(|serial| (table.used.get(serial).copied().unwrap_or(0), *serial)) + .filter(|(used, _)| *used < recent) + .collect(); + let excess = table.lookups.len().saturating_sub(keep); + candidates.sort_unstable(); + candidates.truncate(excess); + for (_, serial) in &candidates { + table.lookups.remove(serial); + table.used.remove(serial); + } + candidates.into_iter().map(|(_, serial)| serial).collect() +} +/// What [`Kernel::release`] gave back. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Released { + pub files: usize, + pub directories: usize, + pub lookups: usize, + /// Silly names whose inode was kept alive until release. + pub silly: usize, + /// Failed mutations still holding engine custody. + pub retained: usize, + /// Client silly renames served as unlinks, over the export's life. + pub silly_renamed: u64, + /// Creations refused for their AppleDouble name, over the export's life. + pub names_refused: u64, + /// AppleDouble companions held in memory (see [`crate::apple_double`]). + pub companions: usize, + /// AppleDouble companions created, over the export's life. + pub names_detached: u64, + /// References collected beyond the bound, over the export's life. + pub collected: u64, + /// Handles of collected serials resolved again by their name. + pub resolved: u64, + /// Handles of collected serials whose name no longer reaches them. + pub unresolved: u64, +} +fn oldest(entries: impl Iterator) -> Option { + entries.min_by_key(|(_, tick)| *tick).map(|(key, _)| key) +} +fn stale(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + while let Some(error) = current { + if matches!( + error.downcast_ref::(), + Some(OverlayError::Stale) + ) { + return true; + } + current = error.source(); + } + false +} +/// Whether a failed step's cause chain holds an engine `Stale`. +pub(crate) fn failure_is_stale(error: &ServiceError) -> bool { + stale(error.as_ref()) +} +/// A failed port step: an engine `Stale` is the client's stale handle; any +/// other failure is an I/O error of this request alone. +pub fn failure(error: ServiceError) -> Status { + if stale(error.as_ref()) { + Status::Stale + } else { + Status::Io + } +} +fn read_failure(failure: ReadFailure) -> Status { + if stale(failure.reason.as_ref()) { + Status::Stale + } else { + Status::Io + } +} diff --git a/engine/crates/layerfs-nfs/src/lib.rs b/engine/crates/layerfs-nfs/src/lib.rs new file mode 100644 index 0000000000..32b1b450f0 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/lib.rs @@ -0,0 +1,36 @@ +//! A mounted Workspace served as a loopback NFSv3 export. +//! +//! The platform-independent half of the native macOS route: the same +//! [`MountServices`](layerfs_fuse::ports::MountServices) ports and kernel +//! operations the FUSE connection uses, answered over ONC RPC instead of +//! `/dev/fuse`. A stateless NFS client holds no lookup references or open +//! descriptors, so the export keeps them itself (see [`kernel`]). +//! +//! Every NFSv3 procedure is served (MKNOD answers `NFS3ERR_NOTSUPP`: the +//! canonical format has no special files) and the MOUNT protocol; an export +//! configured read-only answers every changing procedure `NFS3ERR_ROFS`. +//! A changing procedure's publication tickets are released only after its +//! reply was sent (see [`change`]); names a client invents for itself are +//! kept out of the Workspace (see [`names`] and [`apple_double`]). +//! There is no third-party dependency: XDR, RPC record marking and both +//! programs are implemented here over `std::net`. +#![forbid(unsafe_code)] + +pub mod apple_double; +pub mod attributes; +mod block; +pub mod change; +mod changing; +mod directory; +pub mod handle; +pub mod kernel; +mod mount; +pub mod names; +mod nfs; +pub mod refresh; +pub mod rpc; +mod server; +pub mod xdr; + +pub use kernel::Released; +pub use server::{Count, ExportConfig, Server}; diff --git a/engine/crates/layerfs-nfs/src/mount.rs b/engine/crates/layerfs-nfs/src/mount.rs new file mode 100644 index 0000000000..4b6656de04 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/mount.rs @@ -0,0 +1,66 @@ +//! MOUNT program 100005 version 3 (RFC 1813 appendix I): one export. +use crate::{ + kernel::Kernel, + rpc::Accept, + xdr::{Reader, Writer}, +}; + +pub const PROGRAM: u32 = 100_005; +pub const VERSION: u32 = 3; +const MNTPATHLEN: usize = 1024; +const MNT3_OK: u32 = 0; +const MNT3ERR_NOENT: u32 = 2; +const AUTH_UNIX: u32 = 1; + +/// Decodes and answers one MOUNT call for the export named `path`. +pub fn call( + kernel: &Kernel, + path: &str, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, +) -> Result<(), Accept> { + match procedure { + 0 | 4 => Ok(()), + 1 => { + let asked = arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + if exported(asked, path) { + out.u32(MNT3_OK); + out.opaque(&kernel.encode(kernel.root())); + out.u32(1).u32(AUTH_UNIX); + } else { + out.u32(MNT3ERR_NOENT); + } + Ok(()) + } + 2 => { + out.bool(false); + Ok(()) + } + 3 => { + arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + Ok(()) + } + 5 => { + out.bool(true).opaque(path.as_bytes()).bool(false); + out.bool(false); + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The export path, compared without trailing slashes ("/" names the root). +fn exported(asked: &[u8], path: &str) -> bool { + let trim = |bytes: &[u8]| -> Vec { + let mut bytes = bytes.to_vec(); + while bytes.len() > 1 && bytes.last() == Some(&b'/') { + bytes.pop(); + } + bytes + }; + trim(asked) == trim(path.as_bytes()) +} diff --git a/engine/crates/layerfs-nfs/src/names.rs b/engine/crates/layerfs-nfs/src/names.rs new file mode 100644 index 0000000000..9237471c15 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/names.rs @@ -0,0 +1,41 @@ +//! Names an NFS client invents for its own bookkeeping, kept out of the +//! Workspace by the export itself so they never reach a Commit. +//! +//! - **Silly renames.** A client that unlinks a file it still has open +//! renames it instead, and removes that name after the last close. Linux +//! names it `.nfs` + 16 hex digits (fileid) + 8 hex digits (counter); +//! macOS `.nfs.` + 8 hex + `.` + 4 hex. Such a RENAME is served as an +//! UNLINK of the source name while the export keeps the inode alive by its +//! own reference and an open descriptor; the silly name exists only in the +//! export's table (LOOKUP, GETATTR, READ, WRITE, SETATTR and REMOVE of it +//! work; READDIR does not list it). The final REMOVE gives the inode back. +//! - **AppleDouble.** On NFSv3 the macOS client stores extended attributes +//! and resource forks in `._` companion files. When the export is +//! configured to keep them out, such a regular file is created in the +//! export's memory alone (see [`crate::apple_double`]): the client's +//! attribute write (`copyfile`, `cp`, `std::fs::copy`, `ditto`, `tar`) +//! succeeds and reads back for the export's life, and nothing reaches the +//! Workspace. Any other kind under such a name (directory, symlink, link, +//! rename target) is refused with `NFS3ERR_ACCES`. +//! +//! The patterns are exact so that a user's own `.nfsrc` or `._config` is an +//! ordinary name (the AppleDouble refusal is the one deliberate exception +//! and is configurable). + +/// A client-generated silly-rename name. +pub fn silly(name: &[u8]) -> bool { + let hex = |bytes: &[u8]| bytes.iter().all(u8::is_ascii_hexdigit); + if let Some(rest) = name.strip_prefix(b".nfs.") { + // macOS: .nfs.XXXXXXXX.XXXX + return rest.len() == 13 && rest[8] == b'.' && hex(&rest[..8]) && hex(&rest[9..]); + } + if let Some(rest) = name.strip_prefix(b".nfs") { + // Linux: .nfs + 16 + 8 hex digits + return rest.len() == 24 && hex(rest); + } + false +} +/// A macOS AppleDouble companion name (`._` followed by at least one byte). +pub fn apple_double(name: &[u8]) -> bool { + name.len() > 2 && name.starts_with(b"._") +} diff --git a/engine/crates/layerfs-nfs/src/nfs.rs b/engine/crates/layerfs-nfs/src/nfs.rs new file mode 100644 index 0000000000..b257125300 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/nfs.rs @@ -0,0 +1,269 @@ +//! NFS program 100003 version 3: dispatch of every procedure. +//! +//! The reading procedures are answered here; the changing ones in +//! [`crate::changing`]. A read-only export answers every changing procedure +//! `NFS3ERR_ROFS` with its empty weak cache consistency data. +use crate::{ + attributes::{access, post_op_maybe, Status}, + change::Tickets, + changing, + directory::{list, Listing}, + kernel::Kernel, + rpc::{Accept, Credentials}, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_overlay::{READ_WINDOW, WRITE_WINDOW}; + +pub const PROGRAM: u32 = 100_003; +pub const VERSION: u32 = 3; +/// Procedure names by number, for diagnostics. +pub const PROCEDURES: [&str; 22] = [ + "NULL", + "GETATTR", + "SETATTR", + "LOOKUP", + "ACCESS", + "READLINK", + "READ", + "WRITE", + "CREATE", + "MKDIR", + "SYMLINK", + "MKNOD", + "REMOVE", + "RMDIR", + "RENAME", + "LINK", + "READDIR", + "READDIRPLUS", + "FSSTAT", + "FSINFO", + "PATHCONF", + "COMMIT", +]; +/// Longest handle (`NFS3_FHSIZE`) and the longest name a client may send. +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Declared capacity: fixed values, never an admission signal (as for FUSE statfs). +const DECLARED_BLOCKS: u64 = 1 << 30; +const DECLARED_FILES: u64 = 1 << 30; +const BLOCK_BYTES: u64 = 4096; +/// FSINFO properties: FSF3_LINK | FSF3_SYMLINK | FSF3_HOMOGENEOUS | FSF3_CANSETTIME. +const PROPERTIES: u32 = 0x01 | 0x02 | 0x08 | 0x10; +/// Preferred READDIR reply size the client is told to ask for. +const DIRECTORY_PREFERENCE: u32 = 32 * 1024; + +/// The export's fixed answers. +#[derive(Clone, Copy, Debug)] +pub struct Policy { + pub read_only: bool, +} + +/// Decodes the arguments of `procedure`, runs it and appends its results. +/// `Err` is an RPC-level refusal of the call. +#[allow(clippy::too_many_arguments)] +pub fn call( + kernel: &Kernel, + policy: Policy, + procedure: u32, + credentials: &Credentials, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Accept> { + let garbage = |_: Garbage| Accept::GarbageArguments; + match procedure { + 0 => Ok(()), + 1 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().fattr3(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + Ok(()) + } + 3 => { + let parent = handle(kernel, arguments).map_err(garbage)?; + let name = arguments.opaque(NAME_LIMIT).map_err(garbage)?; + // Both answers carry the directory's attributes (RFC 1813 + // LOOKUP3resok and LOOKUP3resfail), so the client keeps its + // cached view of the directory without a GETATTR of its own. + let directory = parent + .as_ref() + .ok() + .and_then(|parent| kernel.getattr(*parent).ok()); + match parent.and_then(|parent| kernel.lookup(parent, name)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + post_op_maybe(kernel.identity(), out, directory.as_ref()); + } + Err(status) => { + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, directory.as_ref()); + } + } + Ok(()) + } + 4 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let requested = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + let caller = credentials + .unix + .as_ref() + .map(|unix| (unix.uid, unix.gid, unix.groups.as_slice())); + let granted = access( + kernel.identity(), + &stat, + caller, + requested, + policy.read_only, + ); + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + out.u32(granted); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 5 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.readlink(serial)) { + Ok(target) => { + out.u32(Status::Ok.code()).bool(false).opaque(&target); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 6 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let offset = arguments.u64().map_err(garbage)?; + let count = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.read(serial, offset, count)) { + Ok((data, eof)) => { + out.u32(Status::Ok.code()).bool(false); + out.u32(data.len() as u32).bool(eof).opaque(&data); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 16 | 17 => { + let plus = procedure == 17; + let serial = handle(kernel, arguments).map_err(garbage)?; + let cookie = arguments.u64().map_err(garbage)?; + let verifier = arguments.fixed(8).map_err(garbage)?; + let mut budget = arguments.u32().map_err(garbage)?; + if plus { + // dircount bounds the names alone; maxcount the whole reply. + let _dircount = budget; + budget = arguments.u32().map_err(garbage)?; + } + let mut word = [0; 8]; + word.copy_from_slice(verifier); + let listing = |serial| Listing { + serial, + cookie, + verifier: u64::from_be_bytes(word), + budget: budget as usize, + plus, + }; + match serial.and_then(|serial| list(kernel, listing(serial))) { + Ok(body) => { + out.u32(Status::Ok.code()).fixed(&body); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 18..=20 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + filesystem(procedure, out); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 2 | 7..=15 | 21 if !policy.read_only => { + changing::call(kernel, procedure, arguments, out, tickets, verifier).map_err(garbage) + } + 2 | 7..=15 | 21 => { + // Each changing procedure's failure carries wcc_data (two absent + // optional fields) per directory it names; LINK also its object's + // post_op_attr. + let absent = match procedure { + 14 => 4, + 15 => 3, + _ => 2, + }; + out.u32(Status::RoFs.code()); + for _ in 0..absent { + out.bool(false); + } + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The serial of the call's handle argument; a handle that is not accepted is +/// the procedure's own failure status, not an RPC refusal. +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +/// FSSTAT, FSINFO and PATHCONF results after the object's attributes. +fn filesystem(procedure: u32, out: &mut Writer) { + match procedure { + 18 => { + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_FILES) + .u64(DECLARED_FILES) + .u64(DECLARED_FILES); + out.u32(0); // invarsec + } + 19 => { + let (read, write) = (READ_WINDOW as u32, WRITE_WINDOW as u32); + out.u32(read).u32(read).u32(BLOCK_BYTES as u32); + out.u32(write).u32(write).u32(BLOCK_BYTES as u32); + out.u32(DIRECTORY_PREFERENCE); + out.u64(i64::MAX as u64); + out.u32(0).u32(1); // time_delta: 1 ns + out.u32(PROPERTIES); + } + _ => { + out.u32(u32::MAX); // linkmax + out.u32(255); // name_max + out.bool(true); // no_trunc + out.bool(true); // chown_restricted + out.bool(false); // case_insensitive + out.bool(true); // case_preserving + } + } +} diff --git a/engine/crates/layerfs-nfs/src/refresh.rs b/engine/crates/layerfs-nfs/src/refresh.rs new file mode 100644 index 0000000000..32a7f1e071 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/refresh.rs @@ -0,0 +1,75 @@ +//! Cache refresh after a Commit installed another root (the EdenFS method). +//! +//! An NFS client has no invalidation channel: it keeps attributes and +//! listings until `actimeo` or its close-to-open check. A SETATTR it sends +//! itself is answered with fresh post-operation attributes, and a client +//! that sees them changed drops what it cached for that inode. So the daemon +//! names the inodes an install made stale ([`Kernel::refresh`]) and changes +//! their mode to the value it already has through the mount point. The +//! export answers each such mode-only SETATTR on a marked inode with its +//! current attributes and changes nothing (the mark is taken once and ends +//! after [`MARK_LIFE`]), so the refresh never writes into the Workspace and +//! never reverts a mode the install changed. +use crate::{change::Sattr, kernel::Kernel}; +use std::time::{Duration, Instant}; + +/// How long one refresh mark waits for its SETATTR. +pub const MARK_LIFE: Duration = Duration::from_secs(30); +/// Directory depth one path is built to at most. +const DEPTH: usize = 4096; + +impl Kernel { + /// Marks each of `serials` this export handed out and still knows by + /// name, and returns their paths below the export root (empty for the + /// root itself), in the order given. A serial the client never saw has + /// nothing cached and is left out. + pub fn refresh(&self, serials: impl IntoIterator) -> Vec>> { + let root = self.root(); + let mut table = self.table(); + let now = Instant::now(); + table + .refreshing + .retain(|_, marked| now.duration_since(*marked) < MARK_LIFE); + let mut paths = Vec::new(); + for serial in serials { + let mut path = Vec::new(); + let mut at = serial; + let mut known = true; + while at != root { + let Some((parent, name)) = table.names.get(&at) else { + known = false; + break; + }; + if path.len() >= DEPTH { + known = false; + break; + } + path.push(name.clone()); + at = *parent; + } + if known { + path.reverse(); + table.refreshing.insert(serial, now); + paths.push(path); + } + } + paths + } + /// Whether a SETATTR of `serial` is a pending refresh: mode only, on a + /// marked inode whose mark has not ended. Takes the mark. + pub(crate) fn refreshed(&self, serial: u64, sattr: &Sattr) -> bool { + let mode_only = sattr.mode.is_some() + && sattr.uid.is_none() + && sattr.gid.is_none() + && sattr.size.is_none() + && sattr.mtime.is_none(); + if !mode_only { + return false; + } + let mut table = self.table(); + match table.refreshing.remove(&serial) { + Some(marked) => marked.elapsed() < MARK_LIFE, + None => false, + } + } +} diff --git a/engine/crates/layerfs-nfs/src/rpc.rs b/engine/crates/layerfs-nfs/src/rpc.rs new file mode 100644 index 0000000000..43c2ded345 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/rpc.rs @@ -0,0 +1,181 @@ +//! ONC RPC v2 (RFC 5531) over TCP record marking: calls in, replies out. +use crate::xdr::{Garbage, Reader, Writer}; +use std::io::{self, Read, Write}; + +const CALL: u32 = 0; +const REPLY: u32 = 1; +const MSG_ACCEPTED: u32 = 0; +const MSG_DENIED: u32 = 1; +const RPC_MISMATCH: u32 = 0; +const AUTH_ERROR: u32 = 1; +const AUTH_NONE: u32 = 0; +const AUTH_UNIX: u32 = 1; +/// AUTH_TOOWEAK: the credential flavor is not accepted. +const AUTH_TOOWEAK: u32 = 5; +const LAST_FRAGMENT: u32 = 1 << 31; +/// Largest credential or verifier body RFC 5531 allows. +const AUTH_BODY: usize = 400; +/// Largest whole record accepted. A1 serves no WRITE, so every call is small; +/// the bound still admits a full-window WRITE for the later write path. +pub const RECORD_LIMIT: usize = 2 * 1024 * 1024; + +/// The accepted-reply status of one call. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Accept { + Success, + ProgramUnavailable, + /// The program exists in versions `low..=high` only. + ProgramMismatch { + low: u32, + high: u32, + }, + ProcedureUnavailable, + GarbageArguments, +} +/// The caller's identity as AUTH_UNIX states it. AUTH_NONE has none. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Credentials { + pub unix: Option, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UnixCredentials { + pub uid: u32, + pub gid: u32, + pub groups: Vec, +} +/// One decoded call header; its arguments follow in the same record. +#[derive(Debug)] +pub struct Call { + pub xid: u32, + pub program: u32, + pub version: u32, + pub procedure: u32, + pub credentials: Credentials, +} +/// A record that is not an answerable call. +#[derive(Debug)] +pub enum Refused { + /// Not a call message, or not decodable: no reply is possible. + Undecodable, + /// A call whose header can be answered only with a rejection. + Reply(Vec), +} + +/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`]. +/// `Ok(None)` is an orderly end of stream before a record started. +pub fn read_record(stream: &mut impl Read) -> io::Result>> { + let mut record = Vec::new(); + loop { + let mut header = [0; 4]; + if let Err(error) = stream.read_exact(&mut header) { + if error.kind() == io::ErrorKind::UnexpectedEof && record.is_empty() { + return Ok(None); + } + return Err(error); + } + let header = u32::from_be_bytes(header); + let length = (header & !LAST_FRAGMENT) as usize; + if record.len() + length > RECORD_LIMIT { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "RPC record exceeds its bound", + )); + } + let start = record.len(); + record.resize(start + length, 0); + stream.read_exact(&mut record[start..])?; + if header & LAST_FRAGMENT != 0 { + return Ok(Some(record)); + } + } +} +/// Writes one reply as a single last fragment. +pub fn write_record(stream: &mut impl Write, reply: &[u8]) -> io::Result<()> { + let mut framed = Vec::with_capacity(reply.len() + 4); + framed.extend_from_slice(&(reply.len() as u32 | LAST_FRAGMENT).to_be_bytes()); + framed.extend_from_slice(reply); + stream.write_all(&framed) +} + +/// Decodes the call header; the returned reader is positioned at the arguments. +pub fn parse_call(record: &[u8]) -> Result<(Call, Reader<'_>), Refused> { + let mut reader = Reader::new(record); + let header = (|| -> Result<(u32, u32, u32), Garbage> { + Ok((reader.u32()?, reader.u32()?, reader.u32()?)) + })(); + let (xid, kind, rpcvers) = header.map_err(|_| Refused::Undecodable)?; + if kind != CALL { + return Err(Refused::Undecodable); + } + if rpcvers != 2 { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(RPC_MISMATCH); + reply.u32(2).u32(2); + return Err(Refused::Reply(reply.into_bytes())); + } + let body = (|| -> Result<(u32, u32, u32, u32, &[u8]), Garbage> { + let program = reader.u32()?; + let version = reader.u32()?; + let procedure = reader.u32()?; + let flavor = reader.u32()?; + let credential = reader.opaque(AUTH_BODY)?; + let _verifier_flavor = reader.u32()?; + let _verifier = reader.opaque(AUTH_BODY)?; + Ok((program, version, procedure, flavor, credential)) + })(); + let (program, version, procedure, flavor, credential) = + body.map_err(|_| Refused::Undecodable)?; + let credentials = match flavor { + AUTH_NONE => Credentials::default(), + AUTH_UNIX => match unix_credentials(credential) { + Ok(unix) => Credentials { unix: Some(unix) }, + Err(Garbage) => return Err(Refused::Reply(denied_auth(xid))), + }, + _ => return Err(Refused::Reply(denied_auth(xid))), + }; + Ok(( + Call { + xid, + program, + version, + procedure, + credentials, + }, + reader, + )) +} +fn unix_credentials(body: &[u8]) -> Result { + let mut reader = Reader::new(body); + let _stamp = reader.u32()?; + let _machine = reader.opaque(255)?; + let uid = reader.u32()?; + let gid = reader.u32()?; + let count = reader.u32()? as usize; + if count > 16 { + return Err(Garbage); + } + let groups = (0..count) + .map(|_| reader.u32()) + .collect::, _>>()?; + Ok(UnixCredentials { uid, gid, groups }) +} +fn denied_auth(xid: u32) -> Vec { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(AUTH_ERROR); + reply.u32(AUTH_TOOWEAK); + reply.into_bytes() +} +/// The accepted-reply header for `xid`; procedure results follow a success. +pub fn accepted(xid: u32, status: Accept) -> Writer { + let mut reply = Writer::with_capacity(256); + reply.u32(xid).u32(REPLY).u32(MSG_ACCEPTED); + reply.u32(AUTH_NONE).u32(0); + match status { + Accept::Success => reply.u32(0), + Accept::ProgramUnavailable => reply.u32(1), + Accept::ProgramMismatch { low, high } => reply.u32(2).u32(low).u32(high), + Accept::ProcedureUnavailable => reply.u32(3), + Accept::GarbageArguments => reply.u32(4), + }; + reply +} diff --git a/engine/crates/layerfs-nfs/src/server.rs b/engine/crates/layerfs-nfs/src/server.rs new file mode 100644 index 0000000000..19b208866d --- /dev/null +++ b/engine/crates/layerfs-nfs/src/server.rs @@ -0,0 +1,335 @@ +//! One export on one loopback TCP listener: MOUNT and NFS on the same port. +//! +//! Each accepted connection is served by its own thread, one call at a time, +//! in arrival order. A call's reply is written first; only after that single +//! send attempt returned are the publication tickets of the mutations it +//! decided released (see [`crate::change::Tickets`]). Stop closes the listener and every connection, joins +//! their threads and only then gives back the synthetic kernel's holdings, +//! so no call is running when references are released. +use crate::{ + change::Tickets, + kernel::{Kernel, Released}, + mount, + nfs::{self, Policy}, + rpc::{self, Accept, Refused}, +}; +use layerfs_fuse::ports::MountServices; +use layerfs_overlay::KernelMount; +use std::{ + io, + net::{Shutdown, SocketAddr, TcpListener, TcpStream}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +/// What the export shows and allows. +#[derive(Clone, Debug)] +pub struct ExportConfig { + /// The path MNT accepts, normally "/". + pub path: String, + /// The identity every inode is shown as. + pub uid: u32, + pub gid: u32, + /// Changing procedures answer `NFS3ERR_ROFS`. + pub read_only: bool, + /// Keep macOS AppleDouble `._*` names out of the Workspace: a regular + /// file lives in the export's memory, any other kind is refused (see + /// [`crate::apple_double`]). + pub refuse_apple_double: bool, + /// Lookup references kept before the least recently used are collected + /// (see [`crate::kernel`]); [`crate::kernel::LOOKUPS_HELD`] by default. + pub lookups_held: usize, +} +impl ExportConfig { + /// A writable export of "/" shown as `uid`:`gid`, refusing AppleDouble + /// names, with the default reference bound. + pub fn writable(uid: u32, gid: u32) -> Self { + Self { + path: "/".into(), + uid, + gid, + read_only: false, + refuse_apple_double: true, + lookups_held: crate::kernel::LOOKUPS_HELD, + } + } +} + +/// Calls served and time spent, per procedure. +pub struct Counters { + nfs: [(AtomicU64, AtomicU64); 22], + mount: [(AtomicU64, AtomicU64); 6], +} +impl Default for Counters { + fn default() -> Self { + Self { + nfs: std::array::from_fn(|_| Default::default()), + mount: std::array::from_fn(|_| Default::default()), + } + } +} +/// One procedure's totals. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Count { + pub procedure: &'static str, + pub calls: u64, + pub nanoseconds: u64, +} +const MOUNT_PROCEDURES: [&str; 6] = ["MNT_NULL", "MNT", "DUMP", "UMNT", "UMNTALL", "EXPORT"]; +impl Counters { + fn charge(&self, program: u32, procedure: u32, started: Instant) { + let slot = match program { + nfs::PROGRAM => self.nfs.get(procedure as usize), + mount::PROGRAM => self.mount.get(procedure as usize), + _ => None, + }; + if let Some((calls, time)) = slot { + calls.fetch_add(1, Ordering::Relaxed); + let spent = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + time.fetch_add(spent, Ordering::Relaxed); + } + } + /// Procedures called at least once. + pub fn snapshot(&self) -> Vec { + let rows = |names: &[&'static str], slots: &[(AtomicU64, AtomicU64)]| { + names + .iter() + .zip(slots) + .map(|(name, (calls, time))| Count { + procedure: name, + calls: calls.load(Ordering::Relaxed), + nanoseconds: time.load(Ordering::Relaxed), + }) + .filter(|count| count.calls > 0) + .collect::>() + }; + let mut all = rows(&MOUNT_PROCEDURES, &self.mount); + all.extend(rows(&nfs::PROCEDURES, &self.nfs)); + all + } +} + +struct Shared { + kernel: Kernel, + /// The WRITE/COMMIT verifier: fresh for every export. + verifier: [u8; 8], + config: ExportConfig, + counters: Counters, + stopping: AtomicBool, + connections: Mutex)>>, +} +/// A running export. +pub struct Server { + address: SocketAddr, + shared: Arc, + accept: Option>, +} +impl Server { + /// Serves the engine mount `mount` over `services` on `listener`. + pub fn start( + listener: TcpListener, + services: Arc, + mount: KernelMount, + config: ExportConfig, + ) -> io::Result { + let address = listener.local_addr()?; + let shared = Arc::new(Shared { + kernel: Kernel::new(services, mount, config.uid, config.gid) + .refusing_apple_double(config.refuse_apple_double) + .holding(config.lookups_held), + verifier: fresh_verifier(), + config, + counters: Counters::default(), + stopping: AtomicBool::new(false), + connections: Mutex::default(), + }); + let accepting = shared.clone(); + let accept = thread::Builder::new() + .name("layerfs-nfs-accept".into()) + .spawn(move || accept_loop(&listener, &accepting))?; + Ok(Self { + address, + shared, + accept: Some(accept), + }) + } + pub fn local_addr(&self) -> SocketAddr { + self.address + } + pub fn counters(&self) -> Vec { + self.shared.counters.snapshot() + } + /// The synthetic kernel's current holdings. + pub fn held(&self) -> Released { + self.shared.kernel.held() + } + /// Closes the implicit file descriptors no silly name needs, for example + /// before a Commit; the next READ or WRITE reopens one. Returns how many. + pub fn close_files(&self) -> usize { + self.shared.kernel.close_files() + } + /// After a Commit installed a root other than the one the client saw: + /// closes the implicit file and enumeration descriptors, so the next + /// READ reopens the installed file and a continuing READDIR restarts. + /// Attributes the client cached are revalidated on its own schedule + /// (`actimeo`, close-to-open). Returns (files, directories) closed. + pub fn installed(&self) -> (usize, usize) { + ( + self.shared.kernel.close_files(), + self.shared.kernel.close_directories(), + ) + } + /// After an install: marks the stale `serials` the client may cache and + /// returns their paths below the mount point, for the mode-preserving + /// SETATTR that makes the client drop its cache (see [`crate::refresh`]). + pub fn refresh(&self, serials: impl IntoIterator) -> Vec { + use std::os::unix::ffi::OsStrExt; + (self.shared.kernel.refresh(serials).into_iter()) + .map(|names| { + (names.iter()) + .map(|name| std::ffi::OsStr::from_bytes(name)) + .collect() + }) + .collect() + } + /// The write verifier this export answers WRITE and COMMIT with. + pub fn verifier(&self) -> [u8; 8] { + self.shared.verifier + } + /// Stops accepting, closes every connection, joins every thread, then + /// releases the synthetic kernel's descriptors and references. The engine + /// mount itself stays with the caller, who revokes it afterwards. + pub fn stop(mut self) -> Released { + self.shutdown() + } + fn shutdown(&mut self) -> Released { + let Some(accept) = self.accept.take() else { + return Released::default(); + }; + self.shared.stopping.store(true, Ordering::Release); + // Wake the blocking accept; it sees the flag and returns. + drop(TcpStream::connect(self.address)); + drop(accept.join()); + let connections = std::mem::take( + &mut *self + .shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ); + for (stream, _) in &connections { + drop(stream.shutdown(Shutdown::Both)); + } + for (_, thread) in connections { + drop(thread.join()); + } + self.shared.kernel.release() + } +} +impl Drop for Server { + fn drop(&mut self) { + self.shutdown(); + } +} +fn accept_loop(listener: &TcpListener, shared: &Arc) { + for stream in listener.incoming() { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let Ok(stream) = stream else { continue }; + drop(stream.set_nodelay(true)); + let Ok(control) = stream.try_clone() else { + continue; + }; + let serving = shared.clone(); + let spawned = thread::Builder::new() + .name("layerfs-nfs-connection".into()) + .spawn(move || connection(stream, &serving)); + if let Ok(thread) = spawned { + let mut connections = shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + // A closed connection's thread has ended; forget it. + connections.retain(|(_, thread)| !thread.is_finished()); + connections.push((control, thread)); + } + } +} +fn connection(mut stream: TcpStream, shared: &Shared) { + while let Ok(Some(record)) = rpc::read_record(&mut stream) { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let mut tickets = Tickets::default(); + let Some(reply) = answer(shared, &record, &mut tickets) else { + tickets.replied(&shared.kernel); + continue; + }; + // The single reply attempt, then the release of every ticket it + // owed, whether or not the reply reached the client. + let sent = rpc::write_record(&mut stream, &reply); + tickets.replied(&shared.kernel); + if sent.is_err() { + return; + } + } +} +/// A verifier no earlier export of this process or machine is likely to +/// have used: the clock and the process, under a per-process random key. +fn fresh_verifier() -> [u8; 8] { + use std::hash::BuildHasher; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_nanos()); + std::collections::hash_map::RandomState::new() + .hash_one((now, std::process::id())) + .to_be_bytes() +} +/// The reply to one record, or none when the record is not a call. +/// Mutations the call decided go to `tickets`. +fn answer(shared: &Shared, record: &[u8], tickets: &mut Tickets) -> Option> { + let started = Instant::now(); + let (call, mut arguments) = match rpc::parse_call(record) { + Ok(parsed) => parsed, + Err(Refused::Reply(reply)) => return Some(reply), + Err(Refused::Undecodable) => return None, + }; + let mut out = rpc::accepted(call.xid, Accept::Success); + let result = match (call.program, call.version) { + (nfs::PROGRAM, nfs::VERSION) => nfs::call( + &shared.kernel, + Policy { + read_only: shared.config.read_only, + }, + call.procedure, + &call.credentials, + &mut arguments, + &mut out, + tickets, + shared.verifier, + ), + (mount::PROGRAM, mount::VERSION) => mount::call( + &shared.kernel, + &shared.config.path, + call.procedure, + &mut arguments, + &mut out, + ), + (nfs::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + (mount::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + _ => Err(Accept::ProgramUnavailable), + }; + let reply = match result { + Ok(()) => out.into_bytes(), + Err(status) => rpc::accepted(call.xid, status).into_bytes(), + }; + shared + .counters + .charge(call.program, call.procedure, started); + Some(reply) +} diff --git a/engine/crates/layerfs-nfs/src/xdr.rs b/engine/crates/layerfs-nfs/src/xdr.rs new file mode 100644 index 0000000000..18f572ed27 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/xdr.rs @@ -0,0 +1,120 @@ +//! XDR (RFC 4506) primitives over one received record and one reply buffer. +use std::fmt; + +/// The record ended before a field, or a length exceeded its declared bound. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Garbage; +impl fmt::Display for Garbage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("malformed XDR arguments") + } +} +impl std::error::Error for Garbage {} + +/// Sequential decoding of one record. Every read is bounds-checked. +pub struct Reader<'a> { + data: &'a [u8], + at: usize, +} +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data, at: 0 } + } + fn take(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let end = self.at.checked_add(length).ok_or(Garbage)?; + let slice = self.data.get(self.at..end).ok_or(Garbage)?; + self.at = end; + Ok(slice) + } + pub fn u32(&mut self) -> Result { + let bytes = self.take(4)?; + Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + pub fn u64(&mut self) -> Result { + Ok((u64::from(self.u32()?) << 32) | u64::from(self.u32()?)) + } + pub fn bool(&mut self) -> Result { + match self.u32()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(Garbage), + } + } + /// Fixed-length opaque data, followed by its padding. + pub fn fixed(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let value = self.take(length)?; + self.take(pad(length))?; + Ok(value) + } + /// Variable-length opaque data of at most `limit` bytes. + pub fn opaque(&mut self, limit: usize) -> Result<&'a [u8], Garbage> { + let length = self.u32()? as usize; + if length > limit { + return Err(Garbage); + } + self.fixed(length) + } + /// The bytes not yet decoded. + pub fn remaining(&self) -> usize { + self.data.len() - self.at + } +} + +/// Bytes of padding after `length` bytes of opaque data. +pub const fn pad(length: usize) -> usize { + (4 - length % 4) % 4 +} +/// Encoded size of variable-length opaque data of `length` bytes. +pub const fn opaque_size(length: usize) -> usize { + 4 + length + pad(length) +} + +/// One reply under construction. +#[derive(Default)] +pub struct Writer { + data: Vec, +} +impl Writer { + pub fn new() -> Self { + Self::default() + } + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: Vec::with_capacity(capacity), + } + } + pub fn u32(&mut self, value: u32) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn u64(&mut self, value: u64) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn bool(&mut self, value: bool) -> &mut Self { + self.u32(u32::from(value)) + } + /// Fixed-length opaque data and its padding. + pub fn fixed(&mut self, value: &[u8]) -> &mut Self { + self.data.extend_from_slice(value); + self.data.extend_from_slice(&[0; 3][..pad(value.len())]); + self + } + /// Variable-length opaque data; callers bound `value` by its protocol limit. + pub fn opaque(&mut self, value: &[u8]) -> &mut Self { + self.u32(value.len() as u32); + self.fixed(value) + } + pub fn len(&self) -> usize { + self.data.len() + } + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + pub fn into_bytes(self) -> Vec { + self.data + } + pub fn as_bytes(&self) -> &[u8] { + &self.data + } +} diff --git a/engine/crates/layerfs-nfs/tests/collect.rs b/engine/crates/layerfs-nfs/tests/collect.rs new file mode 100644 index 0000000000..b4dfaddbe9 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/collect.rs @@ -0,0 +1,170 @@ +//! Reference collection (A3, plan A): an export configured with a small +//! reference bound walks a tree far larger than the bound, repeatedly, and +//! keeps both its own holdings and the engine's kernel entries bounded. +//! Handles of collected serials keep working (resolved again by their name); +//! a handle whose name now reaches another inode is stale; the product +//! Commit afterwards holds exactly what the client made. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, OK, STALE}; +use std::{fs, net::TcpListener, path::PathBuf, time::Duration}; + +const UID: u32 = 501; +const GID: u32 = 20; +const BOUND: usize = 64; +const DIRECTORIES: usize = 8; +const FILES: usize = 120; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-c{}-{label}", std::process::id())) +} +fn content(directory: usize, file: usize) -> Vec { + format!("d{directory}/f{file}\n") + .repeat(file % 7 + 1) + .into_bytes() +} +fn kernel_entries(rig: &rig::Rig) -> u64 { + let operation = rig.bound.operation().unwrap(); + operation.overlay().resources().unwrap().open.kernel_entries +} + +#[test] +fn references_stay_bounded_and_collected_handles_resolve() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + for directory in 0..DIRECTORIES { + let path = source.join(format!("d{directory}/inner")); + fs::create_dir_all(&path).unwrap(); + for file in 0..FILES { + fs::write( + source.join(format!("d{directory}/f{file}")), + content(directory, file), + ) + .unwrap(); + } + fs::write(path.join("leaf"), b"leaf\n").unwrap(); + } + let mut rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + rig.lookups_held = BOUND; + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let total = DIRECTORIES * (FILES + 3); + // The bound is soft: the most recently used serials, the directories a + // held serial was reached through and serials holding a descriptor are + // never collected. + let ceiling = BOUND + layerfs_nfs::kernel::RECENT_KEPT as usize + DIRECTORIES * 2 + 8; + let mut first: Vec<(Vec, u64, usize, usize)> = Vec::new(); + let mut peak_held = 0; + let mut peak_engine = 0; + for round in 0..6 { + for directory in 0..DIRECTORIES { + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + for file in 0..FILES { + let (handle, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + if round == 0 && file % 15 == 0 { + first.push((handle, attr.fileid, directory, file)); + } + } + let (inner, _) = client.lookup(&dir, b"inner").unwrap(); + client.lookup(&inner, b"leaf").unwrap(); + // Churn: a file made and removed in every directory, every round. + let made = format!("tmp{round}"); + client.touch(&dir, made.as_bytes(), 0o644).unwrap(); + client.remove(&dir, made.as_bytes()).unwrap(); + peak_held = peak_held.max(server.held().lookups); + peak_engine = peak_engine.max(kernel_entries(&rig)); + } + println!( + "ROUND {round} held={:?} engine_kernel_entries={}", + server.held(), + kernel_entries(&rig) + ); + } + assert!(peak_held <= ceiling, "export held {peak_held} > {ceiling}"); + // The engine counts the export's references (plus the root's). + assert!( + peak_engine <= ceiling as u64 + 2, + "engine kernel entries {peak_engine} > {ceiling}" + ); + assert!( + server.held().collected as usize >= total * 5, + "{:?}", + server.held() + ); + // Every early handle was collected long ago; each resolves to its file. + for (handle, fileid, directory, file) in &first { + let attr = client.getattr(handle).unwrap(); + assert_eq!(attr.fileid, *fileid); + assert_eq!(client.read_all(handle, 4096), content(*directory, *file)); + } + let resolved = server.held().resolved; + assert!(resolved >= first.len() as u64, "{:?}", server.held()); + // A collected handle whose name was replaced by another file is stale. + let (victim, victim_fileid, directory, file) = first[1].clone(); + for _ in 0..3 { + // collect it again + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d7").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + } + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + let replacement = client.touch(&dir, b"replacement", 0o644).unwrap(); + client + .rename(&dir, b"replacement", &dir, format!("f{file}").as_bytes()) + .unwrap(); + assert_eq!(client.getattr(&victim).map(|attr| attr.fileid), Err(STALE)); + let (now, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + assert_ne!(attr.fileid, victim_fileid); + assert_eq!(now, replacement.handle); + // `..` of a directory whose parent was collected still answers. + let (d3, _) = client.lookup(&root, b"d3").unwrap(); + let (inner, _) = client.lookup(&d3, b"inner").unwrap(); + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d6").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + let (up, up_attr) = client.lookup(&inner, b"..").unwrap(); + assert_eq!(up_attr.fileid, client.getattr(&d3).unwrap().fileid); + assert_eq!(client.getattr(&up).map(|_| OK), Ok(OK)); + // The Commit holds what the client made. + let closed = server.close_files(); + let done = rig.commit(); + assert!(done.history.is_some()); + println!("COMMITTED closed={closed} held={:?}", server.held()); + let view = rig.view([93; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (replaced, _) = reader + .walk_to(&view_root, &format!("d{directory}/f{file}")) + .unwrap(); + assert!(reader.read_all(&replaced, 4096).is_empty()); + let (kept, _) = reader.walk_to(&view_root, "d5/f119").unwrap(); + assert_eq!(reader.read_all(&kept, 4096), content(5, 119)); + assert_eq!( + reader.lookup(&view_root, b"tmp0").map(|_| ()), + Err(client::NOENT) + ); + drop(reader); + viewer.stop(); + rig.close_view(view); + drop(client); + let released = server.stop(); + println!("RELEASED {released:?}"); + assert!(released.lookups <= ceiling); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(&source).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/daemon.rs b/engine/crates/layerfs-nfs/tests/daemon.rs new file mode 100644 index 0000000000..eacaf9e35b --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/daemon.rs @@ -0,0 +1,292 @@ +//! A3: the daemon's control Service with NFS serving (the native route's +//! mount entry). Every step is a control request through `execute_control`, +//! the function the daemon's control connection answers with: Mount (bind), +//! Attach (export + client mount helper), Status, Commit (client flush, then +//! the unchanged product Commit), Unmount (dirty refusal, reversible busy, +//! terminal) and ForceUnmount. The client mount itself is a recording helper +//! here (this host's kernel has no NFSv3 client); the protocol client of the +//! tests plays the kernel. The kernel-client run is in the A3 VM scripts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, NOENT}; +use layerfs_bridge::control::{ + CommitControl, MountPhase, ReadyMount, Reply, Request, UnmountControl, WorkspaceToken, +}; +use layerfs_daemon::control::{ + ExportServing, ExportServingConfig, Failure, Helper, Mounter, Service, +}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + net::{Ipv4Addr, SocketAddr}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-d{}-{label}", std::process::id())) +} +/// A helper that appends ` ` to `log`; it answers "busy" +/// (exit 32) while `busy` exists. +fn recording(what: &str, log: &Path, busy: Option<&Path>) -> Helper { + let refuse = busy + .map(|flag| { + format!( + "if [ -e {} ]; then echo 'umount: target is busy' >&2; exit 32; fi; ", + flag.display() + ) + }) + .unwrap_or_default(); + Helper(vec![ + "sh".into(), + "-c".into(), + format!("{refuse}echo {what} {{port}} {{dir}} >> {}", log.display()), + ]) +} +fn bind(service: &Service, tag: u8, branch: layerfs_history::BranchId) -> WorkspaceToken { + let workspace = WorkspaceId::from_authority([tag; 32]).unwrap(); + match service + .execute_control(&Request::Mount { workspace, branch }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + } +} +fn attach(service: &Service, token: WorkspaceToken) -> ReadyMount { + match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + } +} +fn phase(service: &Service, token: WorkspaceToken) -> MountPhase { + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => status.kernel.expect("kernel block").phase, + other => panic!("{other:?}"), + } +} +fn commit(service: &Service, token: WorkspaceToken) -> Reply { + service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("over nfs".into()), + })) + .unwrap() + .reply +} +fn unmount(service: &Service, token: WorkspaceToken, discard: bool) -> Result { + service + .execute_control(&Request::Unmount(UnmountControl { token, discard })) + .map(|success| success.reply) +} +fn connect(ready: &ReadyMount) -> Client { + let port = u16::try_from(ready.receipt.mount_id).unwrap(); + Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), UID, GID) +} + +#[test] +fn workspace_control_over_the_nfs_route() { + let source = temp("src"); + let work = temp("work"); + let mounts = temp("mounts"); + let log = temp("helpers.log"); + let busy = temp("busy"); + for path in [&source, &work, &mounts] { + drop(fs::remove_dir_all(path)); + } + drop(fs::remove_file(&log)); + fs::create_dir_all(source.join("src")).unwrap(); + fs::write(source.join("README.md"), b"base readme\n").unwrap(); + fs::write(source.join("src/lib.rs"), b"pub fn base() {}\n").unwrap(); + fs::write(source.join("gone.txt"), b"to be removed\n").unwrap(); + fs::create_dir_all(&mounts).unwrap(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let mounter = Mounter { + mount: recording("mount", &log, None), + unmount: recording("unmount", &log, Some(&busy)), + force: recording("force", &log, None), + flush: Some(recording("flush", &log, None)), + }; + let export = Arc::new(ExportServing::start(ExportServingConfig::new( + mounts.clone(), + UID, + GID, + mounter, + ))); + let service = rig.service(export); + assert!(service.serves_nfs()); + + // Mount + Attach: Ready with an NFS receipt; the helper mounted it. + let token = bind(&service, 31, rig.branch); + assert_eq!(phase(&service, token), MountPhase::Unattached); + let ready = attach(&service, token); + assert_eq!(ready.receipt.abi_major, 0); + assert_eq!( + ready.directory, + mounts.join(token.namespace.to_string()).to_str().unwrap() + ); + assert!(Path::new(&ready.directory).is_dir()); + assert_eq!(phase(&service, token), MountPhase::Ready); + let port = ready.receipt.mount_id; + let logged = fs::read_to_string(&log).unwrap(); + assert_eq!(logged, format!("mount {port} {}\n", ready.directory)); + // A second Attach of the same Workspace is refused before effects. + assert!(service.execute_control(&Request::Attach(token)).is_err()); + + // Writes through the export. + let mut nfs = connect(&ready); + let root = nfs.mnt("/"); + let made = nfs.touch(&root, b"made.txt", 0o644).unwrap(); + nfs.write_all(&made.handle, 0, b"made over nfs\n", 8); + let dir = nfs.mkdir(&root, b"newdir", 0o755).unwrap(); + let inner = nfs.touch(&dir.handle, b"inner.bin", 0o600).unwrap(); + nfs.write_all(&inner.handle, 0, &vec![7u8; 300_000], 65536); + nfs.remove(&root, b"gone.txt").unwrap(); + let (src, _) = nfs.lookup(&root, b"src").unwrap(); + nfs.rename(&src, b"lib.rs", &dir.handle, b"lib.rs").unwrap(); + nfs.symlink(&root, b"link", b"newdir/lib.rs").unwrap(); + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => { + let block = status.kernel.unwrap(); + assert!(block.work.unwrap().served > 10, "{block:?}"); + assert!(status.local.unwrap().dirty_inodes > 0); + } + other => panic!("{other:?}"), + } + // Unmount refuses uncommitted changes, naming them; nothing closes. + match unmount(&service, token, false).unwrap() { + Reply::Dirty(report) => { + assert!(report.paths.iter().any(|p| p == "/made.txt"), "{report:?}") + } + other => panic!("{other:?}"), + } + assert_eq!(phase(&service, token), MountPhase::Ready); + + // Commit: the flush helper ran first, then the product Commit. + match commit(&service, token) { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + assert!(fs::read_to_string(&log) + .unwrap() + .contains(&format!("flush {port} "))); + // The export keeps serving after the Commit. + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + + // An independent view of the committed Branch. + let view = rig.view([92; 32]); + let viewer = rig.serve_view( + std::net::TcpListener::bind("127.0.0.1:0").unwrap(), + &view, + UID, + GID, + ); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (handle, _) = reader.walk_to(&view_root, "made.txt").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"made over nfs\n"); + let (handle, attr) = reader.walk_to(&view_root, "newdir/inner.bin").unwrap(); + assert_eq!((attr.size, attr.mode & 0o777), (300_000, 0o600)); + assert_eq!(reader.read_all(&handle, 65536), vec![7u8; 300_000]); + let (handle, _) = reader.walk_to(&view_root, "newdir/lib.rs").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"pub fn base() {}\n"); + assert_eq!( + reader.walk_to(&view_root, "gone.txt").map(|_| ()), + Err(NOENT) + ); + assert_eq!( + reader.walk_to(&view_root, "src/lib.rs").map(|_| ()), + Err(NOENT) + ); + let (handle, _) = reader.walk_to(&view_root, "link").unwrap(); + assert_eq!(reader.readlink(&handle).unwrap(), b"newdir/lib.rs"); + drop(reader); + viewer.stop(); + rig.close_view(view); + + // A busy client mount is the reversible answer: still Ready, serving. + fs::write(&busy, b"").unwrap(); + let refused = unmount(&service, token, true).unwrap_err(); + assert!(format!("{refused:?}").contains("busy"), "{refused:?}"); + assert_eq!(phase(&service, token), MountPhase::Ready); + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + fs::remove_file(&busy).unwrap(); + + // Terminal unmount: client unmount, export stopped, engine mount + // revoked, directory removed, Workspace closed. + let held = service.export_held(token).unwrap(); + assert_eq!( + unmount(&service, token, true).unwrap(), + Reply::Unmounted(token) + ); + assert!(!Path::new(&ready.directory).exists()); + let released = service.export_released(token).unwrap(); + assert_eq!(released.lookups, held.lookups); + assert_eq!(released.retained, 0); + drop(nfs); + assert!(service.execute_control(&Request::Status(token)).is_err()); + + // ForceUnmount of a second Workspace with uncommitted changes. + let second = bind(&service, 32, rig.branch); + let ready2 = attach(&service, second); + let mut nfs = connect(&ready2); + let root = nfs.mnt("/"); + nfs.touch(&root, b"lost.txt", 0o644).unwrap(); + match service + .execute_control(&Request::ForceUnmount { + token: second, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => assert_eq!(done.token, second), + other => panic!("{other:?}"), + } + assert!(!Path::new(&ready2.directory).exists()); + drop(nfs); + let logged = fs::read_to_string(&log).unwrap(); + println!("HELPERS\n{logged}"); + assert_eq!( + logged.lines().filter(|l| l.starts_with("mount ")).count(), + 2 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("unmount ")).count(), + 1 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("force ")).count(), + 1 + ); + + drop(service); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + for path in [&source, &mounts] { + fs::remove_dir_all(path).unwrap(); + } + fs::remove_file(&log).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/export.rs b/engine/crates/layerfs-nfs/tests/export.rs new file mode 100644 index 0000000000..96c25acc10 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/export.rs @@ -0,0 +1,196 @@ +//! A committed Branch served read-only over loopback NFSv3: every path, +//! byte, link target, mode and identity of the source tree through the +//! protocol alone, the synthetic kernel's reference discipline, cookies, +//! refusals and stale handles across engine mounts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{Attr, Client, BADHANDLE, BAD_COOKIE, NOENT, ROFS, STALE}; +use layerfs_nfs::xdr::Writer; +use std::{ + collections::BTreeSet, + fs, + net::TcpListener, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + time::Duration, +}; +use tree::{oracle, walk, Node}; + +const UID: u32 = 501; +const GID: u32 = 20; +const MANY: usize = 700; +const BIG: usize = 300_001; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested/deep")).unwrap(); + fs::create_dir_all(root.join("many")).unwrap(); + fs::create_dir_all(root.join("emptydir")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write( + root.join("big.bin"), + (0..BIG).map(|n| (n % 251) as u8).collect::>(), + ) + .unwrap(); + fs::write(root.join("empty"), b"").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep/file.txt"), b"deep\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/lib-alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + std::os::unix::fs::symlink("../nowhere", root.join("dangling")).unwrap(); + fs::write(root.join("unicod\u{e9}.txt"), "caf\u{e9}\n").unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\necho run\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); + for n in 0..MANY { + fs::write(root.join(format!("many/f-{n:04}")), format!("{n}\n")).unwrap(); + } +} +#[test] +fn a_committed_branch_reads_completely_over_nfs() { + let base = temp("complete"); + let tree = base.join("tree"); + source(&tree); + let expected = oracle(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + assert_eq!(rig.entries as usize, expected.len()); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + + let (paths, bytes, identities) = walk(&mut client, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + assert!(bytes > BIG); + // One inode behind both names of the hard link, two names for it, and + // distinct inodes everywhere else. + let alias = expected.iter().find_map(|(path, node)| match node { + Node::File { ino, .. } if path == Path::new("src/lib.rs") => Some(*ino), + _ => None, + }); + let fileids = &identities[&alias.unwrap()]; + assert_eq!(fileids.len(), 1, "both names show one fileid"); + let lib = client.mnt("/"); + let (src, _) = client.lookup(&lib, b"src").unwrap(); + let (_, lib_attr) = client.lookup(&src, b"lib.rs").unwrap(); + assert_eq!(lib_attr.nlink, 2); + let all: BTreeSet = identities.values().flatten().copied().collect(); + assert_eq!( + all.len(), + identities.len(), + "distinct inodes, distinct fileids" + ); + + // Re-walking takes no further engine references: one per handed-out serial. + // Every path but the root holds one reference; the hard link's two names + // share one serial. + let serials = expected.len() - 2; + let held = server.held(); + assert_eq!(held.lookups, serials, "one reference per non-root serial"); + walk(&mut client, &expected, 64 * 1024, (UID, GID)); + assert_eq!(server.held().lookups, held.lookups); + + let released = server.stop(); + assert_eq!(released.lookups, serials); + assert!(released.files > 0 && released.directories > 0); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0, "every Store reader was given back"); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn cookies_refusals_and_stale_handles() { + let base = temp("protocol"); + let tree = base.join("tree"); + source(&tree); + let mut rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (many, _) = client.lookup(&root, b"many").unwrap(); + + // A small budget spreads one enumeration over many replies; every name + // appears exactly once, in both procedures. + for plus in [false, true] { + let (entries, calls) = client.list(&many, 1024, plus); + let names: BTreeSet> = entries.iter().map(|entry| entry.name.clone()).collect(); + assert_eq!(names.len(), entries.len(), "no duplicate (plus={plus})"); + assert_eq!(entries.len(), MANY + 2, "plus={plus}"); + assert!(calls > 10, "{calls} calls (plus={plus})"); + } + // A verifier the export never issued, and a budget too small for one entry. + assert_eq!( + client.readdir(&many, 5, 0xdead_beef, 4096, false), + Err(BAD_COOKIE) + ); + // Missing names, a foreign handle and every changing procedure. + assert_eq!(client.lookup(&root, b"absent").map(|_| ()), Err(NOENT)); + // Both LOOKUP answers carry the directory's attributes. + let root_attr = client.getattr(&root).unwrap(); + let (status, found, directory) = client.lookup_with_directory(&root, b"many"); + assert_eq!( + (status, found.map(|(handle, _)| handle)), + (0, Some(many.clone())) + ); + assert_eq!(directory, Some(root_attr.clone())); + let (status, found, directory) = client.lookup_with_directory(&root, b"absent"); + assert_eq!((status, found), (NOENT, None)); + assert_eq!(directory, Some(root_attr)); + // Not a handle this export issues: wrong version or length. A version-1 + // handle of another engine mount is stale (its key is gone); a forged + // tag under this mount is refused (see protocol.rs). + assert_eq!(client.getattr(&[2; 25]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 10]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 25]), Err(STALE)); + for procedure in [2, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21] { + let mut arguments = Writer::new(); + arguments.opaque(&root); + assert_eq!(client.nfs(procedure, &arguments).0, ROFS, "{procedure}"); + } + // A read-only export grants reads only, whatever the mode allows. + let (readme, readme_attr): (Vec, Attr) = client.lookup(&root, b"README.md").unwrap(); + let host = fs::metadata(tree.join("README.md")).unwrap().mode() & 0o7777; + assert_eq!(readme_attr.mode, host); + assert_eq!(host & 0o444, 0o444, "the fixture is readable by everyone"); + assert_eq!(client.access(&readme, 0x3f), Ok(0x01)); + let mut other = Client::connect(server.local_addr(), 1234, 1234); + assert_eq!(other.access(&readme, 0x3f), Ok(0x01), "other: r--"); + let (run, _) = client.lookup(&root, b"run.sh").unwrap(); + assert_eq!(client.access(&run, 0x3f), Ok(0x01 | 0x20)); + // Reads beyond the end, and windows that are not page aligned. + let (big, big_attr) = client.lookup(&root, b"big.bin").unwrap(); + assert_eq!( + client.read(&big, big_attr.size + 10, 4096), + Ok((vec![], true)) + ); + let (window, eof) = client.read(&big, 1000, 70_000).unwrap(); + assert!(!eof); + assert_eq!( + window, + (1000..71_000).map(|n| (n % 251) as u8).collect::>() + ); + + // A restarted export is a new engine mount: earlier handles are stale. + drop(client); + drop(other); + let released = server.stop(); + rig.remount(); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + assert_eq!(client.getattr(&readme), Err(STALE)); + assert_eq!(client.getattr(&root), Err(STALE)); + let fresh = client.mnt("/"); + assert!(client.getattr(&fresh).is_ok()); + drop(client); + let released_again = server.stop(); + assert!(released.lookups > 0); + let (_, outstanding) = rig.stop(released_again); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs new file mode 100644 index 0000000000..781e82710a --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -0,0 +1,142 @@ +//! Real control Service with process helpers, no kernel NFS mount required. +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use layerfs_bridge::control::{MountPhase, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + path::PathBuf, + process::Command, + sync::{mpsc, Arc}, + time::Duration, +}; + +fn base(label: &str) -> PathBuf { + let root = std::env::var_os("H07_TEST_ROOT") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + root.join(format!("helper-{}-{label}", std::process::id())) +} +fn run_case(label: &str, helper: Helper, before: impl FnOnce()) -> (String, MountPhase) { + let base = base(label); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let service = rig.service(Arc::new(ExportServing::start(ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper::words("true").unwrap(), + unmount: helper, + force: Helper::words("true").unwrap(), + flush: None, + }, + )))); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([33; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + service.execute_control(&Request::Attach(token)).unwrap(); + before(); + let reply = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + let phase = match service.execute_control(&Request::Status(token)) { + Ok(done) => match done.reply { + Reply::Status(status) => status.kernel.unwrap().phase, + other => panic!("{other:?}"), + }, + Err(_) => MountPhase::Unattached, + }; + if phase == MountPhase::Retained { + let repeated = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + assert_eq!( + reply, repeated, + "retained request must not launch another helper" + ); + } + // No actual kernel mount exists in this fixture. Service owns retained + // export custody until drop. + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + (reply, phase) +} +#[test] +fn exited_helper_does_not_wait_for_a_descendants_pipe_eof() { + let root = base("pipe-holder"); + fs::create_dir_all(&root).unwrap(); + let pidfile = root.join("descendant.pid"); + let script = format!("sleep 20 & echo $! > {}; exit 0", pidfile.display()); + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = run_case( + "pipes", + Helper(vec!["sh".into(), "-c".into(), script]), + || {}, + ); + let _ = send.send(result); + }); + let result = receive.recv_timeout(Duration::from_secs(3)); + // Always release the descendant, including on failure, before asserting. + if let Ok(pid) = fs::read_to_string(&pidfile) { + let _ = Command::new("kill").args(["-KILL", pid.trim()]).status(); + } + worker.join().unwrap(); + fs::remove_dir_all(root).unwrap(); + let (reply, _) = result.expect("exited helper blocked on descendant's open pipe"); + assert!(reply.contains("Unmounted"), "{reply}"); +} +#[test] +fn timed_out_helper_returns_retained_and_does_not_replay() { + // STOP makes the helper ignore TERM; KILL must reap it without wait(). + // A normal process cannot ignore SIGKILL: this tests the actual deadline, + // not the kernel's uninterruptible U state. + let (reply, phase) = run_case( + "timeout", + Helper(vec!["sh".into(), "-c".into(), "kill -STOP $$".into()]), + || {}, + ); + assert!( + reply.contains("Retained") && reply.contains("60s"), + "{reply}" + ); + assert_eq!(phase, MountPhase::Retained); + assert_eq!(HELPER_WAIT, Duration::from_secs(60)); +} + +#[test] +fn verbose_helper_preserves_busy_refusal_without_pipe_deadlock() { + let (reply, phase) = run_case( + "verbose", + Helper(vec!["sh".into(), "-c".into(), "echo busy >&2; i=0; while [ $i -lt 5000 ]; do echo verbose-helper-output; i=$((i+1)); done; exit 32".into()]), + || {}, + ); + assert!(reply.to_ascii_lowercase().contains("busy"), "{reply}"); + assert_eq!(phase, MountPhase::Ready); +} diff --git a/engine/crates/layerfs-nfs/tests/protocol.rs b/engine/crates/layerfs-nfs/tests/protocol.rs new file mode 100644 index 0000000000..c513df60aa --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/protocol.rs @@ -0,0 +1,123 @@ +//! XDR, RPC record marking, call headers and file handles through the public API. +use layerfs_nfs::{ + handle::{HandleError, Handles, HANDLE_BYTES}, + rpc::{parse_call, read_record, write_record, Refused, RECORD_LIMIT}, + xdr::{opaque_size, Reader, Writer}, +}; + +#[test] +fn xdr_round_trips_with_padding() { + let mut out = Writer::new(); + out.u32(7) + .u64(u64::MAX - 1) + .bool(true) + .opaque(b"abcde") + .fixed(b"xy"); + assert_eq!(out.len(), 4 + 8 + 4 + opaque_size(5) + 4); + let bytes = out.into_bytes(); + let mut reader = Reader::new(&bytes); + assert_eq!(reader.u32(), Ok(7)); + assert_eq!(reader.u64(), Ok(u64::MAX - 1)); + assert_eq!(reader.bool(), Ok(true)); + assert_eq!(reader.opaque(5), Ok(&b"abcde"[..])); + assert_eq!(reader.fixed(2), Ok(&b"xy"[..])); + assert_eq!(reader.remaining(), 0); + assert!(reader.u32().is_err(), "reads past the record fail"); + let mut long = Writer::new(); + long.opaque(&[0; 9]); + let long = long.into_bytes(); + assert!( + Reader::new(&long).opaque(8).is_err(), + "a length beyond its bound" + ); + assert!( + Reader::new(&[0, 0, 0, 2]).bool().is_err(), + "a bool is 0 or 1" + ); +} + +#[test] +fn records_join_fragments_and_respect_their_bound() { + let mut wire = Vec::new(); + wire.extend_from_slice(&3u32.to_be_bytes()); + wire.extend_from_slice(b"abc"); + wire.extend_from_slice(&(2u32 | (1 << 31)).to_be_bytes()); + wire.extend_from_slice(b"de"); + write_record(&mut wire, b"next").unwrap(); + let mut stream = &wire[..]; + assert_eq!(read_record(&mut stream).unwrap(), Some(b"abcde".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), Some(b"next".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), None, "orderly end"); + let huge = (RECORD_LIMIT as u32 + 1) | (1 << 31); + assert!(read_record(&mut &huge.to_be_bytes()[..]).is_err()); +} + +fn call(rpcvers: u32, flavor: u32, credential: &[u8]) -> Vec { + let mut out = Writer::new(); + out.u32(9).u32(0).u32(rpcvers).u32(100_003).u32(3).u32(1); + out.u32(flavor).opaque(credential).u32(0).u32(0); + out.u32(0xabcd); + out.into_bytes() +} + +#[test] +fn call_headers_carry_unix_credentials() { + let mut unix = Writer::new(); + unix.u32(0) + .opaque(b"host") + .u32(501) + .u32(20) + .u32(2) + .u32(12) + .u32(61); + let record = call(2, 1, unix.as_bytes()); + let (header, mut arguments) = parse_call(&record).unwrap(); + assert_eq!( + (header.xid, header.program, header.version, header.procedure), + (9, 100_003, 3, 1) + ); + let credentials = header.credentials.unix.unwrap(); + assert_eq!( + (credentials.uid, credentials.gid, credentials.groups), + (501, 20, vec![12, 61]) + ); + assert_eq!(arguments.u32(), Ok(0xabcd)); + assert!(parse_call(&call(2, 0, b"")) + .unwrap() + .0 + .credentials + .unix + .is_none()); + // RPC version 3 and an unknown flavor are answered with a denial. + for record in [call(3, 0, b""), call(2, 6, b"")] { + match parse_call(&record) { + Err(Refused::Reply(reply)) => { + assert_eq!(&reply[..12], &[0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 1]) + } + Err(Refused::Undecodable) => panic!("undecodable"), + Ok((call, _)) => panic!("{call:?}"), + } + } + assert!(matches!( + parse_call(&[0, 0, 0, 9, 0, 0, 0, 1]), + Err(Refused::Undecodable) + )); +} + +#[test] +fn handles_are_stable_per_mount_and_stale_after_it() { + let handles = Handles::new(41); + let first = handles.encode(1234); + assert_eq!(first.len(), HANDLE_BYTES); + assert_eq!( + handles.encode(1234), + first, + "the same serial, the same bytes" + ); + assert_eq!(handles.decode(&first), Ok(1234)); + let mut forged = first; + forged[16] ^= 1; + assert_eq!(handles.decode(&forged), Err(HandleError::Bad)); + assert_eq!(handles.decode(&first[..24]), Err(HandleError::Bad)); + assert_eq!(Handles::new(42).decode(&first), Err(HandleError::Stale)); +} diff --git a/engine/crates/layerfs-nfs/tests/refresh.rs b/engine/crates/layerfs-nfs/tests/refresh.rs new file mode 100644 index 0000000000..7ff99c4d96 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/refresh.rs @@ -0,0 +1,117 @@ +//! Cache refresh after an install (B): the inodes the daemon names are +//! marked and their paths returned; a mode-only SETATTR on a marked inode is +//! answered with its current attributes and changes nothing, once; every +//! other SETATTR still changes the inode; the Commit holds only real changes. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, Set}; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-r{}-{label}", std::process::id())) +} + +#[test] +fn a_refresh_chmod_changes_nothing_and_other_setattrs_still_do() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + fs::create_dir_all(source.join("a/b")).unwrap(); + fs::write(source.join("a/b/file"), b"inner\n").unwrap(); + fs::write(source.join("top.txt"), b"top\n").unwrap(); + for (path, mode) in [("a/b/file", 0o644), ("top.txt", 0o644), ("a", 0o755)] { + fs::set_permissions(source.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (file, file_attr) = client.walk_to(&root, "a/b/file").unwrap(); + let (a, a_attr) = client.lookup(&root, b"a").unwrap(); + let (top, top_attr) = client.lookup(&root, b"top.txt").unwrap(); + let root_id = client.getattr(&root).unwrap().fileid; + // Paths of the serials the client knows, the root's empty; an unknown serial is left out. + let paths = server.refresh([ + file_attr.fileid, + a_attr.fileid, + root_id, + u64::MAX - 7, + top_attr.fileid, + ]); + assert_eq!( + paths, + vec![ + PathBuf::from("a/b/file"), + PathBuf::from("a"), + PathBuf::new(), + PathBuf::from("top.txt"), + ] + ); + let mode = |set: Result| { + set.map(|wcc| wcc.after.map(|after| after.mode & 0o7777)) + .map_err(|(status, _)| status) + }; + let chmod = |mode: u32| Set { + mode: Some(mode), + ..Set::default() + }; + // A marked inode: the mode-only SETATTR is answered, nothing changes. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o644)) + ); + assert_eq!( + mode(client.setattr(&a, chmod(0o700), None)), + Ok(Some(0o755)) + ); + assert_eq!(client.getattr(&file).unwrap().mode & 0o7777, 0o644); + // The mark is taken once: the next chmod is real. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o600)) + ); + // A SETATTR that is not mode-only is never taken for a refresh. + let truncate = Set { + mode: Some(0o640), + size: Some(0), + ..Set::default() + }; + let after = client.setattr(&top, truncate, None).unwrap().after.unwrap(); + assert_eq!((after.mode & 0o7777, after.size), (0o640, 0)); + // The Commit holds the real changes only. + server.close_files(); + assert!(rig.commit().history.is_some()); + let view = rig.view([71; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let at = |reader: &mut Client, path: &str| reader.walk_to(&view_root, path).unwrap().1; + assert_eq!(at(&mut reader, "a/b/file").mode & 0o7777, 0o600); + assert_eq!(at(&mut reader, "a").mode & 0o7777, 0o755); + let top_view = at(&mut reader, "top.txt"); + assert_eq!((top_view.mode & 0o7777, top_view.size), (0o640, 0)); + drop(reader); + let _released = viewer.stop(); + rig.close_view(view); + drop(client); + let served = server.stop(); + let (_, outstanding) = rig.stop(served); + assert_eq!(outstanding, 0); + for path in [&source, &work] { + drop(fs::remove_dir_all(Path::new(path))); + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/client.rs b/engine/crates/layerfs-nfs/tests/support/client.rs new file mode 100644 index 0000000000..d98b8f6ff0 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/client.rs @@ -0,0 +1,608 @@ +//! A minimal NFSv3/MOUNT client over the export's public XDR and record +//! framing: what a kernel client sends, one call at a time. +use layerfs_nfs::{ + rpc::{read_record, write_record}, + xdr::{Reader, Writer}, +}; +use std::{ + net::TcpStream, + sync::{Arc, Condvar, Mutex}, + time::Duration, +}; + +/// Calls sent and replies fully received by clients sharing it, so a test's +/// port wrapper can tell whether a reply was already on the wire. +#[derive(Default)] +pub struct Wire { + pub counts: Mutex<(u64, u64)>, + pub changed: Condvar, +} + +pub const NFS: u32 = 100_003; +pub const MOUNT: u32 = 100_005; +pub const OK: u32 = 0; +pub const NOENT: u32 = 2; +pub const ROFS: u32 = 30; +pub const STALE: u32 = 70; +pub const BADHANDLE: u32 = 10001; +pub const BAD_COOKIE: u32 = 10003; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Attr { + pub kind: u32, + pub mode: u32, + pub nlink: u32, + pub uid: u32, + pub gid: u32, + pub size: u64, + pub fileid: u64, + pub mtime: (u32, u32), +} +fn attr(reader: &mut Reader<'_>) -> Attr { + let kind = reader.u32().unwrap(); + let mode = reader.u32().unwrap(); + let nlink = reader.u32().unwrap(); + let uid = reader.u32().unwrap(); + let gid = reader.u32().unwrap(); + let size = reader.u64().unwrap(); + let _used = reader.u64().unwrap(); + let _rdev = (reader.u32().unwrap(), reader.u32().unwrap()); + let _fsid = reader.u64().unwrap(); + let fileid = reader.u64().unwrap(); + let _atime = (reader.u32().unwrap(), reader.u32().unwrap()); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let _ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + Attr { + kind, + mode, + nlink, + uid, + gid, + size, + fileid, + mtime, + } +} +fn post_op(reader: &mut Reader<'_>) -> Option { + reader.bool().unwrap().then(|| attr(reader)) +} +/// A LOOKUP answer: its status, the object's handle and attributes when +/// found, and the directory's attributes. +pub type Lookup = (u32, Option<(Vec, Attr)>, Option); +/// One READDIR/READDIRPLUS entry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Entry { + pub name: Vec, + pub fileid: u64, + pub cookie: u64, +} + +pub struct Client { + stream: TcpStream, + xid: u32, + uid: u32, + gid: u32, + pub calls: u64, + pub wire: Option>, +} +impl Client { + pub fn connect(address: std::net::SocketAddr, uid: u32, gid: u32) -> Self { + let stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + Self { + stream, + xid: 1, + uid, + gid, + calls: 0, + wire: None, + } + } + /// One call; the accept status and the results after it. + pub fn call(&mut self, program: u32, procedure: u32, arguments: &[u8]) -> (u32, Vec) { + self.xid += 1; + self.calls += 1; + let mut credential = Writer::new(); + credential + .u32(0) + .opaque(b"test") + .u32(self.uid) + .u32(self.gid) + .u32(0); + let mut call = Writer::new(); + call.u32(self.xid) + .u32(0) + .u32(2) + .u32(program) + .u32(3) + .u32(procedure); + call.u32(1).opaque(credential.as_bytes()); + call.u32(0).u32(0); + call.fixed(arguments); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().0 += 1; + } + write_record(&mut self.stream, call.as_bytes()).unwrap(); + let record = read_record(&mut self.stream).unwrap().unwrap(); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().1 += 1; + wire.changed.notify_all(); + } + let mut reader = Reader::new(&record); + assert_eq!(reader.u32().unwrap(), self.xid); + assert_eq!(reader.u32().unwrap(), 1, "a reply"); + assert_eq!(reader.u32().unwrap(), 0, "accepted"); + let _flavor = reader.u32().unwrap(); + let _verifier = reader.opaque(400).unwrap(); + let status = reader.u32().unwrap(); + let rest = record[record.len() - reader.remaining()..].to_vec(); + (status, rest) + } + /// One NFS procedure that the RPC layer accepted: its nfsstat3 and results. + pub fn nfs(&mut self, procedure: u32, arguments: &Writer) -> (u32, Vec) { + let (accept, rest) = self.call(NFS, procedure, arguments.as_bytes()); + assert_eq!(accept, 0, "procedure {procedure} accepted"); + let mut reader = Reader::new(&rest); + let status = reader.u32().unwrap(); + (status, rest[4..].to_vec()) + } + pub fn mnt(&mut self, path: &str) -> Vec { + let mut arguments = Writer::new(); + arguments.opaque(path.as_bytes()); + let (accept, rest) = self.call(MOUNT, 1, arguments.as_bytes()); + assert_eq!(accept, 0); + let mut reader = Reader::new(&rest); + assert_eq!(reader.u32().unwrap(), 0, "MNT3_OK"); + reader.opaque(64).unwrap().to_vec() + } + pub fn getattr(&mut self, handle: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(1, &arguments); + if status != OK { + return Err(status); + } + Ok(attr(&mut Reader::new(&rest))) + } + pub fn lookup(&mut self, directory: &[u8], name: &[u8]) -> Result<(Vec, Attr), u32> { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let handle = reader.opaque(64).unwrap().to_vec(); + let attributes = post_op(&mut reader).expect("LOOKUP returns object attributes"); + Ok((handle, attributes)) + } + /// LOOKUP with both halves of its answer: the status, the object's + /// handle and attributes when found, and the directory's attributes. + pub fn lookup_with_directory(&mut self, directory: &[u8], name: &[u8]) -> Lookup { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + let mut reader = Reader::new(&rest); + let object = (status == OK).then(|| { + let handle = reader.opaque(64).unwrap().to_vec(); + ( + handle, + post_op(&mut reader).expect("LOOKUP returns object attributes"), + ) + }); + let parent = post_op(&mut reader); + (status, object, parent) + } + pub fn access(&mut self, handle: &[u8], requested: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle).u32(requested); + let (status, rest) = self.nfs(4, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.u32().unwrap()) + } + pub fn readlink(&mut self, handle: &[u8]) -> Result, u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(5, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.opaque(4096).unwrap().to_vec()) + } + /// One READ: data and EOF. + pub fn read(&mut self, handle: &[u8], offset: u64, count: u32) -> Result<(Vec, bool), u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(offset).u32(count); + let (status, rest) = self.nfs(6, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let count = reader.u32().unwrap(); + let eof = reader.bool().unwrap(); + let data = reader.opaque(1 << 20).unwrap().to_vec(); + assert_eq!(count as usize, data.len()); + Ok((data, eof)) + } + /// READs until EOF in windows of `count` bytes. + pub fn read_all(&mut self, handle: &[u8], count: u32) -> Vec { + let mut bytes = Vec::new(); + loop { + let (data, eof) = self.read(handle, bytes.len() as u64, count).unwrap(); + bytes.extend_from_slice(&data); + if eof { + return bytes; + } + assert!(!data.is_empty(), "a short non-EOF read makes progress"); + } + } + /// One READDIR (or READDIRPLUS): entries, verifier, EOF. + pub fn readdir( + &mut self, + handle: &[u8], + cookie: u64, + verifier: u64, + budget: u32, + plus: bool, + ) -> Result<(Vec, u64, bool), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(cookie) + .u64(verifier) + .u32(budget); + if plus { + arguments.u32(budget); + } + let (status, rest) = self.nfs(if plus { 17 } else { 16 }, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let verifier = reader.u64().unwrap(); + let mut entries = Vec::new(); + while reader.bool().unwrap() { + let fileid = reader.u64().unwrap(); + let name = reader.opaque(4096).unwrap().to_vec(); + let cookie = reader.u64().unwrap(); + if plus { + assert!(post_op(&mut reader).is_none()); + assert!(!reader.bool().unwrap(), "no handle in READDIRPLUS"); + } + entries.push(Entry { + name, + fileid, + cookie, + }); + } + let eof = reader.bool().unwrap(); + Ok((entries, verifier, eof)) + } + /// A whole enumeration as a kernel client runs it, and the calls it took. + pub fn list(&mut self, handle: &[u8], budget: u32, plus: bool) -> (Vec, u64) { + let (mut cookie, mut verifier, mut calls) = (0, 0, 0); + let mut all = Vec::new(); + loop { + let (entries, next, eof) = self + .readdir(handle, cookie, verifier, budget, plus) + .unwrap(); + calls += 1; + verifier = next; + if let Some(last) = entries.last() { + cookie = last.cookie; + } + all.extend(entries); + if eof { + return (all, calls); + } + } + } +} + +pub const PERM: u32 = 1; +pub const EXIST: u32 = 17; +pub const NOTDIR: u32 = 20; +pub const ISDIR: u32 = 21; +pub const INVAL: u32 = 22; +pub const NAMETOOLONG: u32 = 63; +pub const NOTEMPTY: u32 = 66; +pub const ACCES: u32 = 13; +pub const NOT_SYNC: u32 = 10002; +pub const NOTSUPP: u32 = 10004; +pub const JUKEBOX: u32 = 10008; + +/// `wcc_attr`: size, mtime and ctime before a change. +pub type Before = (u64, (u32, u32), (u32, u32)); +/// `wcc_data`: the pre-op (size, mtime, ctime) and post-op attributes. +#[derive(Clone, Debug, Default)] +pub struct Wcc { + pub before: Option, + pub after: Option, +} +pub fn wcc(reader: &mut Reader<'_>) -> Wcc { + let before = reader.bool().unwrap().then(|| { + let size = reader.u64().unwrap(); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + (size, mtime, ctime) + }); + Wcc { + before, + after: post_op(reader), + } +} +/// A `sattr3` to send. +#[derive(Clone, Copy, Debug, Default)] +pub struct Set { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + /// None: don't change; Some(None): server time; Some(Some(t)): client time. + pub mtime: Option>, +} +pub fn sattr(out: &mut Writer, set: Set) { + let field = |out: &mut Writer, value: Option| match value { + Some(value) => { + out.bool(true).u32(value); + } + None => { + out.bool(false); + } + }; + field(out, set.mode); + field(out, set.uid); + field(out, set.gid); + match set.size { + Some(size) => { + out.bool(true).u64(size); + } + None => { + out.bool(false); + } + } + out.u32(0); // atime: don't change + match set.mtime { + None => { + out.u32(0); + } + Some(None) => { + out.u32(1); + } + Some(Some((seconds, nanoseconds))) => { + out.u32(2).u32(seconds).u32(nanoseconds); + } + } +} +/// The results of CREATE, MKDIR and SYMLINK. +#[derive(Clone, Debug)] +pub struct Made { + pub handle: Vec, + pub attr: Attr, + pub directory: Wcc, +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Set), + Guarded(Set), + Exclusive([u8; 8]), +} +impl Client { + fn made(&mut self, procedure: u32, arguments: &Writer) -> Result { + let (status, rest) = self.nfs(procedure, arguments); + let mut reader = Reader::new(&rest); + if status != OK { + return Err((status, wcc(&mut reader))); + } + assert!(reader.bool().unwrap(), "a created handle"); + let handle = reader.opaque(64).unwrap().to_vec(); + let attr = post_op(&mut reader).expect("created attributes"); + let directory = wcc(&mut reader); + Ok(Made { + handle, + attr, + directory, + }) + } + pub fn create(&mut self, directory: &[u8], name: &[u8], how: How) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + match how { + How::Unchecked(set) => { + arguments.u32(0); + sattr(&mut arguments, set); + } + How::Guarded(set) => { + arguments.u32(1); + sattr(&mut arguments, set); + } + How::Exclusive(verifier) => { + arguments.u32(2).fixed(&verifier); + } + } + self.made(8, &arguments).map_err(|(status, _)| status) + } + /// CREATE UNCHECKED with a mode, the common case. + pub fn touch(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.create(directory, name, How::Unchecked(set)) + } + pub fn mkdir(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + sattr(&mut arguments, set); + self.made(9, &arguments).map_err(|(status, _)| status) + } + pub fn symlink(&mut self, directory: &[u8], name: &[u8], target: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + sattr(&mut arguments, Set::default()); + arguments.opaque(target); + self.made(10, &arguments).map_err(|(status, _)| status) + } + pub fn mknod_fifo(&mut self, directory: &[u8], name: &[u8]) -> u32 { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name).u32(7); + sattr(&mut arguments, Set::default()); + self.nfs(11, &arguments).0 + } + /// SETATTR, optionally guarded by a ctime. + pub fn setattr( + &mut self, + handle: &[u8], + set: Set, + guard: Option<(u32, u32)>, + ) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + sattr(&mut arguments, set); + match guard { + Some((seconds, nanoseconds)) => { + arguments.bool(true).u32(seconds).u32(nanoseconds); + } + None => { + arguments.bool(false); + } + } + let (status, rest) = self.nfs(2, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err((status, found)) + } + } + /// WRITE: (count, committed, verifier, file wcc). + pub fn write( + &mut self, + handle: &[u8], + offset: u64, + data: &[u8], + ) -> Result<(u32, u32, [u8; 8], Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(offset) + .u32(data.len() as u32) + .u32(0) + .opaque(data); + let (status, rest) = self.nfs(7, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let file = wcc(&mut reader); + let count = reader.u32().unwrap(); + let committed = reader.u32().unwrap(); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok((count, committed, verifier, file)) + } + /// WRITE of everything, in pieces of `piece` bytes. + pub fn write_all(&mut self, handle: &[u8], offset: u64, data: &[u8], piece: usize) { + let mut at = 0; + while at < data.len() { + let end = (at + piece).min(data.len()); + let (count, _, _, _) = self + .write(handle, offset + at as u64, &data[at..end]) + .unwrap(); + assert!(count > 0); + at += count as usize; + } + } + pub fn commit(&mut self, handle: &[u8]) -> Result<[u8; 8], u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(0).u32(0); + let (status, rest) = self.nfs(21, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + wcc(&mut reader); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok(verifier) + } + /// REMOVE (`directory` false) or RMDIR. + pub fn unlink(&mut self, directory: &[u8], name: &[u8], rmdir: bool) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(if rmdir { 13 } else { 12 }, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn remove(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, false) + } + pub fn rmdir(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, true) + } + pub fn rename( + &mut self, + from: &[u8], + name: &[u8], + to: &[u8], + new_name: &[u8], + ) -> Result<(Wcc, Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(from) + .opaque(name) + .opaque(to) + .opaque(new_name); + let (status, rest) = self.nfs(14, &arguments); + let mut reader = Reader::new(&rest); + let found = (wcc(&mut reader), wcc(&mut reader)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn link(&mut self, file: &[u8], directory: &[u8], name: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(file).opaque(directory).opaque(name); + let (status, rest) = self.nfs(15, &arguments); + let mut reader = Reader::new(&rest); + let object = post_op(&mut reader); + wcc(&mut reader); + if status == OK { + Ok(object.expect("LINK returns the file's attributes")) + } else { + Err(status) + } + } + /// A path of names below `root`, looked up one by one. + pub fn walk_to(&mut self, root: &[u8], path: &str) -> Result<(Vec, Attr), u32> { + let mut handle = root.to_vec(); + let mut attr = self.getattr(root)?; + for name in path.split('/').filter(|name| !name.is_empty()) { + let (next, next_attr) = self.lookup(&handle, name.as_bytes())?; + handle = next; + attr = next_attr; + } + Ok((handle, attr)) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/rig.rs b/engine/crates/layerfs-nfs/tests/support/rig.rs new file mode 100644 index 0000000000..3c7b5ff4d1 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/rig.rs @@ -0,0 +1,295 @@ +//! A committed Branch from a host tree, bound to one Workspace and one engine +//! kernel mount, with no kernel connection: the NFS export is the only client. +use layerfs_daemon::{ + bootstrap::open_store, + store::{BindRequest, BoundWorkspace, Store}, + Mounted, Owner, OwnerClient, OwnerConfig, +}; +use layerfs_history::{ + BranchId, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryName, + LayerStackId, WorkspaceId, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{KernelMount, ProfileConfig}; +use layerfs_persistence::{ + Handles, PersistenceConfig, SqliteAcquisitionSchema, SqlitePersistenceProfile, +}; +use layerfs_project::{init, InitRequest}; +use layerfs_storage::{Storage, StoragePolicy}; +use layerfs_telemetry::timer::Timing; +use std::{ + net::TcpListener, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; + +const BINDING: &[u8] = b"layerfs-nfs-rig"; +const CURSOR: [u8; 32] = [71; 32]; +const CACHE: usize = 32 * 1024 * 1024; + +pub struct Rig { + pub directory: PathBuf, + store: Arc, + owner: Owner, + client: OwnerClient, + pub bound: Arc, + pub mount: KernelMount, + pub branch: BranchId, + /// Paths below the root that Init acquired. + pub entries: u64, + /// The reference bound exports of this rig are configured with. + pub lookups_held: usize, +} +impl Rig { + /// Initializes a Store from `source`, publishes Branch `main` at that + /// root, binds one Workspace and creates one engine kernel mount. + /// `directory` must not exist; everything the rig writes goes there. + pub fn new(source: &Path, directory: &Path, timeout: Duration) -> Self { + std::fs::create_dir_all(directory).unwrap(); + let sealed = directory.join("sealed.sqlite"); + let config = PersistenceConfig::sqlite(&sealed) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); + let handles = Handles::create( + config.clone(), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: CURSOR, + incarnation: 1, + }, + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let initialized = Timing::disabled("nfs.rig.init", |timing| { + init( + &storage, + &handles.history, + InitRequest { + source, + acquisition: &handles.acquisition, + stack: LayerStackId::from_authority([72; 16]), + name: HistoryName::new("nfs").unwrap(), + scope_seed: [73; 32], + deadline: Instant::now() + timeout, + }, + timing, + ) + }) + .0 + .unwrap(); + let branch = BranchId::from_authority([74; 16]); + handles + .history + .fork(&ForkRequest { + stack: initialized.stack.id, + branch, + name: HistoryName::new("main").unwrap(), + source: ForkSource::Layer(initialized.stack.head_layer), + }) + .unwrap(); + drop(storage); + let sealed = handles.seal().unwrap(); + let store = open_store( + PersistenceConfig { + path: sealed.path.clone(), + ..config + }, + BINDING, + CURSOR, + 4, + CACHE, + Default::default(), + ) + .unwrap(); + let owner = Owner::start( + &directory.join("overlay"), + ProfileConfig::default(), + OwnerConfig::default(), + ) + .unwrap(); + let client = owner.client(); + let bound = Arc::new( + store + .bind( + client.clone(), + BindRequest { + branch, + workspace: WorkspaceId::from_authority([75; 32]).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = client.kernel_mount(bound.route(), root).unwrap(); + Self { + directory: directory.to_owned(), + store, + owner, + client, + bound, + mount, + branch, + entries: initialized.entries, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } + /// Serves this rig's engine mount read-only on `listener`. + pub fn serve(&self, listener: TcpListener, uid: u32, gid: u32) -> Server { + self.export(listener, uid, gid, true) + } + /// Serves this rig's engine mount on `listener`, writable or not. + pub fn export(&self, listener: TcpListener, uid: u32, gid: u32, read_only: bool) -> Server { + self.export_services( + listener, + Arc::new(Mounted(self.bound.clone())), + uid, + gid, + read_only, + ) + } + /// Serves this rig's engine mount over `services` (a wrapper of the + /// rig's own ports, for example). + pub fn export_services( + &self, + listener: TcpListener, + services: Arc, + uid: u32, + gid: u32, + read_only: bool, + ) -> Server { + Server::start( + listener, + services, + self.mount, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// A daemon control Service over this rig's Store and overlay owner whose + /// Attach serves NFS (the native route's mount entry). + pub fn service( + &self, + export: Arc, + ) -> Arc { + Arc::new(layerfs_daemon::control::Service::with_export( + self.store.clone(), + &self.owner, + export, + )) + } + /// The product Commit of everything the Workspace published locally. + pub fn commit(&self) -> layerfs_daemon::store::CommitSuccess { + self.bound + .commit_captured(&layerfs_daemon::store::CommitOptions::default()) + .unwrap_or_else(|failure| panic!("Commit: {:?}", failure.error)) + } + /// A second Workspace bound to Branch `main` as it is now, with its own + /// engine mount: an independent view of what was committed. + pub fn view(&self, workspace: [u8; 32]) -> (Arc, KernelMount) { + let bound = Arc::new( + self.store + .bind( + self.client.clone(), + BindRequest { + branch: self.branch, + workspace: WorkspaceId::from_authority(workspace).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = self.client.kernel_mount(bound.route(), root).unwrap(); + (bound, mount) + } + /// Serves a view read-only. + pub fn serve_view( + &self, + listener: TcpListener, + view: &(Arc, KernelMount), + uid: u32, + gid: u32, + ) -> Server { + Server::start( + listener, + Arc::new(Mounted(view.0.clone())), + view.1, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only: true, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// Revokes and closes a view. + pub fn close_view(&self, view: (Arc, KernelMount)) { + self.client.kernel_revoke(view.0.route(), view.1).unwrap(); + view.0.close().unwrap(); + } + /// A second engine mount of the same Workspace, as a restarted export + /// would create: handles of the first are stale under it. + pub fn remount(&mut self) { + self.client + .kernel_revoke(self.bound.route(), self.mount) + .unwrap(); + self.mount = self + .client + .kernel_mount(self.bound.route(), self.mount.root_serial()) + .unwrap(); + } + /// Revokes the engine mount, closes the Workspace and stops the owner. + /// Returns the Store readers still open or quarantined afterwards. + pub fn stop(self, served: Released) -> (Released, u64) { + let Self { + directory, + store, + owner, + client, + bound, + mount, + .. + } = self; + client.kernel_revoke(bound.route(), mount).unwrap(); + bound.close().unwrap(); + drop(bound); + // A closed Workspace keeps no Store reader and none was quarantined; + // the Store's own control reader stays open. + let work = store.read_work(); + assert!(!work.poisoned && work.opening_failures == 0, "{work:?}"); + let outstanding = work.readers.saturating_sub(1) + work.quarantined; + owner.stop().unwrap(); + drop(store); + std::fs::remove_dir_all(directory).unwrap(); + (served, outstanding as u64) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/spy.rs b/engine/crates/layerfs-nfs/tests/support/spy.rs new file mode 100644 index 0000000000..0f329a20f3 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/spy.rs @@ -0,0 +1,199 @@ +//! A wrapper of the rig's request ports that watches the publication +//! ticket contract from outside the export: every `reply_attempted` waits +//! until the test's client has received the reply of the call that made the +//! publication, +//! and records whether that happened (a release before the reply reached the +//! wire would wait forever on a synchronous client, so it times out and is +//! counted as a violation). It can also refuse serial reservations, as a +//! contended allocator does. +use super::client::Wire; +use layerfs_fuse::ports::{ + MountServices, RequestServices, ServiceError, ServiceFuture, ServiceReply, +}; +use layerfs_overlay::{KernelCookieOffer, KernelMount, Publication}; +use layerfs_workspace::{ + BaseView, KernelDirectoryWindow, KernelMutationOutcome, KernelReadOperation, KernelReadOutcome, + KernelVisitRequest, KernelWindow, VisitFacts, WriteData, +}; +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +/// What the spy saw. +#[derive(Default)] +pub struct Watch { + pub wire: Arc, + /// Releases made after the reply was received, and before it. + pub released_after: AtomicU64, + pub released_before: AtomicU64, + /// Publishing visits (`Applied` outcomes) and reservations refused. + pub published: AtomicU64, + /// Reservations still to refuse. + pub refuse_serials: AtomicU64, + pub refused: AtomicU64, + pub log: Mutex>, + /// Each publication and the call count when it was made: the client was + /// then waiting for exactly that call's reply. + pub owed: Mutex>, +} +pub struct Spy { + pub inner: Arc, + pub watch: Arc, +} +impl MountServices for Spy { + fn request( + &self, + fence: &layerfs_fuse::ports::Fence, + ) -> Result, ServiceError> { + Ok(Arc::new(Request { + inner: self.inner.request(fence)?, + watch: self.watch.clone(), + })) + } +} +struct Request { + inner: Arc, + watch: Arc, +} +impl RequestServices for Request { + fn directory_visit( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + offset: u64, + after: Option>, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .directory_visit(mount, serial, handle, offset, after) + } + fn publish_cookies( + &self, + offer: KernelCookieOffer, + names: Vec>, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.publish_cookies(offer, names) + } + fn close_directory( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_directory(mount, serial, handle) + } + fn read_visit( + &self, + mount: KernelMount, + serial: u64, + handle: Option, + offset: u64, + length: u32, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner.read_visit(mount, serial, handle, offset, length) + } + fn close_file( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_file(mount, serial, handle) + } + fn provider_failure(&self) -> Result, ServiceError> { + self.inner.provider_failure() + } + fn failed_base_read(&self, step: ServiceError) -> ServiceError { + self.inner.failed_base_read(step) + } + fn observe_visit( + &self, + mount: KernelMount, + request: u64, + serial: u64, + handle: Option, + operation: KernelReadOperation, + facts: Arc, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .observe_visit(mount, request, serial, handle, operation, facts) + } + fn base(&self) -> ServiceFuture<'_, BaseView> { + self.inner.base() + } + fn mutate_visit( + &self, + request: KernelVisitRequest, + ) -> ServiceFuture<'_, ServiceReply>> { + let watch = self.watch.clone(); + let future = self.inner.mutate_visit(request); + Box::pin(async move { + let reply = future.await?; + if let Ok(layerfs_workspace::JobOutcome::Applied { publication, .. }) = + &reply.get().result + { + watch.published.fetch_add(1, Ordering::AcqRel); + let sent = watch.wire.counts.lock().unwrap().0; + watch.owed.lock().unwrap().push((*publication, sent)); + } + Ok(reply) + }) + } + fn recognize(&self, serial: u64, offset: u64, data: WriteData) -> ServiceFuture<'_, WriteData> { + self.inner.recognize(serial, offset, data) + } + fn forget( + &self, + mount: KernelMount, + serial: u64, + count: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.forget(mount, serial, count) + } + fn reserve_serial(&self) -> Result, ServiceError> { + let refuse = self + .watch + .refuse_serials + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |left| { + left.checked_sub(1) + }) + .is_ok(); + if refuse { + self.watch.refused.fetch_add(1, Ordering::AcqRel); + return Ok(None); + } + self.inner.reserve_serial() + } + fn reply_attempted(&self, publication: Publication) -> ServiceFuture<'_, ServiceReply<()>> { + let wire = &self.watch.wire; + let owed = { + let mut owed = self.watch.owed.lock().unwrap(); + let at = owed.iter().position(|(made, _)| *made == publication); + at.map_or(u64::MAX, |at| owed.remove(at).1) + }; + let counts = wire.counts.lock().unwrap(); + // The reply of the call that published this ticket must reach the + // client before the release, whatever the client sends next. + let (counts, timeout) = wire + .changed + .wait_timeout_while(counts, Duration::from_secs(5), |(_, received)| { + *received < owed + }) + .unwrap(); + let (sent, received) = *counts; + drop(counts); + if timeout.timed_out() { + self.watch.released_before.fetch_add(1, Ordering::AcqRel); + self.watch.log.lock().unwrap().push(format!( + "released before reply: sent {sent} received {received}" + )); + } else { + self.watch.released_after.fetch_add(1, Ordering::AcqRel); + } + self.inner.reply_attempted(publication) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/tree.rs b/engine/crates/layerfs-nfs/tests/support/tree.rs new file mode 100644 index 0000000000..e1bb648bc8 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/tree.rs @@ -0,0 +1,136 @@ +//! A host tree as an oracle, and a protocol walk that checks an export +//! against it: every path, kind, mode, byte, link target, listing, `..`, +//! directory link count and the identity behind hard links. +use super::client::Client; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +/// What the host tree holds, read before the export runs. +#[derive(Debug)] +pub enum Node { + File { mode: u32, bytes: Vec, ino: u64 }, + Directory { mode: u32, names: BTreeSet> }, + Link { target: Vec }, +} +pub fn oracle(root: &Path) -> BTreeMap { + use std::os::unix::ffi::OsStrExt; + let mut nodes = BTreeMap::new(); + let mut pending = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + let path = root.join(&relative); + let meta = fs::symlink_metadata(&path).unwrap(); + let node = if meta.file_type().is_symlink() { + Node::Link { + target: fs::read_link(&path) + .unwrap() + .as_os_str() + .as_bytes() + .to_vec(), + } + } else if meta.is_dir() { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(&path).unwrap() { + let name = entry.unwrap().file_name(); + pending.push(relative.join(&name)); + names.insert(name.as_bytes().to_vec()); + } + Node::Directory { + mode: meta.mode() & 0o7777, + names, + } + } else { + Node::File { + mode: meta.mode() & 0o7777, + bytes: fs::read(&path).unwrap(), + ino: meta.ino(), + } + }; + nodes.insert(relative, node); + } + nodes +} + +/// Walks the export from `handle` and checks every node against the oracle. +/// Returns (paths, regular bytes read) and the fileid of every host inode. +pub fn walk( + client: &mut Client, + oracle: &BTreeMap, + budget: u32, + owner: (u32, u32), +) -> (usize, usize, BTreeMap>) { + use std::os::unix::ffi::OsStrExt; + let root = client.mnt("/"); + let root_attr = client.getattr(&root).unwrap(); + let mut identities: BTreeMap> = BTreeMap::new(); + let (mut paths, mut bytes) = (0, 0); + let mut pending = vec![(PathBuf::new(), root, root_attr.clone(), root_attr.fileid)]; + while let Some((relative, handle, attr, parent)) = pending.pop() { + paths += 1; + assert_eq!((attr.uid, attr.gid), owner, "{relative:?}"); + match &oracle[&relative] { + Node::File { + mode, + bytes: expected, + ino, + } => { + assert_eq!(attr.kind, 1, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + assert_eq!(attr.size, expected.len() as u64, "{relative:?} size"); + let actual = client.read_all(&handle, 128 * 1024); + assert_eq!(&actual, expected, "{relative:?} bytes"); + bytes += actual.len(); + identities.entry(*ino).or_default().insert(attr.fileid); + } + Node::Link { target } => { + assert_eq!(attr.kind, 5, "{relative:?}"); + assert_eq!(&client.readlink(&handle).unwrap(), target, "{relative:?}"); + } + Node::Directory { mode, names } => { + assert_eq!(attr.kind, 2, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + let (entries, _) = client.list(&handle, budget, false); + let mut listed = BTreeSet::new(); + for entry in &entries { + assert!(listed.insert(entry.name.clone()), "{relative:?} duplicate"); + } + let dot = entries.iter().find(|entry| entry.name == b".").unwrap(); + let dotdot = entries.iter().find(|entry| entry.name == b".."); + assert_eq!(dot.fileid, attr.fileid, "{relative:?} ."); + if let Some(dotdot) = dotdot { + assert_eq!(dotdot.fileid, parent, "{relative:?} .."); + } + listed.remove(&b"."[..]); + listed.remove(&b".."[..]); + assert_eq!(&listed, names, "{relative:?} complete listing"); + let (up, up_attr) = client.lookup(&handle, b"..").unwrap(); + assert_eq!(up_attr.fileid, parent, "{relative:?} LOOKUP .."); + drop(up); + // Subdirectory link counts: two plus child directories. + let subdirectories = names + .iter() + .filter(|name| { + matches!( + oracle.get(&relative.join(std::ffi::OsStr::from_bytes(name))), + Some(Node::Directory { .. }) + ) + }) + .count(); + assert_eq!( + attr.nlink as usize, + subdirectories + 2, + "{relative:?} nlink" + ); + for name in names { + let (child, child_attr) = client.lookup(&handle, name).unwrap(); + let child_path = relative.join(std::ffi::OsStr::from_bytes(name)); + pending.push((child_path, child, child_attr, attr.fileid)); + } + } + } + } + (paths, bytes, identities) +} diff --git a/engine/crates/layerfs-nfs/tests/write.rs b/engine/crates/layerfs-nfs/tests/write.rs new file mode 100644 index 0000000000..ba11401ff3 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/write.rs @@ -0,0 +1,687 @@ +//! A writable Workspace served over loopback NFSv3: every changing +//! procedure and its error statuses, the publication-ticket contract (each +//! ticket released only after its reply was received, contended serial +//! reservations), the export's own handling of client silly renames and +//! AppleDouble names, and a product Commit whose tree equals what the same +//! operations made of a host directory. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/spy.rs"] +mod spy; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{ + Client, How, Set, ACCES, EXIST, INVAL, ISDIR, JUKEBOX, NAMETOOLONG, NOENT, NOTDIR, NOTEMPTY, + NOTSUPP, NOT_SYNC, OK, PERM, STALE, +}; +use layerfs_daemon::Mounted; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; +const FILE_SYNC: u32 = 2; +const LINUX_SILLY: &[u8] = b".nfs00000000000012ab00000001"; +const MAC_SILLY: &[u8] = b".nfs.20051025.b896"; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-w{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested")).unwrap(); + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep.txt"), b"deep\n").unwrap(); + fs::write(root.join("docs/guide.md"), b"guide\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); +} +fn pattern(seed: u32, length: usize) -> Vec { + (0..length) + .map(|n| (n as u32).wrapping_mul(31).wrapping_add(seed * 97) as u8) + .collect() +} + +#[test] +fn every_changing_procedure_and_its_statuses() { + let base = temp("procedures"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + + // CREATE: UNCHECKED, GUARDED over an existing name, UNCHECKED again. + let made = c.touch(&root, b"new.txt", 0o640).unwrap(); + assert_eq!( + (made.attr.kind, made.attr.mode, made.attr.size), + (1, 0o640, 0) + ); + assert_eq!((made.attr.uid, made.attr.gid), (UID, GID)); + assert!(made.directory.before.is_some() && made.directory.after.is_some()); + let guarded = How::Guarded(Set::default()); + assert_eq!(c.create(&root, b"new.txt", guarded).map(|_| ()), Err(EXIST)); + let again = c.touch(&root, b"new.txt", 0o600).unwrap(); + assert_eq!(again.attr.fileid, made.attr.fileid); + assert_eq!( + again.attr.mode, 0o640, + "UNCHECKED keeps an existing file's mode" + ); + let file = made.handle.clone(); + + // WRITE: FILE_SYNC with this export's verifier; a hole reads as zeros; + // a call larger than one engine window is split and fully written. + let (count, committed, verifier, wcc) = c.write(&file, 0, b"hello").unwrap(); + assert_eq!( + (count, committed, verifier), + (5, FILE_SYNC, server.verifier()) + ); + assert_eq!(wcc.after.unwrap().size, 5); + let big = pattern(1, 300_000); + let (count, _, _, wcc) = c.write(&file, 10, &big).unwrap(); + assert_eq!(count as usize, big.len()); + assert_eq!(wcc.after.unwrap().size, 300_010); + let mut expected = b"hello\0\0\0\0\0".to_vec(); + expected.extend_from_slice(&big); + assert_eq!(c.read_all(&file, 128 * 1024), expected); + assert_eq!(c.commit(&file), Ok(server.verifier())); + + // SETATTR: truncate, mode, client mtime, owner no-op and refusal, guard. + let set = |size, mode| Set { + size, + mode, + ..Set::default() + }; + let after = c + .setattr(&file, set(Some(3), None), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.size, 3); + assert_eq!(c.read_all(&file, 4096), b"hel"); + let after = c + .setattr(&file, set(None, Some(0o600)), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.mode, 0o600); + let stamp = Set { + mtime: Some(Some((1_000_000_000, 5))), + ..Set::default() + }; + let after = c.setattr(&file, stamp, None).unwrap().after.unwrap(); + assert_eq!(after.mtime, (1_000_000_000, 5)); + let same = Set { + uid: Some(UID), + gid: Some(GID), + ..Set::default() + }; + assert!( + c.setattr(&file, same, None).is_ok(), + "chown to the same identity" + ); + let other = Set { + uid: Some(0), + ..Set::default() + }; + assert_eq!( + c.setattr(&file, other, None).map(|_| ()).unwrap_err().0, + PERM + ); + let wrong = c.setattr(&file, set(Some(1), None), Some((7, 7))); + assert_eq!(wrong.map(|_| ()).unwrap_err().0, NOT_SYNC); + assert_eq!( + c.getattr(&file).unwrap().size, + 3, + "a failed guard changes nothing" + ); + let right = c.setattr(&file, set(Some(1), None), Some(after.mtime)); + assert_eq!(right.unwrap().after.unwrap().size, 1); + + // MKDIR, SYMLINK, MKNOD. + let dir = c.mkdir(&root, b"d", 0o750).unwrap(); + assert_eq!( + (dir.attr.kind, dir.attr.mode, dir.attr.nlink), + (2, 0o750, 2) + ); + assert_eq!(c.mkdir(&root, b"d", 0o750).map(|_| ()), Err(EXIST)); + let (_, up) = c.lookup(&dir.handle, b"..").unwrap(); + assert_eq!(up.fileid, c.getattr(&root).unwrap().fileid); + let link = c.symlink(&root, b"s", b"new.txt").unwrap(); + assert_eq!(link.attr.kind, 5); + assert_eq!(c.readlink(&link.handle).unwrap(), b"new.txt"); + let long = vec![b'x'; 4096]; + assert_eq!( + c.symlink(&root, b"long", &long).map(|_| ()), + Err(NAMETOOLONG) + ); + assert_eq!(c.mknod_fifo(&root, b"fifo"), NOTSUPP); + assert_eq!(c.lookup(&root, b"fifo").map(|_| ()), Err(NOENT)); + + // LINK: one more name and link count; directories and taken names refused. + let linked = c.link(&file, &dir.handle, b"alias").unwrap(); + assert_eq!((linked.fileid, linked.nlink), (made.attr.fileid, 2)); + assert_ne!(c.link(&dir.handle, &root, b"dirlink").map(|_| ()), Ok(())); + assert_eq!(c.link(&file, &root, b"README.md").map(|_| ()), Err(EXIST)); + + // RENAME: across directories, of a directory, replacing, into itself. + c.rename(&root, b"new.txt", &dir.handle, b"moved").unwrap(); + assert_eq!(c.lookup(&root, b"new.txt").map(|_| ()), Err(NOENT)); + let (_, moved) = c.lookup(&dir.handle, b"moved").unwrap(); + assert_eq!(moved.fileid, made.attr.fileid); + let sub = c.mkdir(&dir.handle, b"sub", 0o755).unwrap(); + c.rename(&dir.handle, b"sub", &root, b"top").unwrap(); + let (_, up) = c.lookup(&sub.handle, b"..").unwrap(); + assert_eq!( + up.fileid, + c.getattr(&root).unwrap().fileid, + ".. follows a move" + ); + assert_eq!(c.getattr(&dir.handle).unwrap().nlink, 2); + assert_ne!( + c.rename(&root, b"d", &dir.handle, b"inside").map(|_| ()), + Ok(()) + ); + let (readme, readme_attr) = c.lookup(&root, b"README.md").unwrap(); + c.rename(&root, b"run.sh", &root, b"README.md").unwrap(); + assert_eq!(c.getattr(&readme), Err(STALE), "the replaced inode is gone"); + let (_, replaced) = c.lookup(&root, b"README.md").unwrap(); + assert_ne!(replaced.fileid, readme_attr.fileid); + assert_eq!(replaced.mode, 0o755); + let missing = c.rename(&root, b"absent", &root, b"x").map(|_| ()); + assert_eq!(missing, Err(NOENT)); + + // REMOVE and RMDIR. + c.remove(&dir.handle, b"alias").unwrap(); + assert_eq!( + c.getattr(&file).unwrap().nlink, + 1, + "the other name keeps it" + ); + assert_eq!(c.remove(&root, b"absent").map(|_| ()), Err(NOENT)); + assert_eq!(c.remove(&root, b"d").map(|_| ()), Err(ISDIR)); + assert_eq!(c.rmdir(&root, b"d").map(|_| ()), Err(NOTEMPTY)); + assert_eq!(c.rmdir(&root, b"README.md").map(|_| ()), Err(NOTDIR)); + assert_eq!(c.rmdir(&root, b".").map(|_| ()), Err(INVAL)); + c.remove(&dir.handle, b"moved").unwrap(); + assert_eq!(c.getattr(&file), Err(STALE), "its last name is gone"); + c.rmdir(&root, b"d").unwrap(); + assert_eq!(c.getattr(&dir.handle), Err(STALE)); + + // Names the export refuses or the protocol forbids. + // An AppleDouble file is created detached (see the test below); other + // kinds under such a name are refused. + c.touch(&root, b"._new.txt", 0o644).unwrap(); + c.remove(&root, b"._new.txt").unwrap(); + assert_eq!(c.mkdir(&root, b"._d", 0o755).map(|_| ()), Err(ACCES)); + let refused = c.rename(&root, b"s", &root, b"._s").map(|_| ()); + assert_eq!(refused, Err(ACCES)); + assert_eq!(c.touch(&root, b".", 0o644).map(|_| ()), Err(EXIST)); + let long_name = vec![b'n'; 256]; + assert_eq!( + c.touch(&root, &long_name, 0o644).map(|_| ()), + Err(NAMETOOLONG) + ); + + // EXCLUSIVE: a retransmission finds its own file; another verifier does not. + let first = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + let again = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + assert_eq!(first.attr.fileid, again.attr.fileid); + let other = c.create(&root, b"excl", How::Exclusive(*b"another!")); + assert_eq!(other.map(|_| ()), Err(EXIST)); + + // ACCESS on a writable export grants changes by mode. + let (guide, _) = c.walk_to(&root, "docs/guide.md").unwrap(); + assert_eq!(c.access(&guide, 0x3f), Ok(0x01 | 0x04 | 0x08)); + let (docs, _) = c.lookup(&root, b"docs").unwrap(); + assert_eq!(c.access(&docs, 0x3f), Ok(0x01 | 0x02 | 0x04 | 0x08 | 0x10)); + + // Every serial handed out holds one reference: removed ones gave theirs back. + let held = server.held(); + assert_eq!( + ( + held.silly, + held.retained, + held.names_refused, + held.companions, + held.names_detached + ), + (0, 0, 2, 0, 1) + ); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn silly_renames_stay_out_of_the_workspace() { + let base = temp("silly"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + for (silly, name) in [(LINUX_SILLY, &b"open.bin"[..]), (MAC_SILLY, b"open2.bin")] { + let made = c.touch(&root, name, 0o644).unwrap(); + c.write(&made.handle, 0, b"still open").unwrap(); + assert_eq!(c.lookup(&root, silly).map(|_| ()), Err(NOENT)); + c.rename(&root, name, &root, silly).unwrap(); + assert_eq!(c.lookup(&root, name).map(|_| ()), Err(NOENT)); + let (handle, attr) = c.lookup(&root, silly).unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 10); + // The open file stays readable, writable and truncatable. + c.write(&handle, 10, b"!").unwrap(); + assert_eq!(c.read_all(&handle, 4096), b"still open!"); + let truncate = Set { + size: Some(5), + ..Set::default() + }; + assert_eq!( + c.setattr(&handle, truncate, None) + .unwrap() + .after + .unwrap() + .size, + 5 + ); + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| e.name != silly && e.name != name)); + assert_eq!(server.held().silly, 1); + c.remove(&root, silly).unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().silly, 0); + } + // A user's own names that only resemble the patterns are ordinary. + let made = c.touch(&root, b".nfsrc", 0o644).unwrap(); + c.rename(&root, b".nfsrc", &root, b".nfs-old").unwrap(); + assert!(c.lookup(&root, b".nfs-old").is_ok()); + drop(made); + // A silly name still held at Commit is not captured; neither is the + // name it replaced. + let kept = c.touch(&root, b"kept.bin", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + c.rename(&root, b"kept.bin", &root, LINUX_SILLY).unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([76; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b".nfs-old".to_vec())); + assert!(!names + .iter() + .any(|n| n.starts_with(b".nfs0") || n == b"kept.bin")); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + // Release gives the silly inode back. + drop(c); + let released = server.stop(); + assert_eq!((released.silly, released.silly_renamed), (1, 3)); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn apple_double_files_stay_out_of_the_workspace() { + let base = temp("appledouble"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + // What macOS copyfile does for a source carrying extended attributes: + // the data file, then its `._` companion with the attributes. + let data = c.touch(&root, b"copied.rs", 0o644).unwrap(); + c.write(&data.handle, 0, b"fn main() {}").unwrap(); + let made = c.touch(&root, b"._copied.rs", 0o644).unwrap(); + c.write(&made.handle, 0, b"AppleDouble header").unwrap(); + // It reads back under its name for the export's life ... + let (handle, attr) = c.lookup(&root, b"._copied.rs").unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 18); + assert_eq!(c.read_all(&handle, 4096), b"AppleDouble header"); + // ... opening it again truncates it, GUARDED finds it taken ... + let truncate = Set { + size: Some(0), + ..Set::default() + }; + let again = c + .create(&root, b"._copied.rs", How::Unchecked(truncate)) + .unwrap(); + assert_eq!( + (again.handle.clone(), again.attr.size), + (made.handle.clone(), 0) + ); + let guarded = c.create(&root, b"._copied.rs", How::Guarded(Set::default())); + assert_eq!(guarded.map(|_| ()), Err(EXIST)); + // ... READDIR does not list it, it moves only to another AppleDouble name, + // and a real file cannot take such a name. + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| !e.name.starts_with(b"._"))); + c.rename(&root, b"._copied.rs", &root, b"._moved.rs") + .unwrap(); + assert_eq!(c.lookup(&root, b"._copied.rs").map(|_| ()), Err(NOENT)); + let back = c + .rename(&root, b"._moved.rs", &root, b"plain.rs") + .map(|_| ()); + assert_eq!(back, Err(ACCES)); + let onto = c.rename(&root, b"copied.rs", &root, b"._x").map(|_| ()); + assert_eq!(onto, Err(ACCES)); + assert_eq!(server.held().companions, 1); + // REMOVE gives it back. + c.remove(&root, b"._moved.rs").unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().companions, 0); + // One still held at Commit is not captured; the data file is. + let kept = c.touch(&root, b"._kept", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b"copied.rs".to_vec())); + assert!(!names.iter().any(|n| n.starts_with(b"._"))); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + drop(c); + let released = server.stop(); + assert_eq!( + ( + released.silly, + released.companions, + released.names_detached, + released.names_refused + ), + (0, 1, 2, 1) + ); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn tickets_are_released_only_after_the_reply() { + let base = temp("tickets"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let watch = Arc::new(spy::Watch::default()); + let services = Arc::new(spy::Spy { + inner: Arc::new(Mounted(rig.bound.clone())), + watch: watch.clone(), + }); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let server = rig.export_services(listener, services, UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + c.wire = Some(watch.wire.clone()); + let root = c.mnt("/"); + + let made = c.touch(&root, b"f", 0o644).unwrap(); + // 300 000 bytes are three engine windows: three publications, three + // tickets, all released after the one reply. + c.write(&made.handle, 0, &pattern(2, 300_000)).unwrap(); + let truncate = Set { + size: Some(10), + ..Set::default() + }; + c.setattr(&made.handle, truncate, None).unwrap(); + c.mkdir(&root, b"dir", 0o755).unwrap(); + c.rename(&root, b"f", &root, b"g").unwrap(); + c.remove(&root, b"g").unwrap(); + let published = watch.published.load(Ordering::Acquire); + assert_eq!(published, 1 + 3 + 1 + 1 + 1 + 1); + // The last call's tickets are released just after its reply arrived. + let settled = std::time::Instant::now() + Duration::from_secs(5); + while watch.released_after.load(Ordering::Acquire) < published + && std::time::Instant::now() < settled + { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(watch.released_after.load(Ordering::Acquire), published); + assert_eq!( + watch.released_before.load(Ordering::Acquire), + 0, + "{:?}", + watch.log.lock().unwrap() + ); + + // A contended allocator: nothing is reserved and nothing created; the + // client is told to retry later, and the retry succeeds. + watch.refuse_serials.store(1, Ordering::Release); + assert_eq!(c.touch(&root, b"later", 0o644).map(|_| ()), Err(JUKEBOX)); + assert_eq!(watch.refused.load(Ordering::Acquire), 1); + assert_eq!(c.lookup(&root, b"later").map(|_| ()), Err(NOENT)); + assert_eq!(watch.published.load(Ordering::Acquire), published); + assert!(c.touch(&root, b"later", 0o644).is_ok()); + + // No ticket is left pending: the Workspace captures and commits. + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + assert_eq!(server.held().retained, 0); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +/// The same operations, over NFS and on a host directory. +struct Twin<'a> { + client: &'a mut Client, + root: Vec, + host: PathBuf, +} +impl Twin<'_> { + fn handle(&mut self, path: &str) -> Vec { + self.client.walk_to(&self.root, path).unwrap().0 + } + fn split(path: &str) -> (&str, &str) { + path.rsplit_once('/').unwrap_or(("", path)) + } + fn mkdir(&mut self, path: &str, mode: u32) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client.mkdir(&parent, name.as_bytes(), mode).unwrap(); + fs::create_dir(self.host.join(path)).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn write(&mut self, path: &str, offset: u64, data: &[u8]) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let handle = match self.client.lookup(&parent, name.as_bytes()) { + Ok((handle, _)) => handle, + Err(_) => { + let made = self.client.touch(&parent, name.as_bytes(), 0o644).unwrap(); + fs::write(self.host.join(path), b"").unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(0o644)) + .unwrap(); + made.handle + } + }; + self.client.write_all(&handle, offset, data, 64 * 1024); + use std::os::unix::fs::FileExt; + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.write_all_at(data, offset).unwrap(); + } + fn truncate(&mut self, path: &str, size: u64) { + let handle = self.handle(path); + let set = Set { + size: Some(size), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.set_len(size).unwrap(); + } + fn chmod(&mut self, path: &str, mode: u32) { + let handle = self.handle(path); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn rename(&mut self, from: &str, to: &str) { + let (a, an) = Self::split(from); + let (b, bn) = Self::split(to); + let (a, b) = (self.handle(a), self.handle(b)); + self.client + .rename(&a, an.as_bytes(), &b, bn.as_bytes()) + .unwrap(); + fs::rename(self.host.join(from), self.host.join(to)).unwrap(); + } + fn link(&mut self, existing: &str, new: &str) { + let file = self.handle(existing); + let (parent, name) = Self::split(new); + let parent = self.handle(parent); + self.client.link(&file, &parent, name.as_bytes()).unwrap(); + fs::hard_link(self.host.join(existing), self.host.join(new)).unwrap(); + } + fn symlink(&mut self, path: &str, target: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .symlink(&parent, name.as_bytes(), target.as_bytes()) + .unwrap(); + std::os::unix::fs::symlink(target, self.host.join(path)).unwrap(); + } + fn remove(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let directory = fs::symlink_metadata(self.host.join(path)).unwrap().is_dir(); + self.client + .unlink(&parent, name.as_bytes(), directory) + .unwrap(); + if directory { + fs::remove_dir(self.host.join(path)).unwrap(); + } else { + fs::remove_file(self.host.join(path)).unwrap(); + } + } + /// A file the client still has open when it is unlinked: renamed to a + /// silly name over NFS, simply gone on the host. + fn unlink_open(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .rename(&parent, name.as_bytes(), &parent, LINUX_SILLY) + .unwrap(); + fs::remove_file(self.host.join(path)).unwrap(); + } +} + +#[test] +fn a_commit_holds_what_the_same_operations_make_of_a_host_tree() { + let base = temp("commit"); + let tree = base.join("tree"); + source(&tree); + let host = base.join("host"); + fs::create_dir_all(&host).unwrap(); + assert!(std::process::Command::new("cp") + .arg("-a") + .arg(format!("{}/.", tree.display())) + .arg(&host) + .status() + .unwrap() + .success()); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let mut t = Twin { + client: &mut client, + root, + host: host.clone(), + }; + t.mkdir("a", 0o755); + t.mkdir("a/b", 0o700); + t.mkdir("a/b/c", 0o755); + t.write("a/b/c/big.bin", 0, &pattern(3, 400_000)); + t.write("a/b/c/big.bin", 123_456, &pattern(4, 70_000)); + t.truncate("a/b/c/big.bin", 350_001); + t.write("a/empty", 0, b""); + t.write("a/hello.txt", 0, b"hello\n"); + t.chmod("a/hello.txt", 0o755); + t.write("README.md", 0, b"# rewritten\n"); + t.truncate("src/nested/deep.txt", 0); + t.rename("a/hello.txt", "a/b/hello.txt"); + t.rename("a/b/c", "c-moved"); + t.link("a/b/hello.txt", "hello-alias"); + t.symlink("a/to-hello", "b/hello.txt"); + t.remove("src/alias.rs"); + t.remove("docs/guide.md"); + t.remove("docs"); + t.write("tmp-old", 0, b"old\n"); + t.write("tmp-new", 0, b"new\n"); + t.rename("tmp-new", "tmp-old"); + t.write("unicod\u{e9}.txt", 0, "caf\u{e9}\n".as_bytes()); + t.write("open-at-commit", 0, b"open\n"); + t.unlink_open("open-at-commit"); + drop(t); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + + // An independent Workspace bound to the committed Branch, read over NFS, + // equals the host tree path for path and byte for byte. + let expected = tree::oracle(&host); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let (paths, _, identities) = tree::walk(&mut v, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + let hello = fs::metadata(host.join("hello-alias")).unwrap(); + use std::os::unix::fs::MetadataExt; + assert_eq!(identities[&hello.ino()].len(), 1, "both names, one inode"); + drop(v); + viewer.stop(); + rig.close_view(view); + assert_eq!(OK, 0); + drop(client); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-node/Cargo.toml b/engine/crates/layerfs-node/Cargo.toml index 44fca52204..ef0420fa5c 100644 --- a/engine/crates/layerfs-node/Cargo.toml +++ b/engine/crates/layerfs-node/Cargo.toml @@ -22,6 +22,7 @@ layerfs-sandbox = { path = "../layerfs-sandbox" } layerfs-storage = { path = "../../../core/crates/layerfs-storage" } layerfs-store-service = { path = "../layerfs-store-service" } layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } +libc = "=0.2.189" napi = "=2.16.17" napi-derive = "=2.16.13" diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..95083db61d 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -1,4 +1,7 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +#[cfg(target_os = "linux")] +use crate::runtime::NAMESPACES; +use crate::runtime::{LOCAL, NATIVE}; use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, @@ -9,10 +12,11 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; -use layerfs_sandbox::{ - backend::docker::{Docker, Sandbox, SandboxRequest}, - CommandIdentity, ContainerId, -}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider, SandboxRequest}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::backend::native::{self, LocalProcess}; +use layerfs_sandbox::provider::{LocalRecord, SandboxRecord}; +use layerfs_sandbox::{CommandIdentity, ContainerId, Sandbox, WireFailure}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, @@ -26,6 +30,7 @@ use std::{ }; extern "C" { fn geteuid() -> u32; + fn getegid() -> u32; fn flock(fd: i32, operation: i32) -> i32; } const LOCK_EX: i32 = 2; @@ -45,7 +50,7 @@ pub(crate) fn home(args: &SdkValue) -> Result { directory(&home)?; Ok(home) } -fn private(path: &Path, dir: bool) -> Result<()> { +pub(crate) fn private(path: &Path, dir: bool) -> Result<()> { let meta = fs::symlink_metadata(path).map_err(failed)?; if meta.file_type().is_symlink() || meta.is_dir() != dir @@ -68,6 +73,11 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } +/// This process's effective uid and gid: a native Sandbox's command +/// identity (its commands run as this user). +pub(crate) fn user() -> (u32, u32) { + unsafe { (geteuid(), getegid()) } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -127,6 +137,8 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. +// Unmount exchanges run outside this guard; one shared readiness deadline +// covers the in-process turn and the cross-process discovery lock. const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); @@ -328,6 +340,20 @@ pub(crate) fn project(args: &SdkValue) -> Result { installed, }) } +/// The labels a record keeps, as the value map its reader expects. +fn label_values(labels: &[(String, String)]) -> SdkValue { + SdkValue::List( + (labels.iter()) + .map(|(key, value)| { + SdkValue::map([ + ("key", SdkValue::text(key)), + ("value", SdkValue::text(value)), + ]) + }) + .collect(), + ) +} + pub(crate) fn save_sandbox( directory: &Path, state: &SandboxState, @@ -339,41 +365,97 @@ pub(crate) fn save_sandbox( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "deploy"))?; + let mut fields = vec![ + ("id", SdkValue::text(&state.id)), + ("project", SdkValue::text(&state.project)), + ("name", SdkValue::text(&state.name)), + ( + "container", + SdkValue::text(state.runtime.identity().to_string()), + ), + ]; + match state.runtime.as_ref() { + Sandbox::Docker(runtime) => { + let record = runtime.request(); + fields.extend([ + ("docker", SdkValue::text(registry::text(deploy, "docker")?)), + ("image", SdkValue::text(&record.image)), + ("volume", SdkValue::text(&record.store_volume)), + ("labels", label_values(&record.labels)), + ]); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Sandbox::Native(runtime) => { + let record = runtime + .local_record() + .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + fields.extend(local_fields(args, state, &record, NATIVE, LOCAL)?); + } + #[cfg(target_os = "linux")] + Sandbox::Namespaces(runtime) => { + let Some(SandboxRecord::Namespaces(record)) = runtime.record() else { + return Err(refused("Failed", "namespace Sandbox has no init process")); + }; + let image = record.image().as_str().to_owned(); + fields.extend(local_fields( + args, + state, + record.local(), + NAMESPACES, + &image, + )?); + } + } + fields.extend([ + ("secret", SdkValue::Bytes(secret.to_vec())), + ("peer", SdkValue::Bytes(peer.to_vec())), + ("instance", SdkValue::Bytes(instance.to_vec())), + ]); save( &directory.join("sandbox.record"), - SdkValue::map([ - ("id", SdkValue::text(&state.id)), - ("project", SdkValue::text(&state.project)), - ("name", SdkValue::text(&state.name)), - ( - "container", - SdkValue::text(state.runtime.identity().to_string()), - ), - ("docker", SdkValue::text(registry::text(deploy, "docker")?)), - ("image", SdkValue::text(&state.runtime.request().image)), - ( - "volume", - SdkValue::text(&state.runtime.request().store_volume), - ), - ( - "labels", - SdkValue::List( - (state.runtime.request().labels.iter()) - .map(|(key, value)| { - SdkValue::map([ - ("key", SdkValue::text(key)), - ("value", SdkValue::text(value)), - ]) - }) - .collect(), - ), - ), - ("secret", SdkValue::Bytes(secret.to_vec())), - ("peer", SdkValue::Bytes(peer.to_vec())), - ("instance", SdkValue::Bytes(instance.to_vec())), - ]), + SdkValue::Map( + (fields.into_iter()) + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ), ) } +/// The fields a local Sandbox record keeps: its directory, its daemon or init +/// process identity, its listener port and the image it runs, under `runtime` +/// `native` (whose image is `local`) or `namespaces` (whose image is the +/// digest its root is an overlay of). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_fields( + args: &SdkValue, + state: &SandboxState, + record: &LocalRecord, + runtime: &'static str, + image: &str, +) -> Result> { + let boot = record + .boot + .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; + Ok(vec![ + ("runtime", SdkValue::text(runtime)), + ("uid", SdkValue::Int(i64::from(state.identity.uid))), + ("gid", SdkValue::Int(i64::from(state.identity.gid))), + ( + "directory", + SdkValue::text( + crate::local::directory(args, &state.id)? + .display() + .to_string(), + ), + ), + ("pid", SdkValue::Int(i64::from(record.pid))), + ("start", SdkValue::text(record.start_time.to_string())), + ("boot", SdkValue::text(format!("{boot:032x}"))), + ("port", SdkValue::Int(i64::from(record.port))), + ("image", SdkValue::text(image)), + ("labels", label_values(&record.labels)), + ]) +} + /// A Sandbox whose container no longer runs it: the Engine no longer has the /// container, or has it stopped (it closed when idle, or Docker restarted), /// which lost its daemon and every mount in it. Only its records, and a @@ -406,30 +488,260 @@ pub(crate) fn gone_container(refusal: &SdkRefusal) -> Option<(&str, bool)> { _ => None, } } -/// The Engine, container and deployment a Sandbox record names. -pub(crate) fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { +/// The labels a Sandbox record keeps; a record written before #373 has none. +fn labels(value: &SdkValue) -> Result> { + match value.get("labels") { + Some(SdkValue::List(items)) => items + .iter() + .map(|item| { + Ok(( + registry::text(item, "key")?.to_owned(), + registry::text(item, "value")?.to_owned(), + )) + }) + .collect(), + _ => Ok(Vec::new()), + } +} +/// The Engine, container and deployment a Docker Sandbox record names. +fn runtime(value: &SdkValue) -> Result<(DockerProvider, ContainerId, SandboxRecord)> { let docker = Docker::new(registry::text(value, "docker")?, Duration::from_secs(30)).map_err(failed)?; let request = SandboxRequest { image: registry::text(value, "image")?.to_owned(), store_volume: registry::text(value, "volume")?.to_owned(), port: 30421, - // a record written before #373 has no labels, nor has its container - labels: match value.get("labels") { - Some(SdkValue::List(items)) => items - .iter() - .map(|item| { - Ok(( - registry::text(item, "key")?.to_owned(), - registry::text(item, "value")?.to_owned(), - )) - }) - .collect::>()?, - _ => Vec::new(), - }, + labels: labels(value)?, }; let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; - Ok((docker, container, request)) + let provider = DockerProvider::new(docker); + Ok((provider, container, SandboxRecord::Docker(request.into()))) +} +/// What a Sandbox record names: a Docker container, a local daemon process +/// (`runtime` `native`) with its directory and port, or a namespace Sandbox +/// (`runtime` `namespaces`) with its own runtime, record and directory. +pub(crate) enum Recorded { + Docker(DockerProvider, ContainerId, SandboxRecord), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Local { + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + }, + #[cfg(target_os = "linux")] + Namespaces(layerfs_sandbox::Runtime, ContainerId, SandboxRecord), +} +/// Reads Sandbox `id`'s record's runtime. +pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + #[cfg(target_os = "linux")] + if value.get_text("runtime") == Some(NAMESPACES) { + return namespaces_record(args, id, value); + } + if matches!(value.get_text("runtime"), Some(LOCAL | NATIVE)) { + return local_record(args, id, value); + } + let (docker, container, request) = runtime(value)?; + Ok(Recorded::Docker(docker, container, request)) +} +/// A local record is used only where this deployment puts that Sandbox: +/// its directory must be exactly `/-` +/// and named so, so a damaged, edited or stale record (another +/// LAYERFS_LOCAL_DIR or prefix) never points a delete elsewhere. A record +/// without `boot` was written before boot ids were kept: its daemon is +/// matched by id and start time as then, and never signalled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, record) = local_record_of(args, id, value)?; + // with no placement (#428), a mount names its Sandbox: a record is used + // only by a native deployment, and only a native record is. A record of + // the `local` runtime (an earlier, unmerged build) is never used, so + // nothing it names is deleted + let native = args.get("deploy").is_some_and(crate::runtime::native); + if !native || value.get_text("runtime") != Some(NATIVE) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a {} Sandbox, but only a native deployment \ + (LAYERFS_SANDBOX=native) uses a native record and none uses another; \ + Sandboxes of other runtimes are never mixed, so the record is neither used \ + nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + value.get_text("runtime").unwrap_or_default(), + registry::text(args, "project")? + ), + )); + } + Ok(Recorded::Local { + directory, + id: ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + process: LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + }, + port: record.port, + labels: record.labels, + }) +} +/// A record's directory and physical facts, checked against where this +/// deployment puts that Sandbox. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record_of(args: &SdkValue, id: &str, value: &SdkValue) -> Result<(PathBuf, LocalRecord)> { + let directory = PathBuf::from(registry::text(value, "directory")?); + let expected = crate::local::directory(args, id)?; + // the Sandbox's own directory name, exactly as this deployment names it + let name = registry::text(value, "name")?; + if directory != expected || expected.file_name().and_then(|n| n.to_str()) != Some(name) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id}'s local record names {} (as {name}), but this deployment puts it at \ + {}: the record is damaged or from another LAYERFS_LOCAL_DIR or prefix, so it is \ + neither used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + directory.display(), + expected.display(), + registry::text(args, "project")? + ), + )); + } + let number = |key: &str| { + value + .get_int(key) + .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) + }; + let boot = match value.get_text("boot") { + None => None, + Some(hex) => Some( + (hex.len() == 32) + .then(|| u128::from_str_radix(hex, 16).ok()) + .flatten() + .ok_or_else(|| refused("InvalidInput", "local Sandbox record boot"))?, + ), + }; + let record = LocalRecord { + name: name.to_owned(), + pid: u32::try_from(number("pid")?).map_err(failed)?, + start_time: registry::text(value, "start")? + .parse::() + .map_err(failed)?, + boot, + port: u16::try_from(number("port")?).map_err(failed)?, + labels: labels(value)?, + }; + Ok((directory, record)) +} +/// A namespaces record is used only where this backend puts that Sandbox and +/// only by a deployment that selects it: another backend never reopens or +/// deletes what this one made. +#[cfg(target_os = "linux")] +fn namespaces_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, local) = local_record_of(args, id, value)?; + let image = + layerfs_sandbox::Image::parse(registry::text(value, "image")?).map_err(|error| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox whose record names no image digest \ + ({error}); it cannot be given a root, so it is neither reopened nor deleted" + ), + ) + })?; + let record = layerfs_sandbox::provider::NamespaceRecord { local, image }; + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + if crate::runtime::backend(deploy)? != crate::runtime::Backend::Namespaces { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox, but this deployment runs the other local \ + backend; Sandboxes of other runtimes are never mixed, so the record is neither \ + used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + registry::text(args, "project")? + ), + )); + } + let root = (directory.parent()) + .ok_or_else(|| refused("InvalidInput", "namespaces Sandbox directory"))?; + let provider = + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root).map_err(|error| { + refused( + "Unsupported", + format!( + "namespaces Sandbox at {} cannot run on this host: {error}", + directory.display() + ), + ) + })?; + Ok(Recorded::Namespaces( + layerfs_sandbox::Runtime::Namespaces(provider), + ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + SandboxRecord::Namespaces(record), + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { + Err(refused( + "Unsupported", + format!( + "Sandbox {id} is a native Sandbox (LAYERFS_SANDBOX=native), which needs Linux or macOS" + ), + )) +} +impl Recorded { + /// The runtime, observed running once (it starts nothing). + pub(crate) fn reopen(self) -> std::result::Result, Box> { + match self { + Self::Docker(provider, id, record) => provider + .reopen(id, &record) + .map(|s| Box::new(Sandbox::Docker(s))), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => native::Sandbox::reopen(directory, id, process, port, labels) + .map(|s| Box::new(Sandbox::Native(s))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => runtime.reopen(id, &record).map(Box::new), + } + } + /// The runtime found stopped: only its one Delete is admitted. + pub(crate) fn stopped(self) -> Box { + match self { + Self::Docker(provider, id, record) => { + let instance = provider + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"); + Box::new(Sandbox::Docker(instance)) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => Box::new(Sandbox::Native(native::Sandbox::stopped( + directory, id, process, port, labels, + ))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => Box::new( + runtime + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"), + ), + } + } } pub(crate) fn reopen(args: &SdkValue) -> Result> { let id = registry::text(args, "sandbox")?; @@ -447,8 +759,16 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let (docker, selector, request) = runtime(&value)?; - let runtime = Sandbox::reopen(docker, selector, request).map_err(|failure| { + let recorded = recorded(args, id, &value)?; + // a local daemon listens on this kernel's loopback; no other host reaches it + let host = match recorded { + Recorded::Docker(..) => crate::pool::daemon_host(args), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Recorded::Local { .. } => None, + #[cfg(target_os = "linux")] + Recorded::Namespaces(..) => None, + }; + let runtime = recorded.reopen().map_err(|failure| { if failure.container_gone() || failure.container_stopped() { gone(id, container, failure.container_stopped()) } else { @@ -457,7 +777,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { })?; let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, - host: crate::pool::daemon_host(args), + host, secret: bytes(&value, "secret")?, peer: bytes(&value, "peer")?, instance: bytes(&value, "instance")?, @@ -470,7 +790,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { name: value.get_text("name").unwrap_or_default().to_owned(), directory, runtime, - identity: CommandIdentity { uid: 501, gid: 20 }, + identity: recorded_identity(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -478,8 +798,23 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { refresh(&state)?; Ok(state) } +/// The command identity a Sandbox record names: a native Sandbox's is the +/// user who started it (`uid`, `gid`); every other runs as 501:20. +fn recorded_identity(value: &SdkValue) -> CommandIdentity { + let id = |key: &str| value.get_int(key).and_then(|n| u32::try_from(n).ok()); + match (id("uid"), id("gid")) { + (Some(uid), Some(gid)) => CommandIdentity { uid, gid }, + _ => CommandIdentity { uid: 501, gid: 20 }, + } +} /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { + refresh_except(state, &[]) +} +/// Explicit Sandbox close may dispose retained unmounts without asking their +/// daemon for a Ready receipt that no longer exists. Other records still +/// undergo the ordinary refresh and MountsActive admission. +pub(crate) fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); let mut processes = std::collections::BTreeMap::new(); for item in fs::read_dir(&state.directory).map_err(failed)? { @@ -490,6 +825,9 @@ pub(crate) fn refresh(state: &SandboxState) -> Result<()> { continue; }; let path = registry::text(&value, "path")?; + if retained.iter().any(|kept| kept == path) { + continue; + } let token = layerfs_bridge::control::WorkspaceToken { workspace: layerfs_history::WorkspaceId::from_slice(&bytes::<32>( &value, diff --git a/engine/crates/layerfs-node/src/dispatch.rs b/engine/crates/layerfs-node/src/dispatch.rs index 99c80b12b6..365004dc7f 100644 --- a/engine/crates/layerfs-node/src/dispatch.rs +++ b/engine/crates/layerfs-node/src/dispatch.rs @@ -75,6 +75,9 @@ fn listed(answer: SdkValue) -> Outcome { /// the daemon, so it is not sent on. fn daemon_backend(op: &str, args: SdkValue) -> Outcome { let state = registry::sandbox_live(&args)?; + if let Some(path) = args.get_text("path") { + crate::unmount::check(&state, path)?; + } let args = match args { SdkValue::Map(fields) => SdkValue::Map( fields diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index 80a3120b2a..3982d74648 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -5,46 +5,38 @@ //! daemon is not involved. A background command is started in its own session //! with its output in a log file, and this binding keeps its process id. The //! Engine offers no per-exec cancel, so `cancel` signals the process group -//! from a second command and waits (bounded) until it is gone. -use crate::job::{failed, refused, Outcome}; +//! from a second command and waits (bounded) until it is gone. A local +//! Sandbox (`layerfs_sandbox::Sandbox::Native`) runs the same commands as +//! processes of this kernel, so the same probes and signals apply. +use crate::job::{refused, Outcome}; use crate::output::Spool; use crate::registry::{lock, naming, sandbox, sandbox_live, text, Process, SandboxState}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::time::Duration; -/// Home of the command identity, which has no account in the image. -const HOME: &str = "HOME=/tmp"; /// Checks that a cancelled process group is gone. const GONE_CHECKS: u32 = 50; /// Pause between those checks. const GONE_PAUSE: Duration = Duration::from_millis(100); /// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. +/// non-empty stream to its file instead of keeping it inline. Its home is +/// the Sandbox's scratch directory (`/tmp` in a container), since the +/// command identity has no account there. fn run( state: &SandboxState, directory: &str, arguments: Vec, spool: bool, ) -> Result<(i32, Spool, Spool), SdkRefusal> { - let created = layerfs_api::internal::exec( + crate::runtime::run( &state.runtime, state.identity, arguments, - vec![HOME.to_owned()], - directory.to_owned(), - false, + vec![format!("HOME={}", crate::runtime::scratch(&state.runtime))], + directory, + spool, ) - .map_err(failed)?; - let attached = created.start().map_err(failed)?; - let (_input, output, handle) = attached.into_parts().map_err(failed)?; - let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); - output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; - let inspected = handle.inspect().map_err(failed)?; - let code = inspected - .known_root_exit() - .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))?; - Ok((code, stdout, stderr)) } /// `workspace.exec(ws, command, { background?, spool? })`; with @@ -91,11 +83,17 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { state.id, layerfs_bridge::sdk_values::hex(&crate::registry::random()?[..12]) ); - let script = format!("setsid bash -c \"$0\" >/tmp/layerfs-{id}.log 2>&1 \"$1\" 2>&1 Outcome { "background process exited before its identity was observed", ) })?; + // its own session leaves the command's process group; named, it still + // ends with a local Sandbox. Unnamed it would outlive it: it ends here, once. + if let Err(refusal) = crate::runtime::track(&state.runtime, pid, &start) { + let end = format!("kill -KILL -- -{pid} 2>/dev/null; true"); + let _ = run( + &state, + &directory, + vec!["bash".into(), "-c".into(), end], + false, + ); + return Err(refusal); + } let process = Process { path: path.to_owned(), command: command.to_owned(), @@ -136,7 +146,7 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { /// The container's first process is the daemon, which does not reap orphans, /// so a finished member can stay a zombie that `kill -0` still finds. fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { - let (code, out, _) = run(state, "/tmp", vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; + let (code, out, _) = run(state, &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; if code == 0 { Ok(Some(String::from_utf8_lossy(out.kept()).into_owned())) } else { @@ -151,7 +161,7 @@ fn alive(state: &SandboxState, pid: u32) -> Result { ); let (code, _, _) = run( state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), probe], false, )?; @@ -283,7 +293,7 @@ pub(crate) fn cancel(args: &SdkValue) -> Outcome { } run( &state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec![ "bash".into(), "-c".into(), diff --git a/engine/crates/layerfs-node/src/lib.rs b/engine/crates/layerfs-node/src/lib.rs index 298d5020af..f7f89ae788 100644 --- a/engine/crates/layerfs-node/src/lib.rs +++ b/engine/crates/layerfs-node/src/lib.rs @@ -5,12 +5,16 @@ mod discovery; mod dispatch; mod exec; mod job; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local; mod output; mod pool; mod project; mod registry; +mod runtime; mod sandbox; mod trace; +mod unmount; mod workspace; use job::Job; diff --git a/engine/crates/layerfs-node/src/local.rs b/engine/crates/layerfs-node/src/local.rs new file mode 100644 index 0000000000..1556ed7efa --- /dev/null +++ b/engine/crates/layerfs-node/src/local.rs @@ -0,0 +1,440 @@ +//! Native Sandboxes (`deploy.sandbox` `native`, LAYERFS_SANDBOX=native): +//! the project's daemon runs as a process of this user instead of in a +//! container. It isolates nothing beyond the caller's own sandbox. +//! +//! Each Sandbox gets a directory `/-` +//! with what a container held at fixed paths: its setup +//! (`/layerfs-local/config`), overlay template (`/layerfs-local/overlay`) and +//! Workspace mounts (`/workspaces`). The daemon opens the installed Store +//! where this process opens it (`storePath`), so the daemon setup and the +//! Store's locator name that path instead of `/layerfs-store`. The daemon +//! listens on 127.0.0.1 only. macOS serves loopback NFS as this user; Linux +//! keeps the root FUSE daemon and nonroot commands (501:20) from #402. Start, +//! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's +//! daemon stopped and its directory deleted, each once. +use crate::job::{failed, refused}; +use crate::registry::{lock, text, SandboxState, SANDBOXES}; +use layerfs_api::{sandbox::ProviderCreate, SandboxApi}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::{CommandIdentity, Runtime, Sandbox}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +/// Starts the project's next Sandbox as a native daemon. Linux requires +/// root before any effect. Called by +/// [`crate::sandbox::start`] under the project's guard, with the project +/// and its installed manifest already read. +pub(crate) fn start( + args: &SdkValue, + directory: &Path, + project: (&crate::registry::Project, StoreManifest), +) -> Result, SdkRefusal> { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + // Inputs are read first, so a refused request leaves no creation intent. + if cfg!(target_os = "linux") && crate::discovery::user().0 != 0 { + return Err(refused( + "Unsupported", + "LAYERFS_SANDBOX=native on Linux needs root for FUSE and commands as uid 501, gid 20; run inside the VM or CI box that isolates it", + )); + } + let root = deploy.get_text("localDir").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a native Sandbox needs it", + ) + })?; + let backend = crate::runtime::backend(deploy)?; + let runtime = native_runtime(root, backend) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let executable = PathBuf::from(text(deploy, "daemon")?); + let store = match args.get_text("storePath") { + Some(path) if Path::new(path).is_absolute() => path.to_owned(), + _ => { + return Err(refused( + "InvalidInput", + "a local Sandbox's daemon opens the client's storePath (LAYERFS_STORE_PATH), an absolute path", + )) + } + }; + crate::sandbox::unresolved( + directory, + &format!("a directory under {root} named for it, and its daemon process"), + )?; + let (id, saved_directory) = crate::discovery::claim_next(args)?; + let made = start_claimed( + args, + directory, + &saved_directory, + &id, + project, + (runtime, backend, executable, store), + ); + if made.is_err() { + let _ = std::fs::remove_dir(&saved_directory); + } + made +} + +/// Starts the claimed Sandbox `id`: directory and daemon on the installed +/// Store, its first control connection, then its records. +fn start_claimed( + args: &SdkValue, + directory: &Path, + saved_directory: &Path, + id: &str, + (project, installed): (&crate::registry::Project, StoreManifest), + (runtime, backend, executable, store): (Runtime, crate::runtime::Backend, PathBuf, String), +) -> Result, SdkRefusal> { + let name = text(args, "project")?; + let sandbox_name = sandbox_name(args, id)?; + let paths = runtime.native_paths(&sandbox_name).map_err(failed)?; + let path_text = |path: &Path| { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_LOCAL_DIR must be UTF-8")) + }; + let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let mut setup = crate::sandbox::setup(daemon.private, controller.public); + // macOS uses the caller; Linux keeps #402's separate nonroot identity. + #[cfg(target_os = "macos")] + { + (setup.command_uid, setup.command_gid) = crate::discovery::user(); + } + setup.listen = "127.0.0.1:0".into(); + setup.store = store.clone(); + setup.overlay = path_text(&paths.overlay)?; + setup.mounts = path_text(&paths.mounts)?; + let mut existing = project.manifest.clone(); + existing.daemon_sqlite = installed.daemon_sqlite; + // the Store's path in the daemon's filesystem, which is this one + existing.locator = store; + setup.existing_store = Some(existing); + let identity = CommandIdentity { + uid: setup.command_uid, + gid: setup.command_gid, + }; + setup.encode().map_err(failed)?; + // Read only if this process ends before the intent is resolved. + crate::discovery::save( + &directory.join("create.intent"), + SdkValue::text(format!("creation of Sandbox {id} began and never finished")), + )?; + let labels = crate::sandbox::labels(name, id); + let api = SandboxApi::with_runtime(Arc::new(runtime)); + let options = launch_options(backend, identity, &setup.store, args)?; + let mut manifest = project.manifest.clone(); + manifest.locator = setup.store.clone(); + let created = api + .create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: sandbox_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), + labels, + startup_timeout: crate::sandbox::STARTUP, + options, + }, + setup, + controller_private: controller.private, + daemon_peer: daemon.public, + }, + &project.store, + &manifest, + ) + .map_err(|mut failure| { + drop(failure.control.take()); + abandon( + directory, + id, + format!( + "native sandbox stopped at {:?}: {:?}", + failure.phase, failure.cause + ), + failure.runtime.take(), + ) + })?; + let instance = created.startup.instance; + let (sandbox, control, identity) = layerfs_api::internal::split_provider(created); + let endpoint = match sandbox.endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + drop(control); + return Err(abandon( + directory, + id, + format!("native endpoint after startup: {error:?}"), + Some(sandbox), + )); + } + }; + let dial = crate::pool::Dial { + endpoint, + host: None, + secret: controller.private, + peer: daemon.public, + instance, + }; + let state = SandboxState { + id: id.to_owned(), + project: name.to_owned(), + name: sandbox_name, + directory: saved_directory.to_owned(), + runtime: sandbox, + identity, + controls: crate::pool::Controls::new(control, dial), + mounts: Default::default(), + processes: Default::default(), + }; + let recorded = (|| { + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed) + })(); + if let Err(refusal) = recorded { + // The daemon runs and is this start's own: it is stopped and its + // directory deleted like any failed start's, and the records written + // so far are removed, so its number is free again. + let SandboxState { runtime, .. } = state; + for record in ["sandbox.record", "container"] { + let _ = std::fs::remove_file(saved_directory.join(record)); + } + return Err(abandon( + directory, + id, + format!("local sandbox records: {}", refusal.message), + Some(runtime), + )); + } + let state = Arc::new(state); + lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); + Ok(state) +} + +/// The native runtime this host offers, over `root`: macOS loopback NFS as +/// this user, or Linux FUSE with a root daemon. The other platform's +/// provider, and another OS, is refused before root access. +#[cfg(target_os = "macos")] +fn native_runtime( + root: &str, + _: crate::runtime::Backend, +) -> Result { + layerfs_sandbox::backend::native::NativeMacosProvider::new(root).map(Runtime::MacosNative) +} + +#[cfg(target_os = "linux")] +fn native_runtime( + root: &str, + backend: crate::runtime::Backend, +) -> Result { + match backend { + crate::runtime::Backend::Native => { + layerfs_sandbox::backend::native::NativeLinuxProvider::new(root) + .map(Runtime::LinuxNative) + } + crate::runtime::Backend::Namespaces => { + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root) + .map(Runtime::Namespaces) + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn native_runtime( + _: &str, + _: crate::runtime::Backend, +) -> Result { + Err(layerfs_sandbox::RuntimeError::Unsupported( + "native Sandbox runtime platform", + )) +} + +/// How the daemon of `backend` is launched: the same local options, and for +/// the namespace backend the Store file its private root binds and the image +/// archive its root is unpacked from. +#[cfg(target_os = "linux")] +fn launch_options( + backend: crate::runtime::Backend, + identity: CommandIdentity, + store: &str, + args: &SdkValue, +) -> Result { + Ok(match backend { + crate::runtime::Backend::Native => ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }, + crate::runtime::Backend::Namespaces => { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + let (image, archive) = crate::runtime::namespaces_image(deploy)?; + ProviderLaunchOptions::Namespaces { + command_identity: identity, + environment: native_environment(), + store: PathBuf::from(store), + image, + archive, + } + } + }) +} + +#[cfg(not(target_os = "linux"))] +fn launch_options( + _: crate::runtime::Backend, + identity: CommandIdentity, + _: &str, + _: &SdkValue, +) -> Result { + Ok(ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }) +} + +/// What a native Sandbox's daemon starts with besides `PATH`: the NFS +/// route, and the mount helper and reference overrides this process was +/// given (`LAYERFS_NFS_*`), so a test host can name its own client. +#[cfg(target_os = "macos")] +fn native_environment() -> Vec<(String, String)> { + let mut environment = vec![("LAYERFS_KERNEL".to_owned(), "nfs".to_owned())]; + for name in [ + "LAYERFS_NFS_MOUNT", + "LAYERFS_NFS_UNMOUNT", + "LAYERFS_NFS_FORCE", + "LAYERFS_NFS_FLUSH", + "LAYERFS_NFS_REFRESH", + ] { + if let Ok(value) = std::env::var(name) { + environment.push((name.to_owned(), value)); + } + } + environment +} + +#[cfg(target_os = "linux")] +fn native_environment() -> Vec<(String, String)> { + vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())] +} + +/// A local Sandbox's directory name: `-`, the name its +/// container would have. +fn sandbox_name(args: &SdkValue, id: &str) -> Result { + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + Ok(format!("{prefix}{}-{id}", text(args, "project")?)) +} + +/// Where this deployment puts Sandbox `id`: +/// `/-`, computed from the +/// configuration alone, as a record's directory must be. +pub(crate) fn directory(args: &SdkValue, id: &str) -> Result { + let root = (args.get("deploy")) + .and_then(|deploy| deploy.get_text("localDir")) + .map(Path::new) + .ok_or_else(|| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a local Sandbox; its record is used only with \ + LAYERFS_LOCAL_DIR (deploy.localDir) configured" + ), + ) + })?; + let name = sandbox_name(args, id)?; + let plain = |path: &Path| { + !path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + }; + let single = matches!( + Path::new(&name).components().collect::>()[..], + [Component::Normal(_)] + ); + if !root.is_absolute() || !plain(root) || !single { + return Err(refused( + "InvalidInput", + format!("local Sandbox directory {}/{name}", root.display()), + )); + } + Ok(root.join(name)) +} + +/// The refusal for a failed start. Everything a local start makes is its +/// own (a directory and a child process), so each effect is known: the +/// daemon's last log lines are read once, then it is stopped and its +/// directory deleted, each once, and the intent removed. If that cleanup +/// fails, the intent stays and says what is left. +fn abandon( + directory: &Path, + id: &str, + mut cause: String, + sandbox: Option>, +) -> SdkRefusal { + let container = sandbox.as_ref().map(|s| s.identity().to_string()); + let mut log = None; + let mut left = None; + if let Some(mut sandbox) = sandbox { + match sandbox.log_tail(crate::sandbox::LOG_LINES, crate::sandbox::LOG_BYTES) { + Ok(bytes) => { + let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); + cause = format!("{cause}; its daemon's last log lines:\n{text}"); + log = Some(text); + } + Err(error) => cause = format!("{cause}; its daemon log was unreadable: {error:?}"), + } + let removed = sandbox.stop(1).and_then(|()| sandbox.delete()); + left = removed.err().map(|error| { + format!( + "its local Sandbox {} could not be removed: {error:?}", + sandbox.identity() + ) + }); + } + let intent = directory.join("create.intent"); + let (message, removed) = match left { + None => match std::fs::remove_file(&intent) { + Ok(()) => (cause, container.is_some()), + Err(error) => ( + format!("{cause}; its creation intent stays: {error}"), + container.is_some(), + ), + }, + Some(left) => { + let state = format!("creation of Sandbox {id} failed ({cause}); {left}"); + let kept = crate::discovery::save(&intent, SdkValue::text(&state)) + .err() + .map(|error| format!(" (recording it failed: {})", error.message)) + .unwrap_or_default(); + (format!("{state}; its creation intent stays{kept}"), false) + } + }; + SdkRefusal { + code: "Failed".to_owned(), + message, + detail: SdkValue::map([ + ("container", SdkValue::opt_text(container)), + ("removed", SdkValue::Bool(removed)), + ("log", SdkValue::opt_text(log)), + ]), + } +} diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..8bbbc2081c 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -2,7 +2,8 @@ //! project itself is only its home entry (`//`), so a project //! removed there can be initialized again without a restart. //! -//! A Sandbox is one container and its daemon. Its runtime runs commands +//! A Sandbox is one container and its daemon (or, with the local runtime, +//! one daemon process: `runtime`). Its runtime runs commands //! directly; its control connections (`pool`) each carry one exchange at a //! time, and concurrent calls take separate connections. The Sandbox runtime owner (this //! binding) also keeps the background processes it started; the daemon holds @@ -11,7 +12,7 @@ use crate::job::refused; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; +use layerfs_sandbox::CommandIdentity; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -42,10 +43,11 @@ pub(crate) struct Process { pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, - /// Its container's name, `-`. + /// Its container's name, `-` (a local Sandbox's + /// directory name). pub(crate) name: String, pub(crate) directory: std::path::PathBuf, - pub(crate) runtime: Sandbox, + pub(crate) runtime: Box, pub(crate) identity: CommandIdentity, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, @@ -76,7 +78,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// tables as they stand: no daemon call (#352 F-029). A call that names a /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes -/// [`sandbox_refreshed`]. +/// `discovery::refresh`. pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -129,14 +131,6 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { SdkValue::Map(fields) } -/// The Sandbox an operation names, its tables read again from the shared -/// records and the daemon (one status call per mount). -pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { - let state = sandbox(args)?; - crate::discovery::refresh(&state)?; - Ok(state) -} - impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { @@ -154,6 +148,7 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { + crate::unmount::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs new file mode 100644 index 0000000000..a4757f26ab --- /dev/null +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -0,0 +1,156 @@ +//! A Sandbox's runtime: Docker (default), or `LAYERFS_SANDBOX=native` on +//! Linux (root FUSE daemon, nonroot commands) and macOS (loopback NFS, +//! caller identity). Both use the same authenticated daemon control pipeline +//! and the same concrete `layerfs_sandbox::Sandbox` handle. +use crate::job::{failed, refused}; +use crate::output::Spool; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::provider::SandboxRecord; +#[cfg(target_os = "linux")] +use layerfs_sandbox::Image; +use layerfs_sandbox::{CommandIdentity, Sandbox}; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The image a native Sandbox records and reports, and the `runtime` of a +/// record from the former `local` runtime, which is never used (a Docker +/// record has no `runtime`). +pub(crate) const LOCAL: &str = "local"; +/// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local +/// daemon serving FUSE on Linux or loopback NFS on macOS. +pub(crate) const NATIVE: &str = "native"; +/// A namespace Sandbox record's `runtime` (LAYERFS_SANDBOX_BACKEND=namespaces, +/// Linux): a local daemon in its own mount, PID and IPC namespaces and cgroup. +pub(crate) const NAMESPACES: &str = "namespaces"; + +/// Which local backend a native deployment runs inside its Sandbox directory: +/// the plain native one, or the Linux namespace one. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Backend { + Native, + Namespaces, +} + +/// The local backend the deployment selects: `deploy.sandboxBackend` +/// (LAYERFS_SANDBOX_BACKEND), `native` when unset. The namespace backend is +/// Linux only; anywhere else selecting it is refused before any effect. +pub(crate) fn backend(deploy: &SdkValue) -> Result { + match deploy.get_text("sandboxBackend") { + None | Some(NATIVE) => Ok(Backend::Native), + Some(NAMESPACES) if cfg!(target_os = "linux") => Ok(Backend::Namespaces), + Some(NAMESPACES) => Err(refused( + "Unsupported", + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) namespaces is Linux only; use native", + )), + Some(other) => Err(refused( + "InvalidInput", + format!( + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) is native or namespaces, not {other}" + ), + )), + } +} + +/// Whether the deployment selects a local daemon: `deploy.sandbox` unset or +/// `docker` is Docker, `native` is a local daemon; anything else (the +/// former `local` included) is refused before any effect. +pub(crate) fn local(deploy: &SdkValue) -> Result { + match deploy.get_text("sandbox") { + None | Some("docker") => Ok(false), + Some(NATIVE) => Ok(true), + Some(other) => Err(refused( + "InvalidInput", + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not {other}"), + )), + } +} + +/// Whether the deployment selects the native route: FUSE on Linux, +/// loopback NFS on macOS, with no Docker. +pub(crate) fn native(deploy: &SdkValue) -> bool { + deploy.get_text("sandbox") == Some(NATIVE) +} + +/// The image a Sandbox reports: its container's, the digest a namespaces +/// Sandbox's root is an overlay of, or `local` for a native one. +pub(crate) fn image(runtime: &Sandbox) -> String { + match runtime.record() { + Some(SandboxRecord::Docker(docker)) => docker.image, + #[cfg(target_os = "linux")] + Some(SandboxRecord::Namespaces(record)) => record.image().as_str().to_owned(), + _ => LOCAL.to_owned(), + } +} + +/// The image a namespaces deployment's Sandbox root is an overlay of: +/// `deploy.image` (LAYERFS_IMAGE), the same `sha256:<64 hex>` digest the +/// Docker backend uses, and `deploy.imageArchive` (LAYERFS_IMAGE_ARCHIVE), +/// the docker-archive in the bundle that holds it. Both are read before any +/// effect, so a bundle without its image refuses the start. +#[cfg(target_os = "linux")] +pub(crate) fn namespaces_image(deploy: &SdkValue) -> Result<(Image, PathBuf), SdkRefusal> { + let digest = deploy.get_text("image").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.image is not configured (LAYERFS_IMAGE); a namespaces Sandbox's root is the image that digest names", + ) + })?; + let image = Image::parse(digest) + .map_err(|error| refused("InvalidInput", format!("deploy.image: {error}")))?; + let archive = deploy.get_text("imageArchive").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.imageArchive is not configured (LAYERFS_IMAGE_ARCHIVE); a namespaces Sandbox needs the docker-archive the bundle carries", + ) + })?; + let archive = PathBuf::from(archive); + if !archive.is_absolute() { + return Err(refused( + "InvalidInput", + "deploy.imageArchive (LAYERFS_IMAGE_ARCHIVE) must be an absolute path", + )); + } + Ok((image, archive)) +} + +/// Where its commands keep scratch files: `/tmp` in a container, the +/// Sandbox's own `tmp` directory for a native one (deleted with it), so a +/// native Sandbox leaves nothing in this machine's `/tmp`. +pub(crate) fn scratch(runtime: &Sandbox) -> String { + runtime.scratch_directory() +} + +/// Names a background process a command started in a session of its own +/// (`pid`, started at `start`), so it ends with a native Sandbox as it ends +/// with a container. A container's teardown already owns its descendants. +pub(crate) fn track(runtime: &Sandbox, pid: u32, start: &str) -> Result<(), SdkRefusal> { + if matches!(runtime, Sandbox::Docker(_)) { + return Ok(()); + } + let start = start.parse::().map_err(failed)?; + runtime.track(pid, start).map_err(failed) +} + +/// One command to its end: exit code, stdout, stderr. `spool` sends every +/// non-empty stream to its file instead of keeping it inline. +pub(crate) fn run( + runtime: &Sandbox, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, + spool: bool, +) -> Result<(i32, Spool, Spool), SdkRefusal> { + let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); + let request = layerfs_sandbox::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + let exit = runtime + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)?; + Ok((exit.code, stdout, stderr)) +} diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index edf23889a2..45e164f3b3 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -14,15 +14,15 @@ //! the caller's behalf. use crate::job::{failed, refused, Outcome}; use crate::registry::{lock, text, SandboxState, SANDBOXES}; -use layerfs_api::internal::split; -use layerfs_api::sandbox::values; -use layerfs_api::{ - OperationFailure, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase, -}; +use layerfs_api::internal::split_provider; +use layerfs_api::sandbox::{values, ProviderCreate, ProviderFailure}; +use layerfs_api::{OperationFailure, SandboxApi, SandboxCause, SandboxPhase}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::backend::docker::{CreatePhase, Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{CreatePhase, Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -30,12 +30,12 @@ use std::time::Duration; /// Port the daemon listens on inside its container. const PORT: u16 = 30421; /// Wait for the daemon's listener after Start. -const STARTUP: Duration = Duration::from_secs(30); +pub(crate) const STARTUP: Duration = Duration::from_secs(30); /// Wait for one Engine control response. const ENGINE: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. -const LOG_LINES: u16 = 20; -const LOG_BYTES: usize = 2048; +pub(crate) const LOG_LINES: u16 = 20; +pub(crate) const LOG_BYTES: usize = 2048; pub(crate) fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +pub(crate) fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -84,7 +84,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { } /// A container name prefix when the deployment sets none. -const PREFIX: &str = "layerfs-"; +pub(crate) const PREFIX: &str = "layerfs-"; /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. @@ -115,6 +115,15 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result state, - _ => "its record is unreadable".to_owned(), - }; - return Err(refused( - "Unknown", - format!( - "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ - is unknown, so it is neither retried nor taken as done. Remove any container it \ - left (docker ps -a --filter volume={store_volume}), then delete {}", - intent.display() - ), - )); - } + unresolved( + directory, + &format!("docker ps -a --filter volume={store_volume}"), + )?; // The number is claimed before anything is made; a failed start gives it // back (its directory holds no record yet), so the number is used again. let (id, saved_directory) = crate::discovery::claim_next(args)?; @@ -174,6 +170,29 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result<(), SdkRefusal> { + let intent = directory.join("create.intent"); + if intent.exists() { + let state = match crate::discovery::load(&intent) { + Ok(SdkValue::Text(state)) => state, + _ => "its record is unreadable".to_owned(), + }; + return Err(refused( + "Unknown", + format!( + "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ + is unknown, so it is neither retried nor taken as done. Remove any container it \ + left ({left}), then delete {}", + intent.display() + ), + )); + } + Ok(()) +} + /// Starts the claimed Sandbox `id`: container, daemon on the installed /// Store, then its records. fn start_claimed( @@ -204,25 +223,30 @@ fn start_claimed( .map_err(failed)? .naming(&container_name) .map_err(|_| refused("InvalidInput", format!("container name {container_name}")))?; + let provider = Arc::new(Runtime::Docker(DockerProvider::new(docker))); // Read only if this process ends before the intent is resolved. crate::discovery::save( &directory.join("create.intent"), SdkValue::text(format!("creation of Sandbox {id} began and never finished")), )?; - let created = SandboxApi::new(docker).create_file( - SandboxCreate { - deployment: SandboxRequest { - image, - store_volume, - port: PORT, + let created = SandboxApi::with_runtime(provider).create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: container_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), labels: labels(name, id), + startup_timeout: STARTUP, + options: ProviderLaunchOptions::Docker { + image, + store_volume, + port: PORT, + connect_host: crate::pool::daemon_host(args), + }, }, setup: daemon_setup, - executable, controller_private: controller.private, daemon_peer: daemon.public, - startup_wait: STARTUP, - connect_host: crate::pool::daemon_host(args), }, &project.store, &project.manifest, @@ -230,7 +254,7 @@ fn start_claimed( let owner = created.map_err(|failure| abandon(directory, id, failure))?; let container = owner.runtime.identity().to_string(); let instance = owner.startup.instance; - let (runtime, control, identity) = split(owner); + let (runtime, control, identity) = split_provider(owner); let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, host: crate::pool::daemon_host(args), @@ -267,33 +291,46 @@ fn start_claimed( /// Whether a failed create's every effect is known: nothing reached the /// Store volume, and any container it made is acknowledged by identity. -fn settled(failure: &SandboxFailure) -> bool { +fn settled(failure: &ProviderFailure) -> bool { // A Create the Engine refused (409: the container name is taken) made no container. - let unknown_container = matches!( - &failure.cause, - SandboxCause::Create(create) if create.phase == CreatePhase::Create - && !matches!(create.transfer.cause, layerfs_sandbox::RuntimeError::Http(409)) - ); + let unknown_container = match &failure.cause { + SandboxCause::Provider(launch) => match &launch.cause { + layerfs_sandbox::RuntimeError::Docker(docker_failure) => { + match docker_failure.as_ref() { + layerfs_sandbox::backend::docker::ProviderFailure::Create(create) => { + create.phase == CreatePhase::Create + && !matches!( + create.transfer.cause, + layerfs_sandbox::RuntimeError::Http(409) + ) + } + _ => false, + } + } + _ => false, + }, + _ => false, + }; failure.phase != SandboxPhase::Install && failure.installed.is_none() && !unknown_container } /// The refusal for a failed create. A settled one reads its container's log /// tail once, removes the container (stop, then delete, each once) and its /// intent; otherwise the intent stays, rewritten to say what is left. -fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { +fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { let mut cause = format!( "sandbox create stopped at {:?}: {:?}", failure.phase, failure.cause ); let container = failure - .sandbox + .runtime .as_ref() .map(|sandbox| sandbox.identity().to_string()); drop(failure.control.take()); let mut log = None; let left = if !settled(&failure) { Some("its effect on the Store volume or the Engine is unknown".to_owned()) - } else if let Some(sandbox) = failure.sandbox.as_mut() { + } else if let Some(sandbox) = failure.runtime.as_mut() { match sandbox.log_tail(LOG_LINES, LOG_BYTES) { Ok(bytes) => { let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); @@ -344,7 +381,8 @@ fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkR /// `sandbox.create({ image })`: under the project's guard, reclaims the /// project's idle-closed Sandboxes, then starts the lowest free number on /// the call's image (`deploy.image`) and waits for its daemon: -/// `{ sandbox, name, image }`. The only call that starts a Sandbox. Failures +/// `{ sandbox, name, image }`. With `deploy.sandbox` `native` (macOS) it +/// starts a daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures /// are `Failed` or `Unknown`, never retried. pub(crate) fn create(args: &SdkValue) -> Outcome { let project = crate::discovery::project_dir(args)?; @@ -356,7 +394,7 @@ pub(crate) fn create(args: &SdkValue) -> Outcome { ("name", SdkValue::text(&state.name)), ( "image", - SdkValue::text(state.runtime.request().image.as_str()), + SdkValue::text(crate::runtime::image(&state.runtime)), ), ])) } @@ -381,8 +419,16 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { } let state = crate::registry::sandbox_live(args)?; let _guard = crate::discovery::guard(&state.directory)?; - crate::discovery::refresh(&state)?; - let mounted = lock(&state.mounts)?.keys().cloned().collect::>(); + let recoverable = crate::unmount::check_all(&state)?; + crate::discovery::refresh_except(&state, &recoverable)?; + // Explicit close can take over an orphaned/retained unmount, but must + // still refuse every unrelated live Workspace. Stop detaches before + // killing the daemon; Delete independently checks physical detach. + let mounted = lock(&state.mounts)? + .keys() + .filter(|path| !recoverable.contains(path)) + .cloned() + .collect::>(); if !mounted.is_empty() { return Err(SdkRefusal { code: "MountsActive".to_owned(), @@ -397,8 +443,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { }); } let value = crate::discovery::load(&record)?; - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let mut owner = layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request) + let mut owner = crate::discovery::recorded(args, &id, &value)? + .reopen() .map_err(failed)?; owner.stop(STOP_S).map_err(failed)?; owner.delete().map_err(failed)?; @@ -413,7 +459,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { /// `backend.daemon.sandbox.get_log({ sandbox, lines })`: the last `lines` /// lines (1..=2000) of Sandbox `sandbox`'s daemon log, at most -/// [`GET_LOG_BYTES`], read once through the Engine: `{ sandbox, container, +/// [`GET_LOG_BYTES`], read once through the Engine (a local Sandbox's from +/// its log file): `{ sandbox, container, /// text, bytes }`. A stopped container's log is readable until its records /// are reclaimed. Refused `NoSuchSandbox` with no record or no container. /// Read-only: it starts, stops and reclaims nothing. @@ -437,8 +484,7 @@ pub(crate) fn get_log(args: &SdkValue) -> Outcome { crate::discovery::load(&record)? }; let container = text(&value, "container")?.to_owned(); - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let sandbox = layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request); + let sandbox = crate::discovery::recorded(args, &id, &value)?.stopped(); let bytes = match sandbox.log_tail(lines, GET_LOG_BYTES) { Ok(bytes) => bytes, Err(failure) if failure.container_gone() => { @@ -460,7 +506,7 @@ const GET_LOG_BYTES: usize = 64 * 1024; /// Seconds a closing Sandbox's daemon has to exit before the Engine kills it. const STOP_S: u16 = 5; -/// Reclaims Sandbox `id`, whose container a reopen found gone or stopped +/// Reclaims Sandbox `id`, whose container (or local daemon) a reopen found gone or stopped /// (`container`, `stopped`): one unforced Delete of a stopped container, then /// its records, so its number is free again. Called under the project's /// guard; the Sandbox's own guard is taken too. The record is read again @@ -485,8 +531,9 @@ pub(crate) fn reclaim(args: &SdkValue, id: &str, container: &str, stopped: bool) )); } if stopped { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - match layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request).delete() + match crate::discovery::recorded(args, id, &value)? + .stopped() + .delete() { Err(failure) if !failure.container_gone() => return Err(failed(failure)), _ => {} diff --git a/engine/crates/layerfs-node/src/unmount.rs b/engine/crates/layerfs-node/src/unmount.rs new file mode 100644 index 0000000000..34b0c34b95 --- /dev/null +++ b/engine/crates/layerfs-node/src/unmount.rs @@ -0,0 +1,282 @@ +//! Workspace-scoped unmount ownership; Sandbox discovery is held only at +//! admission and shared-record cleanup, never during the daemon's helper wait. +//! Mount paths are unique within one daemon lifetime; Sandbox removal clears +//! the terminal records before that directory can be reused. +use crate::{ + discovery, + job::{failed, refused, Outcome}, + registry::SandboxState, +}; +use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; +use std::{ + fs::{File, OpenOptions}, + os::unix::{fs::OpenOptionsExt, io::AsRawFd}, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +extern "C" { + fn flock(fd: i32, operation: i32) -> i32; +} +const LOCK_SH: i32 = 1; +const LOCK_EX: i32 = 2; +const LOCK_NB: i32 = 4; +const LOCK_UN: i32 = 8; +fn millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(i64::MAX as u128) as i64 +} +fn record(state: &SandboxState, token: WorkspaceToken) -> PathBuf { + discovery::workspace_file(state, token).with_extension("unmount") +} +struct Lease(File); +impl Drop for Lease { + fn drop(&mut self) { + unsafe { flock(self.0.as_raw_fd(), LOCK_UN) }; + } +} +fn lease(path: &std::path::Path, operation: i32) -> Result, SdkRefusal> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .map_err(failed)?; + discovery::private(path, false)?; + if unsafe { flock(file.as_raw_fd(), operation | LOCK_NB) } == 0 { + return Ok(Some(Lease(file))); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::WouldBlock { + Ok(None) + } else { + Err(failed(error)) + } +} +/// Explicit unlock matters when a concurrent fork inherited the descriptor. +pub(crate) struct Attempt { + _lease: Lease, + path: PathBuf, + started: i64, +} +fn result(value: &SdkValue) -> Outcome { + if value.get_text("state") == Some("Unmounted") { + return Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])); + } + let detail = SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "elapsedMs", + SdkValue::Int( + millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0), + ), + ), + ("original", value.get("detail").cloned().unwrap_or_default()), + ]); + Err(SdkRefusal { + code: value.get_text("code").unwrap_or("Unknown").into(), + message: value + .get_text("message") + .unwrap_or("unmount outcome retained") + .into(), + detail, + }) +} +fn progress(value: &SdkValue, active: bool) -> SdkRefusal { + let elapsed = millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0); + SdkRefusal { + code: if active { "Busy" } else { "Unknown" }.into(), + message: if active { + format!("Workspace is Unmounting; waited {elapsed}ms") + } else { + format!("unmount caller ended without publishing its result after {elapsed}ms; inspect daemon custody, do not resend") + }, + detail: SdkValue::map([ + ( + "state", + SdkValue::text(if active { "Unmounting" } else { "Retained" }), + ), + ("elapsedMs", SdkValue::Int(elapsed)), + ]), + } +} +fn observed(path: &std::path::Path) -> Result, SdkRefusal> { + match discovery::load(path) { + Ok(value) => Ok(Some(value)), + Err(_) if !path.exists() => Ok(None), + Err(error) => Err(error), + } +} +/// Consulted before token lookup, including when another process removed +/// the Workspace record. Does not acquire the Sandbox guard or dial daemon. +fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { + let Some(value) = observed(path)? else { + return Ok(None); + }; + if value.get_text("state") != Some("Unmounting") { + return Ok(Some(value)); + } + let available = lease(&path.with_extension("unmount-lock"), LOCK_SH)?; + // Publication may have completed between our read and lease acquisition. + let latest = observed(path)?; + match latest { + Some(latest) if latest.get_text("state") != Some("Unmounting") => Ok(Some(latest)), + None => Ok(None), + Some(_) if available.is_none() => Err(progress(&value, true)), + Some(_) => { + let error = progress(&value, false); + Ok(Some(SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "startedMs", + value.get("startedMs").cloned().unwrap_or_default(), + ), + ("path", value.get("path").cloned().unwrap_or_default()), + ("code", SdkValue::text(error.code)), + ("message", SdkValue::text(error.message)), + ("detail", error.detail), + ]))) + } + } +} +pub(crate) fn check(state: &SandboxState, path: &str) -> Result<(), SdkRefusal> { + if let Some(value) = find(state, path)? { + if value.get_text("state") == Some("Unmounted") { + return Err(refused("NotMounted", "Workspace already unmounted")); + } + return result(&value).map(drop); + } + Ok(()) +} +pub(crate) fn find(state: &SandboxState, path: &str) -> Result, SdkRefusal> { + // Steady-state token/directory lookups already have a token: avoid a + // directory scan proportional to the lifetime's completed unmounts. + let known = crate::registry::lock(&state.mounts)? + .get(path) + .map(|mount| mount.workspace.bound.token); + if let Some(token) = known { + return inspect(&record(state, token)); + } + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + if let Some(value) = observed(&entry.path())? { + if value.get_text("path") == Some(path) { + return inspect(&entry.path()); + } + } + } + } + Ok(None) +} +/// Closing a running Sandbox may not race a registered terminal operation. +pub(crate) fn check_all(state: &SandboxState) -> Result, SdkRefusal> { + let mut recoverable = Vec::new(); + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + // Even a published result may still be cleaning its Workspace + // record. Close must not delete/reuse the Sandbox directory in + // that short interval while the original lease is held. + let probe = lease(&entry.path().with_extension("unmount-lock"), LOCK_SH)?; + if probe.is_none() { + if let Some(value) = observed(&entry.path())? { + return Err(progress(&value, true)); + } + } + if let Some(value) = inspect(&entry.path())? { + if value.get_text("state") == Some("Retained") { + recoverable.push(crate::registry::text(&value, "path")?.to_owned()); + } + } + } + } + Ok(recoverable) +} +/// Called under the Sandbox guard, after all before-effect checks. +pub(crate) fn begin( + state: &SandboxState, + token: WorkspaceToken, + path: &str, +) -> Result { + let path_record = record(state, token); + let file = lease(&path_record.with_extension("unmount-lock"), LOCK_EX)? + .ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?; + let attempt = Attempt { + _lease: file, + path: path_record, + started: millis(), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text("Unmounting")), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ]), + )?; + Ok(attempt) +} +/// Publish while the Workspace lease prevents another terminal request. +/// No Sandbox guard is needed to replace this attempt's own result file. +pub(crate) fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> { + match outcome { + Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => { + std::fs::remove_file(&attempt.path).map_err(failed)?; + } + _ => { + let (state, code, message, detail) = match outcome { + Ok(_) => ("Unmounted", "", "", SdkValue::Null), + Err(error) => ( + "Retained", + error.code.as_str(), + error.message.as_str(), + error.detail.clone(), + ), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text(state)), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ("code", SdkValue::text(code)), + ("message", SdkValue::text(message)), + ("detail", detail), + ]), + )?; + } + } + Ok(()) +} +pub(crate) fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> { + std::fs::remove_file(&attempt.path).map_err(failed) +} +pub(crate) fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome { + // Do not replace a known daemon success with a cleanup Busy/Failed. + match &mut outcome { + Ok(SdkValue::Map(fields)) => { + fields.push(("cleanupError".into(), SdkValue::text(error.message))); + } + Err(original) => { + original.detail = SdkValue::map([ + ("original", original.detail.clone()), + ("cleanupError", SdkValue::text(error.message)), + ]); + } + _ => {} + } + outcome +} +pub(crate) fn repeat(value: &SdkValue) -> Outcome { + result(value) +} diff --git a/engine/crates/layerfs-node/src/workspace.rs b/engine/crates/layerfs-node/src/workspace.rs index 829df67f71..8dcb0c4ca9 100644 --- a/engine/crates/layerfs-node/src/workspace.rs +++ b/engine/crates/layerfs-node/src/workspace.rs @@ -8,9 +8,7 @@ //! by a mount and reclaimed by the next `sandbox.create` or //! `runtime.sandbox.reclaim`. use crate::job::{failed, Outcome}; -use crate::registry::{ - lock, naming, random, sandbox, sandbox_live, sandbox_refreshed, text, Mount, -}; +use crate::registry::{lock, naming, random, sandbox, sandbox_live, text, Mount}; use crate::sandbox::{operation, WORKSPACES}; use layerfs_api::internal::workspace; use layerfs_api::sandbox::{values, Unmounted}; @@ -96,8 +94,10 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { if let SdkValue::Map(fields) = &mut value { fields.push(("sandbox".to_owned(), SdkValue::text(state.id.as_str()))); fields.push(("container".to_owned(), SdkValue::text(state.name.as_str()))); - let image = state.runtime.request().image.as_str(); - fields.push(("image".to_owned(), SdkValue::text(image))); + fields.push(( + "image".to_owned(), + SdkValue::text(crate::runtime::image(&state.runtime)), + )); } crate::discovery::save_workspace(&state, &mounted, branch)?; lock(&state.mounts)?.insert( @@ -112,15 +112,21 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { /// One recorded Sandbox of the project: running (`Ok(true)`), or found /// gone or stopped (`Ok(false)`, with its container and whether it stopped). -/// Its container is inspected once; no daemon is dialed. -fn observed(record: &SdkValue, directory: &Path) -> Result, SdkRefusal> { +/// Its container (or local daemon process) is inspected once; no daemon +/// is dialed. +fn observed( + args: &SdkValue, + id: &str, + record: &SdkValue, + directory: &Path, +) -> Result, SdkRefusal> { let cached = lock(&crate::registry::SANDBOXES)?.get(directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => { - let (docker, selector, request) = crate::discovery::runtime(record)?; - layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request).map(drop) - } + // a container, or with the local runtime a daemon process + None => crate::discovery::recorded(args, id, record)? + .reopen() + .map(drop), }; match observed { Ok(()) => Ok(None), @@ -152,7 +158,7 @@ fn records(args: &SdkValue) -> Result Result<(), SdkRefusal> { for (id, directory, value) in records(args)? { - if let Some((container, stopped)) = observed(&value, &directory)? { + if let Some((container, stopped)) = observed(args, &id, &value, &directory)? { crate::sandbox::reclaim(args, &id, &container, stopped)?; } } @@ -168,7 +174,7 @@ pub(crate) fn reclaim_gone(args: &SdkValue) -> Result<(), SdkRefusal> { pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { let mut items = Vec::new(); for (id, directory, value) in records(args)? { - if observed(&value, &directory)?.is_some() { + if observed(args, &id, &value, &directory)?.is_some() { continue; } items.push(SdkValue::map([ @@ -198,7 +204,7 @@ pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { pub(crate) fn reclaim(args: &SdkValue) -> Outcome { let mut reclaimed = Vec::new(); for (id, directory, value) in records(args)? { - let Some((container, stopped)) = observed(&value, &directory)? else { + let Some((container, stopped)) = observed(args, &id, &value, &directory)? else { continue; }; let _placing = crate::discovery::guard(&crate::discovery::project_dir(args)?)?; @@ -252,10 +258,19 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { /// `workspace.unmount(ws, { discard?, force? })`: `Busy` while a background /// process this binding started still runs there, then the daemon's answer. pub(crate) fn unmount(args: &SdkValue) -> Outcome { - let state = sandbox_refreshed(args)?; - let _guard = crate::discovery::guard(&state.directory)?; + let state = sandbox(args)?; let path = text(args, "path")?; + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } + let guard = crate::discovery::guard(&state.directory)?; + // Another process may have registered while this call waited for the + // short admission section. Check again before sending anything. + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } let token = state.token(path)?; + crate::discovery::refresh(&state)?; // The runtime owner knows its own processes; the daemon's kernel probe // still answers Busy for any other holder. let running = crate::exec::running_in(&state, path)?; @@ -273,22 +288,59 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { )]), }); } - let value = { + let attempt = crate::unmount::begin(&state, token, path)?; + drop(guard); + // Only this Workspace's lease remains held. Other Workspaces and + // processes can take Sandbox ownership while the daemon helper waits. + let mut sent = false; + let outcome = (|| { let mut control = crate::trace::control(&state.controls)?; + sent = true; let mut api = workspace(&mut control); // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. if args.get_bool("force") { match api.unmount(token, true).map_err(operation)? { - Some(outcome) => values::force_unmounted(&outcome), - None => values::unmounted(&Unmounted::Closed)?, + Some(outcome) => Ok(values::force_unmounted(&outcome)), + None => values::unmounted(&Unmounted::Closed), } } else { - let discard = args.get_bool("discard"); - values::unmounted(&api.unmount_checked(token, discard).map_err(operation)?)? + match api.unmount_checked(token, args.get_bool("discard")) { + Ok(answer) => values::unmounted(&answer), + Err(error) => Err(operation(error)), + } } + })(); + let outcome = match outcome { + Err(error) if error.code == "Missing" => { + Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])) + } + other => other, }; - std::fs::remove_file(crate::discovery::workspace_file(&state, token)).map_err(failed)?; - lock(&state.mounts)?.remove(path); - Ok(value) + // Preserve the daemon's known outcome before any discovery-lock wait. + // A file/cleanup failure is separate from an already successful detach. + let publication = if !sent { + crate::unmount::cancel(&attempt) + } else { + crate::unmount::finish(&attempt, path, &outcome) + }; + if let Err(error) = publication { + return crate::unmount::cleanup_failure(outcome, error); + } + if outcome.is_ok() { + let cleanup = (|| { + let _guard = crate::discovery::guard(&state.directory)?; + match std::fs::remove_file(crate::discovery::workspace_file(&state, token)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + lock(&state.mounts)?.remove(path); + Ok(()) + })(); + if let Err(error) = cleanup { + return crate::unmount::cleanup_failure(outcome, error); + } + } + outcome } diff --git a/engine/crates/layerfs-node/tests/README.md b/engine/crates/layerfs-node/tests/README.md index 75af6d4e58..1c5c9e54fd 100644 --- a/engine/crates/layerfs-node/tests/README.md +++ b/engine/crates/layerfs-node/tests/README.md @@ -12,6 +12,7 @@ cargo +1.85.1 build --manifest-path Cargo.toml --locked -p layerfs-node cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --tests --no-run timeout 30 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_discovery timeout 90 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_admission +timeout 60 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_local ``` Use the worktree's registered Cargo resources for these commands. Node 20 or @@ -24,6 +25,15 @@ number replacement during inspection, and discovery-lock release while a forked child retains the descriptor. Its C fixture interposes `flock` only in the child Node process; the fixture's fork is bounded and released on test unwinding. +`native_local` covers native Sandbox records (`LAYERFS_SANDBOX=native`) through +`runtime.sandbox.reclaim`: a record naming any directory other than where its +deployment puts that Sandbox is refused and nothing is deleted, the configured +`localDir` must be absolute and plain, a stopped native Sandbox is deleted and its +number freed, a damaged boot or port is refused, a native record is used only by +a native deployment, and a record of the former `local` runtime is never used or +deleted. Record handling is the same code on Linux and macOS, so it runs here; +starting a native Sandbox is supported on macOS (caller identity, loopback NFS) and Linux (root daemon, nonroot 501:20 commands, FUSE). Linux lifecycle qualification also runs `layerfs-sandbox --test native_provider` as root and the SDK smoke with `LAYERFS_SANDBOX=native`. + `native_admission` covers initial connection admission and its `Busy` deadline. The deadline test pays the product's full 60-second readiness wait. The mock daemon has bounded socket waits and stops independently of a test assertion. diff --git a/engine/crates/layerfs-node/tests/native_discovery.rs b/engine/crates/layerfs-node/tests/native_discovery.rs index f0afde037f..8498ae9424 100644 --- a/engine/crates/layerfs-node/tests/native_discovery.rs +++ b/engine/crates/layerfs-node/tests/native_discovery.rs @@ -32,11 +32,19 @@ fn another_404_is_a_failure_that_removes_nothing() { let home = Home::new(); let engine = engine(&home.0, 404, r#"{"message":"page not found"}"#); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(404\)/); + assert.match(answer.refused.message, /page not found/); + "#, None, Duration::from_secs(5), ); - engine.join().unwrap(); + assert!(engine + .join() + .unwrap() + .starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } @@ -113,11 +121,21 @@ fn a_stopped_container_the_engine_will_not_delete_keeps_its_records() { ], ); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(409\)/); + assert.match(answer.refused.message, /cannot remove container: container is running/); + "#, None, Duration::from_secs(5), ); - assert_eq!(engine.join().unwrap().len(), 2); + let requests = engine.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); + assert!(requests[1].starts_with(&format!( + "DELETE /v1.54/containers/{CID}?force=false&v=false HTTP/1.1\r\n" + ))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs new file mode 100644 index 0000000000..b6c8a7ba2d --- /dev/null +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -0,0 +1,274 @@ +//! External tests of the built Node binding's native Sandbox records +//! (LAYERFS_SANDBOX=native). No private Rust source is included: every case +//! writes a Sandbox record as a previous call would have, then calls the +//! addon's public `runtime.sandbox.reclaim` and observes the reply and the +//! files it left. Neither Docker, FUSE, nor a Store is needed. +#![cfg(target_os = "linux")] +mod support; +use layerfs_bridge::control::{Answer, Reply, SdkValue}; +use std::{ + io::Write, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + time::Duration, +}; +use support::*; + +/// This boot's id as a record keeps it: 32 lowercase hex digits. +fn boot() -> String { + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .unwrap() + .trim() + .replace('-', "") +} + +/// Sandbox 1 of `proj` recorded as a `runtime` (native, or the former local) Sandbox +/// in `directory` named `name`, its daemon long gone (no process has id +/// u32::MAX). `edit` changes the fields before the record is written. +fn record( + home: &Home, + runtime: &str, + directory: &str, + name: &str, + edit: impl FnOnce(&mut Vec<(String, SdkValue)>), +) { + let bytes = |n: u8| SdkValue::Bytes(vec![n; 32]); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("id".into(), SdkValue::text("1")), + ("project".into(), SdkValue::text("proj")), + ("name".into(), SdkValue::text(name)), + ("container".into(), SdkValue::text(CID)), + ("runtime".into(), SdkValue::text(runtime)), + ("directory".into(), SdkValue::text(directory)), + ("pid".into(), SdkValue::Int(i64::from(u32::MAX))), + ("start".into(), SdkValue::text("1")), + ("boot".into(), SdkValue::text(boot())), + ("port".into(), SdkValue::Int(40404)), + ("image".into(), SdkValue::text("local")), + ("secret".into(), bytes(1)), + ("peer".into(), bytes(2)), + ("instance".into(), bytes(3)), + ]; + edit(&mut fields); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(home.0.join("proj/sandboxes/1/sandbox.record")) + .unwrap(); + file.write_all( + &Answer { + id: 1, + reply: Reply::Sdk(SdkValue::Map(fields)), + } + .encode() + .unwrap(), + ) + .unwrap(); +} + +/// Replaces (or with `None` removes) one field of a record. +fn set(key: &'static str, value: Option) -> impl FnOnce(&mut Vec<(String, SdkValue)>) { + move |fields| { + fields.retain(|(k, _)| k != key); + if let Some(value) = value { + fields.push((key.into(), value)); + } + } +} + +/// `LAYERFS_LOCAL_DIR` of these tests, created like a deployment's. +fn root(home: &Home) -> PathBuf { + let root = home.0.join("local"); + std::fs::create_dir_all(&root).unwrap(); + root +} + +/// Where a `root` deployment puts Sandbox 1 of `proj`, created with a file. +fn own(root: &Path) -> PathBuf { + let own = root.join("layerfs-proj-1"); + std::fs::create_dir_all(own.join("tmp")).unwrap(); + std::fs::write(own.join("daemon.log"), b"log").unwrap(); + own +} + +/// The public reclaim of a `sandbox` (native or docker) deployment, with +/// `localDir` when given, followed by `then` (JavaScript, `r` is the reply). +fn reclaim(home: &Home, sandbox: &str, local_dir: Option<&str>, then: &str) { + let deploy = match local_dir { + Some(dir) => format!("{{sandbox:{sandbox:?}, localDir:{dir:?}}}"), + None => format!("{{sandbox:{sandbox:?}}}"), + }; + home.node( + &format!( + "const r = await native.backendCall('runtime.sandbox.reclaim', {{...args, deploy:{deploy}}});\n{then}" + ), + None, + Duration::from_secs(10), + ); +} + +const NEITHER: &str = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /neither used nor deleted/);"; +const RECLAIMED: &str = "assert.deepEqual(r, {value:{kind:'ok',reclaimed:['1']}});"; + +fn text(path: &Path) -> &str { + path.to_str().unwrap() +} + +#[test] +fn a_native_record_naming_another_directory_is_neither_used_nor_deleted() { + let home = Home::new(); + let root = root(&home); + // a data directory a damaged or edited record names by its own name + let data = home.0.join("data"); + std::fs::create_dir(&data).unwrap(); + std::fs::write(data.join("keep"), b"x").unwrap(); + let old = home.0.join("old/layerfs-proj-1"); + std::fs::create_dir_all(&old).unwrap(); + let r = text(&root); + for (directory, name) in [ + (text(&data).to_owned(), "data"), + (format!("{r}/data"), "data"), + // another LAYERFS_LOCAL_DIR, another Sandbox, another name + (text(&old).to_owned(), "layerfs-proj-1"), + (format!("{r}/layerfs-proj-2"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1/sub"), "sub"), + ("relative/layerfs-proj-1".to_owned(), "layerfs-proj-1"), + (format!("{r}/../etc/layerfs-proj-1"), "layerfs-proj-1"), + ("/".to_owned(), ""), + ] { + record(&home, "native", &directory, name, |_| {}); + reclaim(&home, "native", Some(r), NEITHER); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "the record of {directory} was removed" + ); + } + assert!( + data.join("keep").exists(), + "a delete reached {}", + data.display() + ); + assert!(old.exists(), "a delete reached {}", old.display()); +} + +#[test] +fn a_native_record_is_used_only_with_its_deployment_s_absolute_local_dir() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + let refused = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r));"; + reclaim(&home, "native", None, refused); + reclaim(&home, "native", Some("relative"), refused); + reclaim( + &home, + "native", + Some(&format!("{}/../local", text(&root))), + refused, + ); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} + +#[test] +fn a_stopped_native_sandbox_where_its_deployment_puts_it_is_deleted_and_reclaimed() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!own.exists(), "the stopped Sandbox's directory is deleted"); + assert!( + !home.0.join("proj/sandboxes/1").exists(), + "its number is free again" + ); + assert!(root.exists(), "LAYERFS_LOCAL_DIR itself stays"); +} + +#[test] +fn a_record_from_before_boot_ids_is_read_without_one() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record( + &home, + "native", + text(&own), + "layerfs-proj-1", + set("boot", None), + ); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} + +#[test] +fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + for (key, value, code) in [ + ("boot", SdkValue::text("not-hex"), "InvalidInput"), + ("port", SdkValue::Int(70000), "Failed"), + ] { + record( + &home, + "native", + text(&own), + "layerfs-proj-1", + set(key, Some(value)), + ); + reclaim( + &home, + "native", + Some(text(&root)), + &format!("assert.equal(r.refused?.code, {code:?}, JSON.stringify(r));"), + ); + assert!(own.join("daemon.log").exists(), "{key}"); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "{key}" + ); + } +} + +#[test] +fn a_docker_record_stays_docker_under_a_native_deployment() { + // the default record of Home is a Docker one: with no Engine at its + // socket it is neither reclaimed nor read as a native Sandbox + let home = Home::new(); + let root = root(&home); + reclaim( + &home, + "native", + Some(text(&root)), + "assert.equal(r.refused?.code, 'Failed', JSON.stringify(r));", + ); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} + +#[test] +fn a_record_of_the_former_local_runtime_is_never_used_nor_deleted() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + let mixed = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /never mixed/); + assert.match(r.refused.message, /neither used nor deleted/);"; + // a record of the former `local` runtime is used by no deployment + record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), mixed); + reclaim(&home, "docker", Some(text(&root)), mixed); + // nor is a native record by any deployment but a native one + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "docker", Some(text(&root)), mixed); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); + // a native deployment uses its own record: stopped, it is reclaimed + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!own.exists()); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} diff --git a/engine/crates/layerfs-node/tests/support/engine.rs b/engine/crates/layerfs-node/tests/support/engine.rs index 6fb656c2d4..302c2f8e38 100644 --- a/engine/crates/layerfs-node/tests/support/engine.rs +++ b/engine/crates/layerfs-node/tests/support/engine.rs @@ -1,5 +1,5 @@ use super::*; -/// The Engine at `home/docker.sock`, answering one request with `status` and `body`. +/// The Engine at `home/docker.sock`, answering one business request with `status` and `body`. pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { let engine = engines(home, vec![(status, body.to_owned())]); thread::spawn(move || engine.join().unwrap().remove(0)) @@ -7,6 +7,7 @@ pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { /// The Engine at `home/docker.sock`, answering each request, one connection /// each, with the next `(status, body)`; it returns their request heads. +/// Version probes get API 1.54 without consuming a response or calling `before`. pub fn engines(home: &Path, responses: Vec<(u16, String)>) -> JoinHandle> { engines_before(home, responses, || {}) } @@ -22,11 +23,9 @@ pub fn engines_before( let mut before = Some(before); let deadline = Instant::now() + Duration::from_secs(3); let mut requests = Vec::new(); - for (status, body) in responses { - let response = format!( - "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); + let mut responses = responses.into_iter().peekable(); + while responses.peek().is_some() { + assert!(Instant::now() < deadline, "bounded engine request deadline"); let stream = loop { match listener.accept() { Ok((stream, _)) => break stream, @@ -43,6 +42,9 @@ pub fn engines_before( stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); let mut request = String::new(); loop { @@ -52,11 +54,23 @@ pub fn engines_before( } request.push_str(&line); } - if let Some(before) = before.take() { - before(); - } + let probe = request.split_whitespace().nth(1) == Some("/version"); + let (status, body) = if probe { + (200, r#"{"ApiVersion":"1.54"}"#.to_owned()) + } else { + if let Some(before) = before.take() { + before(); + } + responses.next().unwrap() + }; + let response = format!( + "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); (&stream).write_all(response.as_bytes()).unwrap(); - requests.push(request); + if !probe { + requests.push(request); + } } requests }) diff --git a/engine/crates/layerfs-overlay/Cargo.toml b/engine/crates/layerfs-overlay/Cargo.toml index a0de581508..842ac36348 100644 --- a/engine/crates/layerfs-overlay/Cargo.toml +++ b/engine/crates/layerfs-overlay/Cargo.toml @@ -11,5 +11,5 @@ blake3 = { version = "=1.8.5", default-features = false } nix = { version = "=0.31.3", default-features = false, features = ["fs"] } rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 747630e708..ff9c423fc5 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -22,5 +22,5 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs"] } [target.'cfg(target_os = "linux")'.dev-dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthread", "signal"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/engine/crates/layerfs-sandbox/Cargo.toml b/engine/crates/layerfs-sandbox/Cargo.toml index e75b3fd4ce..a1f1141f86 100644 --- a/engine/crates/layerfs-sandbox/Cargo.toml +++ b/engine/crates/layerfs-sandbox/Cargo.toml @@ -7,4 +7,13 @@ rust-version.workspace = true license.workspace = true [dependencies] -nix = { version = "=0.31.3", features = ["poll", "socket", "fs"] } +nix = { version = "=0.31.3", features = [ + "poll", + "socket", + "fs", + "signal", + "mount", + "user", + "sched", + "process", +] } diff --git a/engine/crates/layerfs-sandbox/PROVIDERS.md b/engine/crates/layerfs-sandbox/PROVIDERS.md new file mode 100644 index 0000000000..85c740797c --- /dev/null +++ b/engine/crates/layerfs-sandbox/PROVIDERS.md @@ -0,0 +1,134 @@ +# Sandbox runtimes + +Status: Current general guide. + +`backend::runtime` holds the concrete runtime choice and the launched owner. +There are no provider or instance traits: each backend is an ordinary concrete +type and the only dispatch is one `match` per method. + +- `Runtime` is one deployment's selection and is kept for every Sandbox it + starts: `Docker(DockerProvider)` (the default), `MacosNative(NativeMacosProvider)`, + `LinuxNative(NativeLinuxProvider)`, `Namespaces(NamespaceProvider)` (Linux + only) and `Windows`, a placeholder with no backend. + `Runtime::{capabilities, local_directory, native_paths, launch, reopen, + reopen_stopped}` dispatch to them. +- `Sandbox` is what `Runtime::launch`/`reopen` return: `Docker(docker::Sandbox)`, + `Native(native::Sandbox)` or `Namespaces(Box)`, with + `identity`, `endpoint`, `run`, `track`, `log_tail`, `inspect`, `stop`, + `delete`, `scratch_directory`, `local_record` and `record`. + +`layerfs_sandbox::{Runtime, Sandbox}` re-export both. `provider` keeps the +inputs and observations they share: `LaunchRequest`, `ProviderLaunchOptions`, +`LocalRecord`, `DockerRecord`, `SandboxRecord`, `ExecExit`, +`SandboxInspection`, `ProviderCapabilities` and the `LaunchPhase`/`LaunchFailure` +pair. The runtime layer has no daemon control dependency. `backend::HelperSpec` +is the namespace helper's line format; it is compiled and tested on every +platform, while only the helper that reads it is Linux only. + +## Docker + +`Runtime::Docker(DockerProvider::new(docker))` adapts the existing Engine client +and `SandboxRequest`. It shares its lifecycle admission, API 1.48–1.54 +negotiation cache, streamed private archive, logs, exact deployment inspection, +and explicit stop/delete. Call `runtime.launch(LaunchRequest { .. })` with the +executable, private configuration, name, explicit startup wait, and +`ProviderLaunchOptions::Docker { image, store_volume, port, connect_host }`; +Docker validates the immutable image digest, existing volume name, port, +connect_host format and labels before any Engine effect. The runtime does not +decode daemon configuration or perform authentication. + +`Sandbox::run` streams stdout/stderr to caller sinks with bounded windows and +returns `ExecExit` only after one inspection knows the root process exit. It +rejects `stdin=true` before effects: interactive users keep the existing +concrete `create_exec/start/into_parts` stream API. EOF alone is not an exit and +no automatic poll follows an unknown exit. `track` needs no host registration +because container teardown owns descendants; scratch is `/tmp` and +`local_record()` is absent. `inspect` checks the recorded deployment but neither +adopts it nor changes lifecycle admission; unknown PID/exit/storage fields stay +absent. + +## Native + +`Runtime::MacosNative(NativeMacosProvider::new(root))` runs the daemon as this +user and serves loopback NFS; `Runtime::LinuxNative(NativeLinuxProvider::new(root))` +runs a root daemon serving FUSE with ordinary commands as a nonroot identity. +Each refuses the wrong platform before root access, and Linux refuses a non-root +caller the same way. `backend::native` owns the one lifecycle both use; only the +platform facts differ: + +- `backend::macos`: `darwin.rs` (`proc_pidinfo`, `sysctl`, the `mount` table and + `umount -f`) and `mounts.rs` (force detach, deepest first). +- `backend::linux`: `process.rs` (`/proc//stat`, the boot id and the + process table) and `mounts.rs` (`/proc/self/mountinfo`, lazy `MNT_DETACH`). + +The provider validates the command identity, the environment it is given and +the daemon executable, then starts one daemon in a private directory with the +platform's kernel marker (`LAYERFS_KERNEL=fuse` on Linux, `nfs` on macOS); it +rejects conflicting overrides. Launch preserves the original single-attempt +process lifecycle and partial owner on failure. `inspect` reports exact process +liveness; unknown exit/storage facts stay absent. Records remain the Node +binding's responsibility; runtimes neither scan directories nor infer project +ownership. + +## Namespaces (Linux) + +`Runtime::Namespaces(NamespaceProvider::new(root))` is the same local root and +daemon control as `LinuxNative`, with each Sandbox's daemon in its own mount, +PID and IPC namespaces and its own cgroup v2 directory. The VM that holds this +kernel is the boundary (one per Store); the namespaces keep Sandboxes on one +Store out of each other. + +- `backend::namespaces::create` makes the Sandbox's private directory, copies + the daemon to `/bin/layerfs-daemon` (0755), writes the setup (0600) + and a `HelperSpec`, makes `/sys/fs/cgroup/layerfs/`, then spawns the + helper's init stage. +- `backend::namespaces::sandbox` owns the launched handle: the init process's + identity, `cgroup.kill` for stop (a `SIGKILL` to everything the Sandbox + started, `populated 0` awaited), cgroup and directory removal on delete, and + reopen by the recorded process plus its cgroup membership. +- `backend::namespaces::exec` runs one command through the helper's exec stage. +- `src/bin/layerfs-sandbox-helper` performs the calls that act on the calling + process, so the product crate stays `unsafe`-free: the init stage creates the + namespaces, joins the cgroup and starts the inner stage (PID 1), which mounts + the private root (the VM's userland read-only, a private `/proc`, `/dev`, + `/dev/pts`, `/dev/shm`, `/tmp`, plus the Sandbox's directory and the Store's + directory at their own paths), pivots, starts the daemon and reaps; the exec + stage enters the namespaces, sets no-new-privs and runs the command as the + requested uid:gid (its own exit is a signal, never a command's status). + +Preconditions are checked before any effect and refused with the existing +`RuntimeError::Unsupported`, never replaced by a weaker Sandbox: root, the +unified cgroup v2 hierarchy (and `cgroup.kill`), `/dev/fuse`, and +`user.max_user_namespaces=0` — without the last one a command could unshare a +user namespace and hold its own copy of the Workspace mount. The runtime never +sets that limit itself. There is no image: the private root binds the VM's own +userland, so the Sandbox runs the daemon and libraries the VM already has. +`ProviderLaunchOptions::Namespaces { command_identity, environment, store }` +names the Store file whose directory is bound in; `SandboxRecord::Namespaces` +keeps the same physical facts as `LocalRecord`. + +## Windows + +`Runtime::Windows` is a placeholder: `capabilities`, `local_directory`, +`native_paths`, `launch`, `reopen` and `reopen_stopped` all refuse with the +existing `RuntimeError::Unsupported` before any effect, and no `Sandbox` variant +exists for it. No Windows host runtime is implemented; see +[`src/backend/windows/README.md`](src/backend/windows/README.md) for the planned +routes. + +## Failure and records + +A `LaunchFailure` retains the acknowledged `Sandbox`, when any. Its cause may be +`RuntimeError::Docker(Box)`: match the typed variant for +the original Create/Upload/Start/Listener receipt, partial IDs, upload work and +pending source bytes. Upload is within existing `LaunchPhase::Create`; inspect +the typed cause to distinguish it. A partially decoded Create ID is never +adopted, and retaining a handle permits an explicit later observation, never a +replay of creation or Start. + +`Runtime::reopen(id, &SandboxRecord)` validates the exact existing deployment or +process and endpoint without starting it; `reopen_stopped` requires the caller's +prior exact stopped observation and admits only non-forcing Delete. Both accept +`SandboxRecord`, which represents `LocalRecord`, `Namespaces(LocalRecord)` or +`DockerRecord`; each runtime refuses another runtime's record before effects. `Sandbox::record()` exposes the persisted deployment facts. No disk +format changes occur: a record without a `runtime` field is Docker's. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs index 0b1ceaa8a6..88ba0e4eb1 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs @@ -1,11 +1,11 @@ //! One-attempt create/start/stop/delete, with no implicit adoption or volume deletion. use super::{ container_types::{CreatePhase, Sandbox, SandboxCreateFailure, SandboxRequest}, - json::{once, Json}, - strings::quoted, topology::SECCOMP, Docker, }; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; use crate::{ContainerId, RuntimeError, WireFailure}; use std::io::{self, Read, Write}; impl SandboxRequest { diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs index 0d207f1049..16ddfe2e62 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs @@ -1,8 +1,6 @@ //! Structurally observed exact container and loopback-published daemon endpoint. -use super::{ - json::{once, Json}, - Sandbox, -}; +use super::Sandbox; +use crate::backend::json::{once, Json}; use crate::{ContainerId, RuntimeError, WireFailure}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; impl Sandbox { @@ -61,6 +59,20 @@ impl Sandbox { if !self.started || self.stop_attempted || self.delete_attempted { return Err(self.invalid("running endpoint admission")); } + let (_, endpoint) = self.observe(true)?; + endpoint.ok_or_else(|| self.invalid("missing daemon endpoint")) + } + /// One exact deployment/state observation; absent optional facts remain absent. + pub fn inspect(&self) -> Result> { + self.observe(false).map(|(inspection, _)| inspection) + } + fn observe( + &self, + require_running: bool, + ) -> Result<(crate::provider::SandboxInspection, Option), Box> { + if self.deleted { + return Err(self.invalid("deleted container observation")); + } let mut r = self .docker .exchange( @@ -80,6 +92,8 @@ impl Sandbox { let mut topology = true; let mut running = None; let mut endpoint = None; + let mut exit_code = None; + let mut process_id = None; let mut j = Json::new(&mut r.body); j.object(|j, key| { if key.equals("Id") { @@ -108,6 +122,20 @@ impl Sandbox { if key.equals("Running") { once(&mut bits, 1)?; running = Some(j.boolean()?); + } else if key.equals("ExitCode") { + once(&mut bits, 16)?; + if !j.null()? { + exit_code = + Some(i32::try_from(j.integer()?).map_err(|_| { + RuntimeError::Protocol("container exit width") + })?); + } + } else if key.equals("Pid") { + once(&mut bits, 32)?; + process_id = Some( + u32::try_from(j.integer()?) + .map_err(|_| RuntimeError::Protocol("container PID width"))?, + ); } else if key.equals("Paused") || key.equals("Restarting") || key.equals("Dead") @@ -126,7 +154,7 @@ impl Sandbox { } Ok(()) })?; - topology &= bits == 15; + topology &= bits & 15 == 15; } else if key.equals("NetworkSettings") { once(&mut fields, 4)?; let mut bits = 0; @@ -137,6 +165,9 @@ impl Sandbox { j.object(|j, key| { if key.equals(&format!("{}/tcp", self.request.port)) { once(&mut selected, 1)?; + if j.null()? { + return Ok(()); + } let mut count = 0; j.array(|j| { count += 1; @@ -213,27 +244,36 @@ impl Sandbox { Ok(()) })?; j.finish()?; - // the exact recorded container, intact but not running: its daemon - // and every mount in it are gone (Docker restarted, for instance) - if observed == Some(self.id) - && fields == 127 - && image - && topology - && running == Some(false) - { - return Err(RuntimeError::Protocol(crate::types::STOPPED)); - } if observed != Some(self.id) || fields != 127 || !image || !topology - || running != Some(true) + || running.is_none() { return Err(RuntimeError::Protocol( "exact running container observation", )); } - endpoint.ok_or(RuntimeError::Protocol("missing daemon endpoint")) + if require_running && running == Some(false) { + return Err(RuntimeError::Protocol(crate::types::STOPPED)); + } + if running == Some(true) && endpoint.is_none() { + return Err(RuntimeError::Protocol("missing daemon endpoint")); + } + Ok(( + crate::provider::SandboxInspection { + identity: self.id, + running: running.expect("required"), + exit_code: if running == Some(false) { + exit_code + } else { + None + }, + process_id: process_id.filter(|pid| *pid != 0), + scratch_bytes: None, + }, + endpoint, + )) })(); result.map_err(|e| { let mut f = self.selected(r.fail(e, None)); diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs index f4d5e88b4a..1a81fc2baf 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs @@ -1,8 +1,6 @@ //! One exact runtime observation, structurally decoded without retaining Arguments. -use super::{ - json::{once, Json}, - Docker, -}; +use super::Docker; +use crate::backend::json::{once, Json}; use crate::{ContainerId, ExecId, ExecInspection, RuntimeError, WireFailure}; impl Docker { /// Reads one coherent runtime snapshot. PID is diagnostic, never a signal target. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs index e0eedb87b1..e02b4a03b2 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs @@ -7,13 +7,14 @@ mod endpoint; mod headers; mod http; mod inspect; -mod json; mod listener; mod mux; +mod provider; +mod provider_exec; +mod provider_failure; mod request; mod socket; mod streams; -mod strings; mod topology; mod version; pub use archive::{UploadFailure, UploadWork}; @@ -22,6 +23,9 @@ pub use container_types::{ }; pub use http::Docker; pub use listener::{ListenerFailure, ListenerObservation}; +pub use provider::{DockerInstance, DockerProvider}; +pub(crate) use provider_exec::run as run_command; +pub use provider_failure::ProviderFailure; pub use request::{ExecCreateFailure, ExecCreated, ExecRequest}; pub use streams::{ Attached, ExecHandle, InputCloseFailure, InputEnd, OutputFailure, OutputProgress, RuntimeInput, diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs new file mode 100644 index 0000000000..d4171e4002 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs @@ -0,0 +1,124 @@ +//! Docker's existing one-attempt runtime as a concrete backend. +use super::{Docker, Sandbox, SandboxRequest}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::fs::File; + +/// Negotiation and all transfers use the supplied client's original policy and version cache. +#[derive(Clone, Debug)] +pub struct DockerProvider { + docker: Docker, +} +/// The original container owner, including its exact attempted/acknowledged lifecycle state. +pub type DockerInstance = Sandbox; +impl DockerProvider { + /// Negotiation and all transfers use the supplied client's original policy and version cache. + pub fn new(docker: Docker) -> Self { + Self { docker } + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "docker", + isolation: IsolationModel::Container, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let mut phase = LaunchPhase::Validate; + let mut instance = None; + let result = (|| { + let ProviderLaunchOptions::Docker { + image, + store_volume, + port, + connect_host, + } = request.options + else { + return Err(RuntimeError::Unsupported( + "Docker requires Docker launch options", + )); + }; + if let Some(host) = &connect_host { + if host.is_empty() || host.contains(['\0', ' ', '\t', '\r', '\n']) { + return Err(RuntimeError::Protocol("Docker connect_host format")); + } + } + let deployment = SandboxRequest { + image, + store_volume, + port, + labels: request.labels, + }; + deployment.check()?; + if request.startup_timeout.is_zero() + || request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + { + return Err(RuntimeError::Protocol("Docker startup wait/private config")); + } + let docker = self.docker.clone().naming(&request.name)?; + // Opening the executable precedes any Engine effect, as in the original API route. + let mut file = File::open(request.daemon_executable)?; + phase = LaunchPhase::Create; + instance = + Some(docker.create_sandbox(deployment).map_err(|f| { + RuntimeError::Docker(Box::new(super::ProviderFailure::Create(f))) + })?); + let owner = instance.as_mut().expect("acknowledged container"); + owner + .upload(&mut file, &request.daemon_config) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Upload(f))))?; + phase = LaunchPhase::Start; + owner + .start() + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Start(f))))?; + phase = LaunchPhase::Listener; + owner + .wait_listener(request.startup_timeout) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Listener(f))))?; + Ok(()) + })(); + match result { + Ok(()) => Ok(instance.expect("launched")), + Err(cause) => Err(Box::new(LaunchFailure { + phase, + instance: instance + .map(|sandbox| Box::new(crate::backend::Sandbox::Docker(sandbox))), + cause, + })), + } + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + Sandbox::reopen(self.docker.clone(), id, record.clone().into()) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + let request: SandboxRequest = record.clone().into(); + request.check().map_err(WireFailure::local)?; + Ok(Sandbox::stopped(self.docker.clone(), id, request)) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs new file mode 100644 index 0000000000..59a415d3a5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs @@ -0,0 +1,100 @@ +//! Ordinary noninteractive provider execution, with streamed output and exact failure custody. +use super::{ProviderFailure, Sandbox}; +use crate::{provider::ExecExit, ExecRequest, RuntimeError, WireFailure}; +use std::io::Write; +fn failure(container: crate::ContainerId, cause: ProviderFailure) -> Box { + // Scalar facts describe only the deciding exchange. Owned pending bytes, + // diagnostics and fence errors remain in the original nested receipt. + let wire = match &cause { + ProviderFailure::ExecCreate(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecStart(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecInspect { failure, .. } => Some(failure.as_ref()), + _ => None, + }; + let mut outer = WireFailure::local(RuntimeError::Protocol("provider execution")); + if let Some(wire) = wire { + outer.attempted = wire.attempted; + outer.sent_bytes = wire.sent_bytes; + outer.request_header_bytes = wire.request_header_bytes; + outer.sent_body_bytes = wire.sent_body_bytes; + outer.status = wire.status; + outer.observed_exec = wire.observed_exec; + outer.observed_container = wire.observed_container; + outer.requested_exec = wire.requested_exec; + } else { + // Attachment/splitting, output and exit observation follow an attempted Start. + outer.attempted = true; + } + outer.requested_container = Some(container); + outer.requested_exec = outer.requested_exec.or_else(|| match &cause { + ProviderFailure::ExecStart(f) => Some(f.created.identity().1), + ProviderFailure::ExecOutput { handle, .. } + | ProviderFailure::ExecInspect { handle, .. } + | ProviderFailure::ExecExitUnknown { handle, .. } => Some(handle.identity().1), + _ => None, + }); + outer.cause = RuntimeError::Docker(Box::new(cause)); + outer +} +pub(crate) fn run( + sandbox: &Sandbox, + request: &ExecRequest, + mut stdout: &mut (dyn Write + Send), + mut stderr: &mut (dyn Write + Send), +) -> Result> { + if request.stdin { + return Err( + sandbox.invalid("provider run has no stdin source; use ordinary Docker streams") + ); + } + let id = sandbox.identity(); + let created = sandbox + .create_exec(request.clone()) + .map_err(|f| failure(id, ProviderFailure::ExecCreate(f)))?; + let attached = created + .start() + .map_err(|f| failure(id, ProviderFailure::ExecStart(f)))?; + let (input, output, handle) = attached + .into_parts() + .map_err(|f| failure(id, ProviderFailure::ExecSplit(f)))?; + // Stdin was disabled in the creation request; dropping it sends no cancellation. + drop(input); + let progress = match output.copy_to(&mut stdout, &mut stderr) { + Ok(progress) => progress, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecOutput { failure: f, handle }, + )) + } + }; + let inspection = match handle.inspect() { + Ok(inspection) => inspection, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + )) + } + }; + let Some(code) = inspection.known_root_exit() else { + return Err(failure( + id, + ProviderFailure::ExecExitUnknown { + inspection, + handle, + progress, + }, + )); + }; + Ok(ExecExit { + code, + pid: inspection.pid, + stdout_bytes: progress.stdout, + stderr_bytes: progress.stderr, + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs new file mode 100644 index 0000000000..c97c9a596d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs @@ -0,0 +1,32 @@ +//! Original Docker stage failures retained across the provider's common error port. +use super::{ + ExecCreateFailure, ExecHandle, ListenerFailure, OutputFailure, OutputProgress, + SandboxCreateFailure, SplitFailure, StartFailure, UploadFailure, +}; +use crate::{ExecInspection, WireFailure}; +/// No failed creation/Start/stream owner is reconstructed, replayed or silently cleaned up. +#[derive(Debug)] +pub enum ProviderFailure { + Create(Box), + Upload(Box), + Start(Box), + Listener(Box), + ExecCreate(Box), + ExecStart(Box), + ExecSplit(Box), + ExecOutput { + failure: Box, + handle: ExecHandle, + }, + ExecInspect { + failure: Box, + handle: ExecHandle, + progress: OutputProgress, + }, + /// EOF is not proof of process exit. The original handle permits explicit observation. + ExecExitUnknown { + inspection: ExecInspection, + handle: ExecHandle, + progress: OutputProgress, + }, +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs index b7622d7efc..5b283c9c85 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs @@ -1,58 +1,11 @@ //! One ordinary non-TTY Exec creation; exact acknowledged runtime custody. -use super::{ - json::{once, Json}, - strings::quoted, - Docker, -}; -use crate::{CommandIdentity, ContainerId, ExecId, RuntimeError, WireFailure}; -use std::{ - fmt, - io::{self, Write}, -}; -/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. -#[derive(Clone)] -pub struct ExecRequest { - pub identity: CommandIdentity, - pub arguments: Vec, - pub environment: Vec, - pub directory: String, - pub stdin: bool, -} -impl fmt::Debug for ExecRequest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ExecRequest") - .field("identity", &self.identity) - .field("arguments_count", &self.arguments.len()) - .field("environment_count", &self.environment.len()) - .field("directory", &self.directory) - .field("stdin", &self.stdin) - .finish() - } -} +use super::Docker; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; +pub use crate::ExecRequest; +use crate::{ContainerId, ExecId, RuntimeError, WireFailure}; +use std::io::{self, Write}; impl ExecRequest { - fn check(&self) -> Result<(), RuntimeError> { - self.identity.check()?; - if self.arguments.is_empty() - || self.arguments[0].is_empty() - || !std::path::Path::new(&self.directory).is_absolute() - || self - .arguments - .iter() - .chain(&self.environment) - .chain(std::iter::once(&self.directory)) - .any(|v| v.as_bytes().contains(&0)) - { - return Err(RuntimeError::Protocol("ordinary command fields")); - } - if self - .environment - .iter() - .any(|v| !v.contains('=') || v.starts_with('=')) - { - return Err(RuntimeError::Protocol("ordinary environment field")); - } - Ok(()) - } fn encode(&self, out: &mut dyn Write) -> io::Result<()> { out.write_all(b"{\"AttachStdout\":true,\"AttachStderr\":true,\"AttachStdin\":")?; out.write_all(if self.stdin { b"true" } else { b"false" })?; diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs index 9ee9ae628d..3626ab3bbe 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs @@ -1,8 +1,6 @@ //! Exact observed daemon identity, security and kernel-access settings, and volume placement. -use super::{ - json::{once, Json}, - SandboxRequest, -}; +use super::SandboxRequest; +use crate::backend::json::{once, Json}; use crate::RuntimeError; use std::io::Read; /// The container's whole system-call filter. It denies creation of a user diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs index ae3504b098..973623a478 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs @@ -1,9 +1,9 @@ //! The Engine API version a client speaks: the daemon's, up to the newest this client targets. use super::{ http::{failure, Mode}, - json::{once, Json}, Docker, }; +use crate::backend::json::{once, Json}; use crate::{ErrorDiagnostic, RuntimeError, WireFailure}; /// The oldest Engine API minor version (1.N) this client speaks: Engine 28.0. /// Before it, uploading the daemon fails (`getent unable to find entry "0:0"`: diff --git a/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs b/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs new file mode 100644 index 0000000000..6e1f4d4e08 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/helper_spec.rs @@ -0,0 +1,159 @@ +//! The init stage's inputs: absolute paths only, one per line, in a 0600 file: +//! what it mounts, the three directories of the Sandbox root's overlay (the +//! unpacked image as its read-only lower layer, and the Sandbox's own upper +//! and work directories), the daemon it starts, and the cgroup it joins. +//! +//! A format, not a platform behaviour: it is compiled wherever the crate is and +//! tested without root; only the helper that reads it is Linux only. +//! +//! The daemon's setup (its keys and every secret in it) is a separate file the +//! helper never reads; this spec names only what the init stage mounts and +//! runs, so it carries no secret. Values may contain `=` (the key ends at the +//! first one); a path may not contain a control character, because the format +//! is line based. +use crate::RuntimeError; +use std::{ + os::unix::ffi::OsStrExt, + path::{Component, Path, PathBuf}, +}; + +/// The fields, in the order they are written. +const FIELDS: [&str; 9] = [ + "root", "sandbox", "store", "daemon", "config", "cgroup", "lower", "upper", "work", +]; + +/// One init stage's decoded inputs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct HelperSpec { + /// The private root the mount namespace pivots to; a tmpfs is mounted on + /// it by the init stage. Sibling of `sandbox`, never inside it. + pub root: PathBuf, + /// The Sandbox directory, bound at its own path inside the private root, + /// so the daemon's setup, overlay and Workspace mounts keep one naming. + pub sandbox: PathBuf, + /// The Store's directory, bound read-write: SQLite's WAL and the writers' + /// directory lock then see the same inode as every other opener. + pub store: PathBuf, + /// The daemon executable inside the private root. + pub daemon: PathBuf, + /// The daemon's setup file, 0600. + pub config: PathBuf, + /// The Sandbox's cgroup v2 directory. + pub cgroup: PathBuf, + /// The unpacked image: the read-only lower layer of the Sandbox's root. + pub lower: PathBuf, + /// The Sandbox's own writable layer, and the overlay's work directory; + /// both inside its private directory, deleted with it. + pub upper: PathBuf, + pub work: PathBuf, +} + +impl HelperSpec { + /// Encodes the six fields; refuses a path that is not absolute, has `.` + /// or `..` components, or holds a control character. + pub fn encode(&self) -> Result, RuntimeError> { + let mut out = Vec::new(); + for (index, (field, path)) in self.fields().into_iter().enumerate() { + check(path)?; + if index >= LAYERS { + check_overlay(path)?; + } + out.extend_from_slice(field.as_bytes()); + out.push(b'='); + let text = path.to_str().ok_or(RuntimeError::Protocol(PATH))?; + out.extend_from_slice(text.as_bytes()); + out.push(b'\n'); + } + Ok(out) + } + + /// Decodes a spec written by [`Self::encode`]: exactly the six fields, + /// each once, in any order. + pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MAX { + return Err(RuntimeError::Protocol(PATH)); + } + let text = std::str::from_utf8(bytes).map_err(|_| RuntimeError::Protocol(PATH))?; + let mut values: [Option; 9] = Default::default(); + for line in text.lines() { + let (key, value) = line.split_once('=').ok_or(RuntimeError::Protocol(FIELD))?; + let index = FIELDS + .iter() + .position(|name| *name == key) + .ok_or(RuntimeError::Protocol(FIELD))?; + if values[index].is_some() { + return Err(RuntimeError::Protocol(FIELD)); + } + let path = PathBuf::from(value); + check(&path)?; + values[index] = Some(path); + } + let mut take = |index: usize| values[index].take().ok_or(RuntimeError::Protocol(FIELD)); + let spec = Self { + root: take(0)?, + sandbox: take(1)?, + store: take(2)?, + daemon: take(3)?, + config: take(4)?, + cgroup: take(5)?, + lower: take(6)?, + upper: take(7)?, + work: take(8)?, + }; + for path in [&spec.lower, &spec.upper, &spec.work] { + check_overlay(path)?; + } + Ok(spec) + } + + fn fields(&self) -> [(&'static str, &Path); 9] { + [ + (FIELDS[0], &self.root), + (FIELDS[1], &self.sandbox), + (FIELDS[2], &self.store), + (FIELDS[3], &self.daemon), + (FIELDS[4], &self.config), + (FIELDS[5], &self.cgroup), + (FIELDS[6], &self.lower), + (FIELDS[7], &self.upper), + (FIELDS[8], &self.work), + ] + } +} + +/// The largest spec accepted: six absolute paths, each well under a page. +const MAX: usize = 8192; + +/// Rejects a path the line format cannot carry or the init stage cannot +/// mount: relative, `..`/`.`, a control character, or longer than [`MAX`]. +fn check(path: &Path) -> Result<(), RuntimeError> { + let text = path.to_str().ok_or(RuntimeError::Protocol(PATH))?; + if !path.is_absolute() + || text.len() > MAX + || text + .chars() + .any(|c| c == '\n' || c == '\r' || c == '\0' || c == '\t') + || (path.components()).any(|c| matches!(c, Component::CurDir | Component::ParentDir)) + { + return Err(RuntimeError::Protocol(PATH)); + } + Ok(()) +} + +/// The three fields from here on are the Sandbox root's overlay. +const LAYERS: usize = 6; + +/// The three overlay paths travel in one mount option string, where `,` +/// separates options and `:` would separate layers, so neither may appear. +fn check_overlay(path: &Path) -> Result<(), RuntimeError> { + let bytes = path.as_os_str().as_bytes(); + if bytes.iter().any(|byte| matches!(byte, b',' | b':')) { + return Err(RuntimeError::Protocol("an overlay path holds a `,` or `:`")); + } + Ok(()) +} + +/// Path text is UTF-8, absolute and free of control characters. +const PATH: &str = "helper spec path"; +/// A line that is not one known field, or a field given twice or not at all. +const FIELD: &str = "helper spec field"; diff --git a/engine/crates/layerfs-sandbox/src/backend/identity.rs b/engine/crates/layerfs-sandbox/src/backend/identity.rs new file mode 100644 index 0000000000..8e1c174c73 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/identity.rs @@ -0,0 +1,16 @@ +//! A fresh random identity in the Engine's form (64 lowercase hex, not all +//! zero), so records, names and the cgroup directory treat a local Sandbox +//! like a container. +use crate::{ContainerId, RuntimeError}; +use std::{fs::File, io::Read}; + +pub(crate) fn random() -> Result { + let mut bytes = [0u8; 32]; + File::open("/dev/urandom")?.read_exact(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let text: String = (bytes.iter()) + .flat_map(|b| [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]]) + .map(char::from) + .collect(); + ContainerId::parse(&text) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/json.rs b/engine/crates/layerfs-sandbox/src/backend/json.rs similarity index 100% rename from engine/crates/layerfs-sandbox/src/backend/docker/json.rs rename to engine/crates/layerfs-sandbox/src/backend/json.rs diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs new file mode 100644 index 0000000000..46c121526d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs @@ -0,0 +1,7 @@ +//! Linux platform facts for the native runtime: the daemon runs as root and +//! serves FUSE, so its process facts come from `/proc` and its mounts are +//! detached lazily. Only facts live here; the Sandbox lifecycle is shared +//! with macOS in [`super::native`]. +pub(crate) mod mounts; +mod process; +pub(crate) use process::{boot, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs new file mode 100644 index 0000000000..2fc4e41737 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs @@ -0,0 +1,108 @@ +//! Kernel mounts left under a native Sandbox's directory once its daemon is +//! gone. A container's mounts end with its namespace; this daemon's stay in +//! the one kernel until detached. The table is `/proc/self/mountinfo` and a +//! mount is detached lazily with `umount2(MNT_DETACH)`. +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// Mount points at or under `directory`, deepest first, from this +/// process's mount table. +pub(crate) fn under(directory: &Path) -> io::Result> { + let table = std::fs::read_to_string("/proc/self/mountinfo")?; + let mut found = Vec::new(); + for line in table.lines() { + // field 5 is the mount point, with space, tab, newline and backslash octal-escaped + let Some(point) = line.split(' ').nth(4) else { + continue; + }; + let point = PathBuf::from(unescape(point)); + if point.starts_with(directory) { + found.push(point); + } + } + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + +/// The filesystem type holding `path`: the deepest mount point in this +/// process's mount table that is one of its ancestors, and the type recorded +/// after the separator. A namespaces Sandbox's root is an overlay whose +/// writable layer must live on a filesystem that can be an overlay upper. +pub(crate) fn filesystem(path: &Path) -> io::Result { + let table = std::fs::read_to_string("/proc/self/mountinfo")?; + let mut deepest: Option<(usize, String)> = None; + for line in table.lines() { + let fields: Vec<&str> = line.split(' ').collect(); + let Some(separator) = fields.iter().position(|field| *field == "-") else { + continue; + }; + let (Some(point), Some(kind)) = (fields.get(4), fields.get(separator + 1)) else { + continue; + }; + let point = PathBuf::from(unescape(point)); + if !path.starts_with(&point) { + continue; + } + let depth = point.components().count(); + if deepest + .as_ref() + .is_none_or(|(deepest, _)| depth >= *deepest) + { + deepest = Some((depth, (*kind).to_owned())); + } + } + deepest.map(|(_, kind)| kind).ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("{} is not under any mount of this process", path.display()), + ) + }) +} + +fn unescape(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let octal = bytes + .get(i + 1..i + 4) + // one byte is at most octal 377, so its first digit is 0 to 3 + .filter(|digits| { + bytes[i] == b'\\' + && digits[0] <= b'3' + && digits.iter().all(|d| (b'0'..=b'7').contains(d)) + }); + match octal { + Some(digits) => { + out.push(digits.iter().fold(0u8, |n, d| n * 8 + (d - b'0'))); + i += 4; + } + None => { + out.push(bytes[i]); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Detaches every mount under `directory` once (lazy unmount: a process +/// still inside keeps its view; new lookups no longer reach it), then checks +/// that none is left. +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + use nix::mount::{umount2, MntFlags}; + for point in under(directory)? { + match umount2(&point, MntFlags::MNT_DETACH) { + Ok(()) | Err(nix::errno::Errno::EINVAL) | Err(nix::errno::Errno::ENOENT) => {} + Err(errno) => return Err(errno.into()), + } + } + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::other("a mount under the local Sandbox remains")) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/process.rs b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs new file mode 100644 index 0000000000..6882044b13 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs @@ -0,0 +1,53 @@ +//! Linux process facts for the native runtime: state, process group and start +//! time from `/proc//stat`, this boot's identity from +//! `/proc/sys/kernel/random/boot_id`, and every process from `/proc`. +use super::super::native::process::Stat; +use std::io; +use std::sync::OnceLock; + +/// This boot's id (`/proc/sys/kernel/random/boot_id`, a UUID) as a number, +/// read once per process. +pub(crate) fn boot() -> io::Result { + static BOOT: OnceLock = OnceLock::new(); + if let Some(boot) = BOOT.get() { + return Ok(*boot); + } + let text = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?; + let hex: String = text.trim().chars().filter(|c| *c != '-').collect(); + let boot = (hex.len() == 32) + .then(|| u128::from_str_radix(&hex, 16).ok()) + .flatten() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "boot id"))?; + Ok(*BOOT.get_or_init(|| boot)) +} + +/// State, process group and start time of `pid`. The command name may hold +/// spaces and parentheses, so fields are counted after its last `)`. +pub(crate) fn stat(pid: u32) -> io::Result { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat"))?; + let invalid = || io::Error::new(io::ErrorKind::InvalidData, "process stat fields"); + let rest = text.rsplit_once(')').ok_or_else(invalid)?.1; + // field 3 (the state) is the first after the name, so field n is at n - 3 + let fields: Vec<&str> = rest.split_ascii_whitespace().collect(); + let field = |n: usize| fields.get(n - 3).copied().ok_or_else(invalid); + Ok(Stat { + state: field(3)?.bytes().next().ok_or_else(invalid)?, + group: field(5)?.parse().map_err(|_| invalid())?, + start: field(22)?.parse().map_err(|_| invalid())?, + }) +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +pub(crate) fn processes() -> io::Result> { + let mut table = Vec::new(); + for item in std::fs::read_dir("/proc")? { + let Some(pid) = item?.file_name().to_str().and_then(|n| n.parse().ok()) else { + continue; + }; + if let Ok(stat) = stat(pid) { + table.push((pid, stat)); + } + } + Ok(table) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/listener.rs b/engine/crates/layerfs-sandbox/src/backend/listener.rs new file mode 100644 index 0000000000..c8fe639ce1 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/listener.rs @@ -0,0 +1,68 @@ +//! The daemon's startup line: a bounded wait for the loopback address it +//! prints once it listens. Both local backends spawn a daemon whose output is +//! a log file, so both read that same marker from it, each look taking only +//! what was appended since the last one. Nothing is retried: the wait ends at +//! the marker, at the daemon's exit, or at the caller's deadline. +use crate::RuntimeError; +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom}, + path::Path, + time::{Duration, Instant}, +}; + +/// The daemon's startup line; the address follows it. +pub(crate) const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; +/// Pause between two reads of a starting daemon's output. +const PAUSE: Duration = Duration::from_millis(10); + +/// Waits until `log` names the daemon's loopback address, the daemon is no +/// longer alive (as `alive` reports), or `wait` ends. +pub(crate) fn await_listener( + log: &Path, + wait: Duration, + alive: &dyn Fn() -> bool, +) -> Result { + let deadline = Instant::now() + wait; + let mut file = File::open(log)?; + let mut offset = 0u64; + let mut line = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + file.seek(SeekFrom::Start(offset))?; + loop { + let n = file.read(&mut chunk)?; + if n == 0 { + break; + } + offset += n as u64; + for &byte in &chunk[..n] { + if byte != b'\n' { + // a startup line is short; a longer one is not the marker + if line.len() < 256 { + line.push(byte); + } + continue; + } + if let Some(address) = line.strip_prefix(MARKER) { + let address = std::str::from_utf8(address) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|a| a.ip().is_loopback() && a.port() != 0) + .ok_or(RuntimeError::Protocol("local daemon loopback listener"))?; + return Ok(address.port()); + } + line.clear(); + } + } + if !alive() { + return Err(RuntimeError::Protocol( + "local daemon exited before its listener line", + )); + } + if Instant::now() >= deadline { + return Err(io::Error::from(io::ErrorKind::TimedOut).into()); + } + std::thread::sleep(PAUSE); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs new file mode 100644 index 0000000000..99ba3e4a52 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs @@ -0,0 +1,277 @@ +//! macOS facts the native runtime reads from `/proc` on Linux: a process's +//! state, group and start time (`proc_pidinfo`, a system call: `ps` is setuid +//! root, which a Seatbelt-sandboxed caller may not execute), this boot's +//! identity (`sysctl`), the mount table (`mount`) and a forced unmount +//! (`umount -f`). Each tool is one run with a fixed path and the C locale, +//! read once; nothing is retried. +use crate::backend::native::process::Stat; +use nix::{ + fcntl::{fcntl, FcntlArg, OFlag}, + libc, +}; +use std::{ + io::{self, Read}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +const SYSCTL: &str = "/usr/sbin/sysctl"; +const MOUNT: &str = "/sbin/mount"; +const UMOUNT: &str = "/sbin/umount"; +const UNMOUNT_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const OUTPUT: usize = 2048; + +/// One tool's standard output, or its failure with what it said. +fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { + let out = Command::new(program) + .args(args) + .env_clear() + .env("LC_ALL", "C") + .env("PATH", "/usr/bin:/bin:/usr/sbin:/sbin") + .stdin(Stdio::null()) + .output()?; + Ok(( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + )) +} + +fn invalid(what: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, what) +} + +/// State, process group and start time of `pid` from the kernel +/// (`proc_pidinfo`, `PROC_PIDTBSDINFO`); `NotFound` when no such process +/// exists. Not `ps`: it is setuid root, and a sandboxed caller (dsh's +/// Seatbelt profile for agents' bash) may not execute it. +#[allow(unsafe_code)] +pub(crate) fn stat(pid: u32) -> io::Result { + let id = i32::try_from(pid).map_err(|_| invalid("process id width"))?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: the buffer is a proc_bsdinfo of exactly `size` bytes. + let got = unsafe { + libc::proc_pidinfo( + id, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if got <= 0 { + let error = io::Error::last_os_error(); + return Err(match error.raw_os_error() { + Some(libc::ESRCH) | Some(0) | None => io::Error::from(io::ErrorKind::NotFound), + _ => error, + }); + } + if got != size { + return Err(invalid("proc_bsdinfo size")); + } + // SAFETY: the kernel filled all `size` bytes. + let info = unsafe { info.assume_init() }; + if info.pbi_pid != pid { + return Err(invalid("proc_bsdinfo pid")); + } + Ok(from_info(&info)) +} + +/// `ps`'s state letters for `p_stat`, and the start time in microseconds: +/// equal for the same process, different for another under the same id. +fn from_info(info: &libc::proc_bsdinfo) -> Stat { + let state = match info.pbi_status { + 1 => b'I', + 2 => b'R', + 3 => b'S', + 4 => b'T', + libc::SZOMB => b'Z', + _ => b'?', + }; + Stat { + state, + group: info.pbi_pgid, + start: info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec, + } +} + +/// Every process now, with its stat (`proc_listallpids`); a process that +/// ends or is not readable between the listing and its read is left out. +#[allow(unsafe_code)] +pub(crate) fn processes() -> io::Result> { + // SAFETY: a null buffer asks only for the count. + let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + let mut pids = vec![0 as libc::pid_t; count as usize + 64]; + let bytes = (pids.len() * std::mem::size_of::()) as libc::c_int; + // SAFETY: `pids` holds `bytes` bytes of pid_t. + let listed = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + if listed < 0 { + return Err(io::Error::last_os_error()); + } + pids.truncate((listed as usize).min(pids.len())); + Ok(pids + .into_iter() + .filter(|pid| *pid > 0) + .filter_map(|pid| { + let pid = pid as u32; + stat(pid).ok().map(|stat| (pid, stat)) + }) + .collect()) +} + +/// This boot's identity as a number, read once per process: +/// `kern.bootsessionuuid`, else `kern.boottime` (seconds and microseconds). +pub(crate) fn boot() -> io::Result { + static BOOT: OnceLock = OnceLock::new(); + if let Some(boot) = BOOT.get() { + return Ok(*boot); + } + let uuid = output(SYSCTL, &["-n", "kern.bootsessionuuid"]) + .ok() + .and_then(|(ran, text)| if ran { uuid(&text) } else { None }); + let boot = match uuid { + Some(boot) => boot, + None => { + let (_, text) = output(SYSCTL, &["-n", "kern.boottime"])?; + boottime(&text).ok_or_else(|| invalid("kern.boottime"))? + } + }; + Ok(*BOOT.get_or_init(|| boot)) +} + +fn uuid(text: &str) -> Option { + let hex: String = text.trim().chars().filter(|c| *c != '-').collect(); + (hex.len() == 32) + .then(|| u128::from_str_radix(&hex, 16).ok()) + .flatten() +} + +/// `{ sec = 1729500000, usec = 123456 } Tue Oct 21 ...` as sec * 10^6 + usec. +fn boottime(text: &str) -> Option { + let field = |name: &str| -> Option { + let after = text.split(name).nth(1)?; + let digits: String = after + .trim_start_matches([' ', '=']) + .chars() + .take_while(char::is_ascii_digit) + .collect(); + digits.parse().ok() + }; + Some(field("sec")? * 1_000_000 + field("usec")?) +} + +/// Every mount point now, from `mount`'s ` on ()`. +pub(crate) fn mount_points() -> io::Result> { + let (ran, text) = output(MOUNT, &[])?; + if !ran { + return Err(io::Error::other("mount listing failed")); + } + Ok(text.lines().filter_map(mount_point).collect()) +} + +fn mount_point(line: &str) -> Option { + let (_, rest) = line.split_once(" on ")?; + let (point, _) = rest.rsplit_once(" (")?; + Some(PathBuf::from(point)) +} + +/// Forces the mount at `point` off once; a point no longer mounted is done. +pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { + static RETAINED: Mutex> = Mutex::new(Vec::new()); + { + let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let mut child = Command::new(UMOUNT) + .arg("-f") + .arg(point) + .env_clear() + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stderr = child.stderr.take().and_then(|pipe| { + // If nonblocking setup fails, drop the pipe rather than risk an + // unbounded read while stopping a Sandbox. + fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)) + .ok() + .map(|_| pipe) + }); + let mut output = Vec::new(); + let deadline = Instant::now() + UNMOUNT_WAIT; + let status = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) => { + break Err(io::Error::new( + io::ErrorKind::TimedOut, + "umount exceeded 60s", + )) + } + Err(error) => break Err(error), + } + }; + let status = match status { + Ok(status) => status, + Err(error) => { + let pid = child.id(); + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) | Err(_) => break false, + } + }; + let disposition = if reaped { + "exit reaped" + } else { + RETAINED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; child custody retained (possible kernel wait)" + }; + return Err(io::Error::new( + error.kind(), + format!( + "umount -f {}: {error}; pid={pid}; KILL={kill:?}; {disposition}: {}", + point.display(), + String::from_utf8_lossy(&output).trim() + ), + )); + } + }; + unmount_output(&mut stderr, &mut output); + if status.success() || !mount_points()?.iter().any(|p| p == point) { + return Ok(()); + } + Err(io::Error::other(format!( + "umount -f {}: {status}: {}", + point.display(), + String::from_utf8_lossy(&output).trim() + ))) +} + +fn unmount_output(stderr: &mut Option, output: &mut Vec) { + let Some(pipe) = stderr else { return }; + let mut bytes = [0; OUTPUT]; + if let Ok(count) = pipe.read(&mut bytes) { + output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs new file mode 100644 index 0000000000..f9bf26868c --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs @@ -0,0 +1,7 @@ +//! macOS platform facts for the native runtime: the daemon serves loopback +//! NFS as this user, so its process facts come from `proc_pidinfo`/`sysctl` +//! and its mounts from `mount`/`umount -f`. Only facts live here; the Sandbox +//! lifecycle is shared with Linux in [`super::native`]. +mod darwin; +pub(crate) mod mounts; +pub(crate) use darwin::{boot, force_unmount, mount_points, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs new file mode 100644 index 0000000000..23547afae8 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs @@ -0,0 +1,33 @@ +//! Kernel mounts left under a native Sandbox's directory once its daemon is +//! gone. A container's mounts end with its namespace; this daemon's stay in +//! the one kernel ("transport endpoint is not connected") until detached. The +//! table is `mount`'s output and a mount is forced off with `umount -f`. +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// Mount points at or under `directory`, deepest first, from this +/// process's mount table. +pub(crate) fn under(directory: &Path) -> io::Result> { + let mut found: Vec = (super::mount_points()?.into_iter()) + .filter(|point| point.starts_with(directory)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + +/// Unmounts every mount under `directory` once with `umount -f` (a dead +/// NFS export's mount is forced off; macOS has no lazy detach), then checks +/// that none is left. +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + for point in under(directory)? { + super::force_unmount(&point)?; + } + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::other("a mount under the local Sandbox remains")) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/mod.rs b/engine/crates/layerfs-sandbox/src/backend/mod.rs index 509d7e29fa..75d331e04a 100644 --- a/engine/crates/layerfs-sandbox/src/backend/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,2 +1,22 @@ -//! Concrete ordinary runtime providers. +//! Concrete ordinary runtime backends and the enum that selects one. pub mod docker; +mod helper_spec; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod identity; +mod json; +#[cfg(target_os = "linux")] +pub(crate) mod linux; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod listener; +#[cfg(target_os = "macos")] +pub(crate) mod macos; +#[cfg(target_os = "linux")] +pub mod namespaces; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub mod native; +pub mod runtime; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod streams; +mod strings; +pub use helper_spec::HelperSpec; +pub use runtime::{Runtime, Sandbox}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs new file mode 100644 index 0000000000..bc72e71cd7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/cgroup.rs @@ -0,0 +1,84 @@ +//! One cgroup v2 directory per Sandbox: the whole Sandbox ends with one +//! write to `cgroup.kill`, and nothing it started can leave that group, so a +//! process that left its process group still ends with it. The hierarchy is +//! the host's unified one; there is no second, nested controller setup. +use crate::ContainerId; +use std::{ + fs, + io::{self, Write}, + os::unix::fs::DirBuilderExt, + path::{Path, PathBuf}, +}; + +/// This host's unified cgroup v2 hierarchy. +pub(crate) const ROOT: &str = "/sys/fs/cgroup"; +/// The directory this runtime owns under it, one child per Sandbox. +const GROUP: &str = "layerfs"; + +/// Sandbox `id`'s cgroup directory. +pub(crate) fn path(id: &ContainerId) -> PathBuf { + Path::new(ROOT).join(GROUP).join(id.as_str()) +} + +/// Makes this runtime's own directory once (idempotent) and returns Sandbox +/// `id`'s directory, made here and empty, with the files this runtime needs. +pub(crate) fn create(id: &ContainerId) -> io::Result { + let parent = Path::new(ROOT).join(GROUP); + match fs::DirBuilder::new().mode(0o755).create(&parent) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + let directory = path(id); + fs::DirBuilder::new().mode(0o755).create(&directory)?; + // a kernel without cgroup.kill (before 5.14) cannot end a Sandbox as a whole + if !directory.join("cgroup.kill").exists() { + let _ = fs::remove_dir(&directory); + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "this kernel's cgroup v2 has no cgroup.kill", + )); + } + Ok(directory) +} + +/// Ends every process of the Sandbox with one write: the kernel sends +/// `SIGKILL` to all of them, descendants included, however they left their +/// process groups. +pub(crate) fn kill(directory: &Path) -> io::Result<()> { + let mut file = fs::OpenOptions::new() + .write(true) + .open(directory.join("cgroup.kill"))?; + file.write_all(b"1") +} + +/// The number of processes the Sandbox's group still has. +pub(crate) fn populated(directory: &Path) -> io::Result { + let events = fs::read_to_string(directory.join("cgroup.events"))?; + let mut count = None; + for line in events.lines() { + if let Some(value) = line.strip_prefix("populated ") { + count = value.trim().parse().ok(); + } + } + count.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "cgroup.events populated")) +} + +/// Removes the Sandbox's directory once nothing is in it. A group that still +/// has processes is refused rather than reported as gone. +pub(crate) fn remove(directory: &Path) -> io::Result<()> { + match populated(directory) { + Ok(0) => {} + Ok(_) => { + return Err(io::Error::other( + "a process of the namespaces Sandbox still runs after SIGKILL", + )) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + match fs::remove_dir(directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + other => other, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs new file mode 100644 index 0000000000..0ad0c7c5c9 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/create.rs @@ -0,0 +1,283 @@ +//! One namespace Sandbox start: its private directory (setup, executable, +//! overlay, mounts), its cgroup and one spawn of the init stage, then the +//! wait for its daemon's listener. Nothing is retried, and a failure returns +//! what it made so the caller can end and delete it once. +use super::{ + cgroup, host, + paths::Paths, + rootfs::{self, ImageRoot}, + sandbox::Sandbox, +}; +use crate::backend::listener; +use crate::backend::native::{search::search_path, Local, LocalPhase, LocalProcess}; +use crate::backend::{identity, HelperSpec}; +use crate::{CommandIdentity, Image, RuntimeError}; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, +}; + +/// What one namespaces launch was asked for, after validation. +#[derive(Clone, Debug)] +pub(crate) struct Options { + pub(crate) command_identity: CommandIdentity, + pub(crate) environment: Vec<(String, String)>, + /// The exact Store file the daemon opens; its directory is bound in. + pub(crate) store: PathBuf, + /// The image digest the Sandbox's root is an overlay of, and the + /// docker-archive in the bundle that holds it. + pub(crate) image: Image, + pub(crate) archive: PathBuf, +} + +/// A failed start. `sandbox` holds what it made (its directory, cgroup and, +/// once spawned, its init stage) for the caller to end and delete once. +#[derive(Debug)] +pub(crate) struct CreateFailure { + pub(crate) phase: LocalPhase, + pub(crate) sandbox: Option, + pub(crate) cause: RuntimeError, +} + +/// The init stage of `request`'s Sandbox, started once over `local`'s root. +pub(crate) fn create( + local: &Local, + helper: &Path, + request: &crate::provider::LaunchRequest, + options: &Options, +) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(CreateFailure { + phase, + sandbox, + cause, + }) + }; + let (paths, image) = start_request(local, helper, request, options) + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let cgroup = cgroup::path(&id); + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.local.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted( + id, + paths.clone(), + cgroup.clone(), + request.labels.clone(), + helper.to_owned(), + image.clone(), + ); + let log = match prepare(&paths, request, options, &id, &image) { + Ok(log) => log, + Err(cause) => return Err(fail(LocalPhase::Create, Some(sandbox), cause)), + }; + match cgroup::create(&id) { + Ok(_) => {} + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + } + let spawned = log.try_clone().and_then(|output| { + Command::new(helper) + .arg("init") + .arg("--spec") + .arg(&paths.spec) + .env_clear() + .env("PATH", search_path()) + .envs(options.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.local.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + match LocalProcess::observe(child.id()) { + Ok(init) => sandbox.init = Some(init), + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + } + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.local.log, request.startup_timeout, &|| { + sandbox.reap(); + sandbox.init.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } +} + +/// Everything a start refuses before it makes anything: the request's own +/// fields, this host's runtime prerequisites, the Store's directory against +/// the private root's, the helper, and the image the bundle carries. Getting +/// the image's root may unpack it, once per digest for the whole runtime root. +fn start_request( + local: &Local, + helper: &Path, + request: &crate::provider::LaunchRequest, + options: &Options, +) -> Result<(Paths, ImageRoot), RuntimeError> { + let paths = { + let named = local.clone().naming(&request.name)?; + Paths::of(named.paths()?)? + }; + host::require_root()?; + host::require_user_namespaces_blocked()?; + host::require_cgroup_v2()?; + host::require_fuse()?; + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + host::require_overlay_upper(root)?; + options.command_identity.check()?; + let executable = &request.daemon_executable; + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("namespaces daemon executable")); + } + let helper_metadata = fs::metadata(helper)?; + if !helper.is_absolute() || !helper_metadata.is_file() || helper_metadata.len() == 0 { + return Err(RuntimeError::Protocol("namespaces helper executable")); + } + if request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + || request.startup_timeout.is_zero() + { + return Err(RuntimeError::Protocol( + "namespaces daemon setup/startup wait", + )); + } + check_environment(&options.environment)?; + check_store(&paths, &options.store)?; + for path in [&paths.local.directory, &paths.root] { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + Ok(_) => return Err(RuntimeError::Protocol("namespaces Sandbox path exists")), + } + } + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + let image = rootfs::ensure(&options.archive, &options.image, &rootfs::cache(root))?; + Ok((paths, image)) +} + +/// The daemon starts with the search path and nothing else but these: no +/// `PATH` override, and only this backend's kernel choice. +fn check_environment(environment: &[(String, String)]) -> Result<(), RuntimeError> { + let invalid = environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != "fuse") + }); + if invalid { + return Err(RuntimeError::Protocol("namespaces environment")); + } + Ok(()) +} + +/// The Store's directory is bound into every Sandbox, so it must not hold the +/// runtime root (which would expose every Sandbox's directory to all of them) +/// and must be a real directory that is not the filesystem root. +fn check_store(paths: &Paths, store: &Path) -> Result<(), RuntimeError> { + let directory = (store.parent()) + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or(RuntimeError::Protocol("namespaces Store file"))?; + let metadata = fs::metadata(directory)?; + let root = paths + .local + .directory + .parent() + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + if !store.is_absolute() + || !metadata.is_dir() + || directory == Path::new("/") + || (store.components()).any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + || root.starts_with(directory) + || paths.local.directory.starts_with(directory) + || paths.root.starts_with(directory) + { + return Err(RuntimeError::Protocol( + "namespaces Store directory (must not hold the runtime root)", + )); + } + Ok(()) +} + +/// The Sandbox's own directories and files, its copied executable, its init +/// spec and the empty log; returns the log file. Its overlay's writable layer +/// and work directory are made here, empty: the init stage mounts them with +/// the image underneath. +fn prepare( + paths: &Paths, + request: &crate::provider::LaunchRequest, + options: &Options, + id: &crate::ContainerId, + image: &ImageRoot, +) -> Result { + fs::set_permissions(&paths.local.directory, Permissions::from_mode(0o755))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + for file in [&paths.local.config, &paths.local.overlay] { + let parent = file + .parent() + .ok_or(RuntimeError::Protocol("namespaces path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + for directory in [&paths.bin_directory, &paths.local.mounts] { + fs::DirBuilder::new().mode(0o755).create(directory)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.local.tmp)?; + fs::set_permissions(&paths.local.tmp, Permissions::from_mode(0o1777))?; + fs::DirBuilder::new().mode(0o755).create(&paths.root)?; + for directory in [&paths.upper, &paths.work] { + fs::DirBuilder::new().mode(0o700).create(directory)?; + } + private(&paths.local.config)?.write_all(&request.daemon_config)?; + fs::copy(&request.daemon_executable, &paths.bin)?; + fs::set_permissions(&paths.bin, Permissions::from_mode(0o755))?; + let spec = HelperSpec { + root: paths.root.clone(), + sandbox: paths.local.directory.clone(), + store: (options.store.parent()) + .ok_or(RuntimeError::Protocol("namespaces Store directory"))? + .to_owned(), + daemon: paths.bin.clone(), + config: paths.local.config.clone(), + cgroup: cgroup::path(id), + lower: image.directory().to_owned(), + upper: paths.upper.clone(), + work: paths.work.clone(), + }; + private(&paths.spec)?.write_all(&spec.encode()?)?; + Ok(private(&paths.local.log)?) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs new file mode 100644 index 0000000000..d17372a70e --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/exec.rs @@ -0,0 +1,120 @@ +//! One ordinary command in a namespace Sandbox: the helper enters the +//! Sandbox's mount, PID and IPC namespaces, becomes uid:gid with no +//! capabilities and no-new-privs, and runs the command there. The streams are +//! the helper's own pipes; nothing of this process's environment but the +//! search path reaches it, and no stdin is offered. +use super::sandbox::Sandbox; +use crate::backend::native::search::search_path; +use crate::backend::streams; +use crate::provider::ExecExit; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +impl Sandbox { + /// Runs `request` once while this Sandbox's init stage runs, as its + /// uid:gid, in its directory inside the Sandbox, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + let init = self + .init + .ok_or_else(|| self.invalid("running command admission"))?; + self.endpoint()?; + let mut command = Command::new(&self.helper); + command + .arg("exec") + .arg("--init") + .arg(init.pid.to_string()) + .arg("--cgroup") + .arg(&self.cgroup) + .arg("--uid") + .arg(request.identity.uid.to_string()) + .arg("--gid") + .arg(request.identity.gid.to_string()) + .arg("--cwd") + .arg(&request.directory) + .arg("--") + .args(&request.arguments) + .env_clear() + .env("PATH", search_path()) + .current_dir(self.directory()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = (status.code()) + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("namespaces command exit status"))?; + if status.code().is_none() { + // the helper ends itself with a signal when it cannot set the + // command up (or the Sandbox ended under it), so this is a + // reported failure, never a command's exit code + return Err(self.invalid("namespaces exec: the helper ended without a status")); + } + Ok(ExecExit { + code, + pid: Some(pid), + stdout_bytes: out_bytes, + stderr_bytes: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs new file mode 100644 index 0000000000..0107cf02bd --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/host.rs @@ -0,0 +1,115 @@ +//! What this host must provide before a namespace Sandbox may start, and the +//! two host facts its lifecycle reads: whether user namespaces are blocked +//! and which cgroup a process is in. Every condition is checked before any +//! effect; a host that cannot meet them is refused, never quietly given a +//! weaker Sandbox. +use crate::RuntimeError; +use std::{fs, os::unix::fs::FileTypeExt, path::Path}; + +/// The kernel's limit on user namespaces; `0` blocks them host-wide. +const MAX_USER_NAMESPACES: &str = "/proc/sys/user/max_user_namespaces"; +/// This host's unified cgroup v2 hierarchy. +const CGROUP_ROOT: &str = "/sys/fs/cgroup"; + +/// The init stage must create mount, PID and IPC namespaces and mount the +/// daemon's private root, which needs the real root. +pub(crate) fn require_root() -> Result<(), RuntimeError> { + if nix::unistd::Uid::effective().is_root() { + Ok(()) + } else { + Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime runs its daemon as root (mount, PID and IPC namespaces, \ + /dev/fuse); start it as root inside the project's VM", + )) + } +} + +/// Refuses to start when a command could create a user namespace: it would +/// hold its own copy of the Workspace mount, outside the daemon's unmount. +/// The block is host-wide (`user.max_user_namespaces = 0`), set by whoever +/// owns the VM; this runtime never sets it. +pub(crate) fn require_user_namespaces_blocked() -> Result<(), RuntimeError> { + let text = fs::read_to_string(MAX_USER_NAMESPACES).map_err(|_| { + RuntimeError::Unsupported( + "the namespaces Sandbox runtime cannot read /proc/sys/user/max_user_namespaces, so it \ + cannot prove that user namespaces are blocked", + ) + })?; + match text.trim().parse::() { + Ok(0) => Ok(()), + Ok(_) => Err(RuntimeError::Unsupported( + "user namespaces are not blocked; a command could unshare one and keep its own copy of \ + the Workspace mount. Set user.max_user_namespaces=0 in the project's VM before a \ + namespaces Sandbox starts", + )), + Err(_) => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime cannot parse /proc/sys/user/max_user_namespaces, so it \ + cannot prove that user namespaces are blocked", + )), + } +} + +/// The unified cgroup v2 hierarchy, which ends a Sandbox as a whole. +pub(crate) fn require_cgroup_v2() -> Result<(), RuntimeError> { + let controllers = Path::new(CGROUP_ROOT).join("cgroup.controllers"); + match fs::read_to_string(&controllers) { + Ok(_) => Ok(()), + Err(_) => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs the unified cgroup v2 hierarchy at /sys/fs/cgroup", + )), + } +} + +/// The daemon serves FUSE from inside the Sandbox. +pub(crate) fn require_fuse() -> Result<(), RuntimeError> { + match fs::metadata("/dev/fuse") { + Ok(metadata) if metadata.file_type().is_char_device() => Ok(()), + _ => Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs a character device /dev/fuse in the project's VM", + )), + } +} + +/// The Sandbox's root is an overlay whose writable layer is the Sandbox's own +/// directory under this runtime root, so the root must be on a filesystem that +/// can be an overlay upper. overlayfs refuses another overlay or a network +/// filesystem there, and a host that cannot hold it is refused before any +/// Sandbox is made rather than failing inside the Sandbox's mount. +pub(crate) fn require_overlay_upper(root: &Path) -> Result<(), RuntimeError> { + let resolved = fs::canonicalize(root)?; + match super::super::linux::mounts::filesystem(&resolved) { + Ok(kind) if !UNSUITABLE.contains(&kind.as_str()) && !kind.starts_with("fuse") => Ok(()), + // an unreadable mount table is no proof either way: the mount itself + // then reports what is wrong, and nothing is invented here + Err(_) => Ok(()), + Ok(_) => Err(RuntimeError::Unsupported( + "the namespaces runtime root is on a filesystem that cannot hold a Sandbox root's writable layer (overlayfs or a network filesystem); put LAYERFS_LOCAL_DIR on a local filesystem such as the project VM's ext4, xfs or btrfs disk", + )), + } +} + +/// Filesystems an overlay upper layer cannot live on. +const UNSUITABLE: [&str; 7] = ["overlay", "nfs", "nfs4", "cifs", "smb3", "9p", "virtiofs"]; + +/// The cgroup of `pid`, as the kernel reports it, e.g. `0::/layerfs/`. +pub(crate) fn cgroup_of(pid: u32) -> std::io::Result { + fs::read_to_string(format!("/proc/{pid}/cgroup")) +} + +/// Whether the live process `pid` is in `directory`'s cgroup: its recorded +/// start time can outlive a reboot-free id reuse, but its cgroup membership +/// cannot be inherited from a sandbox that never made it. +pub(crate) fn in_cgroup(pid: u32, directory: &Path) -> bool { + let Ok(table) = cgroup_of(pid) else { + return false; + }; + let relative = directory + .strip_prefix(CGROUP_ROOT) + .ok() + .and_then(|r| r.to_str()); + let Some(relative) = relative else { + return false; + }; + let suffix = format!("/{relative}"); + (table.lines()).any(|line| line.ends_with(&suffix)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs new file mode 100644 index 0000000000..72a70747a3 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs @@ -0,0 +1,23 @@ +//! Linux namespace Sandboxes: each Sandbox is one daemon inside its own +//! mount, PID and IPC namespaces and its own cgroup v2 directory, on the one +//! kernel and one Store disk of the project's VM. Commands enter those +//! namespaces as a nonroot identity with no capabilities and no-new-privs, +//! and `cgroup.kill` ends everything the Sandbox started. +//! +//! The lifecycle is the local one (`crate::backend::native`): a private +//! directory per Sandbox, one daemon start and one listener wait, a record of +//! the daemon's exact process and its port. What this backend adds is outside +//! that daemon: the helper that builds the private root, the namespaces and +//! the cgroup. Nothing here is retried and no effect is undone implicitly. +mod cgroup; +mod create; +mod exec; +mod host; +mod paths; +mod provider; +pub mod rootfs; +mod sandbox; + +pub use provider::NamespaceProvider; +pub use rootfs::ImageRoot; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs new file mode 100644 index 0000000000..469f5f4f0f --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/paths.rs @@ -0,0 +1,56 @@ +//! Where one namespace Sandbox keeps its parts: the same private directory a +//! local Sandbox uses, plus the executable it runs and the private root its +//! mount namespace pivots to. +use crate::backend::native::LocalPaths; +use crate::RuntimeError; +use std::path::PathBuf; + +/// The daemon executable inside the Sandbox's own `bin` directory, copied +/// there once per Sandbox so nothing about it is read from the host later. +pub(crate) const DAEMON: &str = "layerfs-daemon"; + +/// One Sandbox's paths, all fixed from its directory name. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Paths { + /// The Sandbox directory with the setup, overlay, mounts, log and `tmp`. + pub(crate) local: LocalPaths, + /// `/bin`, 0755. + pub(crate) bin_directory: PathBuf, + /// `/bin/`, 0755 and root-owned. + pub(crate) bin: PathBuf, + /// The init stage's inputs, `/config/helper.spec`, 0600. + pub(crate) spec: PathBuf, + /// The private root, `.root`; the init stage mounts the + /// Sandbox's overlay on it and pivots to it, so it is empty again once the + /// namespaces end. + pub(crate) root: PathBuf, + /// The Sandbox's own writable layer of that overlay, + /// `/overlay/upper`, and the overlay's work directory beside + /// it: root-owned 0700, and deleted with the directory. + pub(crate) upper: PathBuf, + pub(crate) work: PathBuf, +} + +impl Paths { + /// The paths of the Sandbox named by `local`: its directory name is the + /// Sandbox's name, and its sibling `.root` is the private root. + pub(crate) fn of(local: LocalPaths) -> Result { + if (local.directory.file_name()) + .and_then(|name| name.to_str()) + .is_none() + { + return Err(RuntimeError::Protocol("namespaces Sandbox directory name")); + } + let mut root = local.directory.clone().into_os_string(); + root.push(".root"); + Ok(Self { + bin_directory: local.directory.join("bin"), + bin: local.directory.join("bin").join(DAEMON), + spec: local.directory.join("config").join("helper.spec"), + root: PathBuf::from(root), + upper: local.directory.join("overlay").join("upper"), + work: local.directory.join("overlay").join("work"), + local, + }) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs new file mode 100644 index 0000000000..794a5c9ffa --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs @@ -0,0 +1,229 @@ +//! The Linux namespace runtime as a concrete backend: the same local runtime +//! root and daemon control as the native one, with each Sandbox's daemon in +//! its own mount, PID and IPC namespaces and its own cgroup, and its commands +//! entering them as a nonroot identity. The VM that holds this kernel is the +//! boundary; the namespaces keep Sandboxes on one Store out of each other. +use super::{ + cgroup, + create::{self, CreateFailure, Options}, + host, + paths::Paths, + rootfs, + sandbox::Sandbox, +}; +use crate::backend::native::{Local, LocalPaths}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::{ + env, + path::{Path, PathBuf}, +}; + +/// The helper executable, installed beside the daemon it starts. +const HELPER: &str = "layerfs-sandbox-helper"; +/// Names the helper explicitly, when it is not beside the daemon. +const HELPER_ENVIRONMENT: &str = "LAYERFS_SANDBOX_HELPER"; + +/// A Linux namespace runtime over one local root directory. +#[derive(Clone, Debug)] +pub struct NamespaceProvider { + local: Local, +} + +impl NamespaceProvider { + /// Selects `root`, the parent of every Sandbox directory, and refuses a + /// host that cannot hold namespace Sandboxes before any effect: not root, + /// user namespaces not blocked, no cgroup v2, or no `/dev/fuse`. + pub fn new(root: impl AsRef) -> Result { + let local = Local::new(&root)?; + host::require_root()?; + host::require_user_namespaces_blocked()?; + host::require_cgroup_v2()?; + host::require_fuse()?; + host::require_overlay_upper(root.as_ref())?; + Ok(Self { local }) + } + + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + + /// What this backend supports. The isolation boundary is the VM that + /// holds this one kernel (one per Store); a Sandbox inside it owns its + /// mount, PID and IPC namespaces and its cgroup, not a VM of its own. + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "namespaces-linux", + isolation: IsolationModel::MicroVm, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + + /// Starts one Sandbox once; a failed start keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Namespaces { + command_identity, + environment, + store, + image, + archive, + } = request.options.clone() + else { + return Err(fail(RuntimeError::Unsupported( + "namespaces provider requires namespaces launch options", + ))); + }; + let helper = helper(Some(&request.daemon_executable)).map_err(fail)?; + self.local.clone().naming(&request.name).map_err(fail)?; + let options = Options { + command_identity, + environment, + store, + image, + archive, + }; + create::create(&self.local, &helper, &request, &options).map_err(|failure| { + let CreateFailure { + phase, + sandbox, + cause, + } = *failure; + Box::new(LaunchFailure { + phase: match phase { + crate::backend::native::LocalPhase::Validate => LaunchPhase::Validate, + crate::backend::native::LocalPhase::Create => LaunchPhase::Create, + crate::backend::native::LocalPhase::Start => LaunchPhase::Start, + crate::backend::native::LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: sandbox.map(|sandbox| { + Box::new(crate::backend::Sandbox::Namespaces(Box::new(sandbox))) + }), + cause, + }) + }) + } + + /// Reopen starts nothing: it checks the recorded init process, its cgroup + /// and its listener. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let record = record.namespaces().ok_or_else(|| { + WireFailure::local(RuntimeError::Unsupported( + "namespaces provider requires a namespaces record", + )) + })?; + let paths = self + .record_paths(&record.local().name) + .map_err(WireFailure::local)?; + let image = self.image(record).map_err(WireFailure::local)?; + let helper = helper(None).map_err(WireFailure::local)?; + let recorded = super::sandbox::Recorded { + init: super::sandbox::process(record.local()), + port: record.local().port, + labels: record.local().labels.clone(), + image, + }; + Sandbox::reopen(paths, cgroup::path(&id), id, helper, recorded) + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let record = record.namespaces().ok_or_else(|| { + WireFailure::local(RuntimeError::Unsupported( + "namespaces provider requires a namespaces record", + )) + })?; + let paths = self + .record_paths(&record.local().name) + .map_err(WireFailure::local)?; + let image = self.image(record).map_err(WireFailure::local)?; + let helper = helper(None).map_err(WireFailure::local)?; + let recorded = super::sandbox::Recorded { + init: super::sandbox::process(record.local()), + port: record.local().port, + labels: record.local().labels.clone(), + image, + }; + Ok(Sandbox::stopped( + paths, + cgroup::path(&id), + id, + helper, + recorded, + )) + } + + fn record_paths(&self, name: &str) -> Result { + Paths::of(self.paths(name)?) + } + + /// The record's image, as the root this runtime already holds: a reopen + /// starts and unpacks nothing, so a record whose image is gone is refused + /// rather than given a different root. + fn image( + &self, + record: &crate::provider::NamespaceRecord, + ) -> Result { + let paths = self.record_paths(&record.local().name)?; + let root = (paths.local.directory.parent()) + .ok_or(RuntimeError::Protocol("namespaces Sandbox root"))?; + rootfs::lookup(record.image(), &rootfs::cache(root)) + } +} + +/// The helper this backend runs: `LAYERFS_SANDBOX_HELPER` when set, else +/// beside `daemon` or beside this process (or its Cargo test runner). +fn helper(daemon: Option<&Path>) -> Result { + if let Some(path) = env::var_os(HELPER_ENVIRONMENT) { + let path = PathBuf::from(path); + if let Ok(metadata) = std::fs::metadata(&path) { + if path.is_absolute() && metadata.is_file() && metadata.len() > 0 { + return Ok(path); + } + } + } + let mut candidates = Vec::new(); + if let Some(parent) = daemon.and_then(Path::parent) { + candidates.push(parent.join(HELPER)); + } + if let Ok(exe) = env::current_exe() { + if let Some(parent) = exe.parent() { + candidates.push(parent.join(HELPER)); + if let Some(grandparent) = parent.parent() { + candidates.push(grandparent.join(HELPER)); + } + } + } + for candidate in candidates { + if let Ok(metadata) = std::fs::metadata(&candidate) { + if candidate.is_absolute() && metadata.is_file() && metadata.len() > 0 { + return Ok(candidate); + } + } + } + Err(RuntimeError::Unsupported( + "the namespaces Sandbox runtime needs the layerfs-sandbox-helper executable beside the daemon (or LAYERFS_SANDBOX_HELPER set)", + )) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs new file mode 100644 index 0000000000..3afd573689 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/archive.rs @@ -0,0 +1,330 @@ +//! One docker-archive: its `manifest.json`, the digest it names its config by, +//! and its layers, unpacked into the tree bottom first. +use super::{ + layers::apply, + refuse, + tar::{path_of, Kind, Tape}, + PATH_MAX, +}; +use crate::{backend::json::Json, Image, RuntimeError}; +use std::{ + fs::{self, File, OpenOptions}, + io::{self, BufReader, Read, Seek, SeekFrom}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, +}; + +/// The largest `manifest.json` accepted: it names a config and a few layers. +const MANIFEST_MAX: usize = 1 << 20; +/// The largest number of layers one archive may name. +const LAYERS_MAX: usize = 1024; + +/// One image archive's decided layout: the config blob and the layers, in the +/// order they are applied. +#[derive(Debug)] +struct Manifest { + config: PathBuf, + layers: Vec, +} + +/// Where one blob of the archive is: the offset its bytes start at, and how +/// many there are. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Blob { + offset: u64, + size: u64, +} + +/// What a layer's first bytes say it is. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Magic { + Tar, + Gzip, + Zstd, +} + +/// Unpacks `archive`'s layers, bottom first, into `root`. +pub(super) fn unpack(archive: &Path, image: &Image, root: &Path) -> Result<(), RuntimeError> { + let manifest = manifest(archive, image)?; + let mut wanted = Vec::with_capacity(manifest.layers.len() + 1); + wanted.push(manifest.config.clone()); + wanted.extend(manifest.layers.iter().cloned()); + let blobs = locate(archive, &wanted)?; + // the config digest is the name the archive gave that blob; it was checked + // before this point, so only its presence is still open here + if blobs.first().copied().flatten().is_none() { + return Err(refuse(format!( + "the image archive has no config blob {}", + manifest.config.display() + ))); + } + for (index, layer) in manifest.layers.iter().enumerate() { + let blob = blobs.get(index + 1).copied().flatten().ok_or_else(|| { + refuse(format!( + "the image archive has no layer {}", + layer.display() + )) + })?; + extract_layer(archive, blob, &layer.display().to_string(), root)?; + } + Ok(()) +} + +/// The archive's `manifest.json`, as the entry whose config digest is `image`. +fn manifest(archive: &Path, image: &Image) -> Result { + let located = locate(archive, &[PathBuf::from("manifest.json")])?; + let blob = located + .first() + .copied() + .flatten() + .ok_or_else(|| refuse("the image archive has no manifest.json"))?; + if blob.size > MANIFEST_MAX as u64 { + return Err(refuse("the image archive's manifest.json is too long")); + } + read_blob(archive, blob, MANIFEST_MAX).and_then(|bytes| parse_manifest(&bytes, image)) +} + +/// Reads `manifest.json` and keeps the entry whose config digest is `image`. +fn parse_manifest(bytes: &[u8], image: &Image) -> Result { + let mut found: Option = None; + let mut named: Vec = Vec::new(); + let mut json = Json::new(bytes); + json.array(|element| { + let mut config: Option> = None; + let mut layers: Option>> = None; + let mut seen: u8 = 0; + element.object(|reader, key| { + if key.overflow { + return Ok(()); + } + match key.text() { + "Config" => { + crate::backend::json::once(&mut seen, 1)?; + let mut text = Vec::new(); + reader.text_into(&mut text, PATH_MAX)?; + config = Some(text); + } + "Layers" => { + crate::backend::json::once(&mut seen, 2)?; + let mut items = Vec::new(); + reader.array(|entry| { + if items.len() == LAYERS_MAX { + return Err(refuse("the image archive names too many layers")); + } + let mut text = Vec::new(); + entry.text_into(&mut text, PATH_MAX)?; + items.push(text); + Ok(()) + })?; + layers = Some(items); + } + _ => reader.skip(0)?, + } + Ok(()) + })?; + let (Some(config), Some(layers)) = (config, layers) else { + return Ok(()); + }; + let config = path_of(&config)?; + if named.len() < 8 { + named.push(config.display().to_string()); + } + if found.is_none() && check_config(image, &config).is_ok() { + found = Some(Manifest { + config, + layers: (layers.iter()) + .map(|layer| path_of(layer)) + .collect::, _>>()?, + }); + } + Ok(()) + })?; + json.finish()?; + found.ok_or_else(|| { + refuse(format!( + "the image archive's config digest is {}, not the requested {}: the bundle does not \ + hold that image", + if named.is_empty() { + "missing".to_owned() + } else { + named.join(", ") + }, + image.as_str() + )) + }) +} + +/// The config digest check: the archive names its config blob by that digest +/// (`<64 hex>.json` or `blobs/sha256/<64 hex>`), and it must be `image`'s. +fn check_config(image: &Image, config: &Path) -> Result<(), RuntimeError> { + let name = config + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| refuse("the image archive's config name"))?; + let digest = name.strip_suffix(".json").unwrap_or(name); + if digest != image.hex() { + return Err(refuse(format!( + "the image archive's config digest is {digest}, not the requested {}", + image.as_str() + ))); + } + Ok(()) +} + +/// One scan of the archive for the entries named by `wanted`, in that order; +/// everything else is skipped by seeking, never read. +fn locate(archive: &Path, wanted: &[PathBuf]) -> Result>, RuntimeError> { + let mut found = vec![None; wanted.len()]; + let mut remaining = wanted.len(); + let mut tape = Tape::new(BufReader::with_capacity(64 * 1024, open(archive)?)); + while remaining > 0 { + let Some(entry) = tape.next()? else { break }; + if entry.kind != Kind::Regular { + continue; + } + for (index, name) in wanted.iter().enumerate() { + if found[index].is_none() && *name == entry.path { + found[index] = Some(Blob { + offset: tape.data_offset(), + size: entry.size, + }); + remaining -= 1; + break; + } + } + } + if remaining > 0 { + let missing: Vec = (wanted.iter().enumerate()) + .filter(|(index, _)| found[*index].is_none()) + .map(|(_, name)| name.display().to_string()) + .collect(); + return Err(refuse(format!( + "the image archive is missing {}", + missing.join(", ") + ))); + } + Ok(found) +} + +/// The first bytes of `blob`, up to `max`. +fn read_blob(archive: &Path, blob: Blob, max: usize) -> Result, RuntimeError> { + let mut file = open(archive)?; + file.seek(SeekFrom::Start(blob.offset))?; + let mut bytes = Vec::new(); + let limit = blob.size.min(max as u64 + 1); + file.take(limit).read_to_end(&mut bytes)?; + if bytes.len() > max { + return Err(refuse("the image archive's blob is longer than allowed")); + } + Ok(bytes) +} + +/// Applies one layer, whose bytes are the `size` at `blob`. +fn extract_layer(archive: &Path, blob: Blob, name: &str, root: &Path) -> Result<(), RuntimeError> { + let mut file = open(archive)?; + file.seek(SeekFrom::Start(blob.offset))?; + match magic(&mut file)? { + Magic::Tar => {} + Magic::Gzip => { + return Err(refuse(format!( + "the image archive's layer {name} is gzip-compressed; a docker-archive must hold \ + uncompressed layers (docker save writes them so)" + ))) + } + Magic::Zstd => { + return Err(refuse(format!( + "the image archive's layer {name} is zstd-compressed; a docker-archive must hold \ + uncompressed layers (docker save writes them so)" + ))) + } + } + file.seek(SeekFrom::Start(blob.offset))?; + let mut tape = Tape::new(Window { + input: file, + left: blob.size, + }); + while let Some(entry) = tape.next()? { + let mut data = (&mut tape).take(entry.size); + apply(root, &entry, &mut data)?; + } + Ok(()) +} + +/// What the first bytes at the current position say the layer is. +fn magic(file: &mut File) -> Result { + let mut header = [0u8; 4]; + let mut read = 0; + while read < header.len() { + let count = file.read(&mut header[read..])?; + if count == 0 { + break; + } + read += count; + } + Ok(match &header[..read] { + [0x1f, 0x8b, ..] => Magic::Gzip, + [0x28, 0xb5, 0x2f, 0xfd] => Magic::Zstd, + _ => Magic::Tar, + }) +} + +/// A bounded window of an archive: one layer's bytes. Seeking forward is what +/// skipping an entry needs; anything else is refused. +struct Window { + input: R, + left: u64, +} + +impl Read for Window { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.left == 0 || buffer.is_empty() { + return Ok(0); + } + let wanted = (buffer.len() as u64).min(self.left) as usize; + let read = self.input.read(&mut buffer[..wanted])?; + self.left -= read as u64; + Ok(read) + } +} + +impl Seek for Window { + fn seek(&mut self, position: SeekFrom) -> io::Result { + let SeekFrom::Current(skip) = position else { + return Err(io::Error::from(io::ErrorKind::Unsupported)); + }; + if skip < 0 { + return Err(io::Error::from(io::ErrorKind::Unsupported)); + } + let skip = (skip as u64).min(self.left); + self.input.seek(SeekFrom::Current(skip as i64))?; + self.left -= skip; + Ok(self.left) + } +} + +/// An archive opened for reading, refused when it is not a regular file (a +/// symbolic link would let the archive be swapped after the check). +fn open(archive: &Path) -> Result { + let metadata = fs::symlink_metadata(archive).map_err(|error| { + refuse(format!( + "the image archive {} cannot be read: {error}", + archive.display() + )) + })?; + if !metadata.file_type().is_file() { + return Err(refuse(format!( + "the image archive {} is not a regular file", + archive.display() + ))); + } + OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(archive) + .map_err(|error| { + refuse(format!( + "the image archive {} cannot be opened: {error}", + archive.display() + )) + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs new file mode 100644 index 0000000000..90db445af5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/cache.rs @@ -0,0 +1,149 @@ +//! Where unpacked images live and how one digest is unpacked exactly once: +//! `/images/<64 hex>`, published by a rename, guarded by a per-digest +//! `flock`. +use super::{archive::unpack, refuse, ImageRoot}; +use crate::{Image, RuntimeError}; +use nix::fcntl::{Flock, FlockArg}; +use std::{ + fs::{self, DirBuilder, File, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +/// The cache's own name under a runtime root: `/images`. +const CACHE: &str = "images"; + +/// The cache directory below a runtime root, `/images`, where unpacked +/// images and their locks live. +pub fn cache(root: &Path) -> PathBuf { + root.join(CACHE) +} + +/// The image's root under `cache`, unpacking `archive` first when this host +/// has no complete copy of that digest yet. One unpack per digest, however +/// many Sandboxes ask for it at once. +pub fn ensure(archive: &Path, image: &Image, cache: &Path) -> Result { + let cache = beneath(cache)?; + let lock = lock(&cache, image)?; + let directory = cache.join(image.hex()); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => { + drop(lock); + return Ok(ImageRoot::new(image.clone(), directory)); + } + Ok(_) => { + return Err(refuse(format!( + "the image cache {} is not a directory", + directory.display() + ))) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + // a crash of an earlier unpack left only its temporary directory: it is + // never the real name, so it is removed before this one is made + remove_temporaries(&cache, image)?; + let staging = cache.join(format!( + ".tmp-{}-{}", + image.hex(), + crate::backend::identity::random()?.as_str() + )); + DirBuilder::new().mode(0o700).create(&staging)?; + if let Err(cause) = unpack(archive, image, &staging) { + let _ = fs::remove_dir_all(&staging); + return Err(cause); + } + fs::set_permissions(&staging, fs::Permissions::from_mode(0o555))?; + match fs::rename(&staging, &directory) { + Ok(()) => {} + Err(error) => { + // another Sandbox published the same digest first: its copy is + // this one's too, and this one is the duplicate + let _ = fs::remove_dir_all(&staging); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + _ => return Err(error.into()), + } + } + } + drop(lock); + Ok(ImageRoot::new(image.clone(), directory)) +} + +/// The image's root under `cache` when this host already holds it, unpacking +/// nothing. A record whose image is gone cannot be reopened as the same root. +pub fn lookup(image: &Image, cache: &Path) -> Result { + let directory = cache.join(image.hex()); + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => Ok(ImageRoot::new(image.clone(), directory)), + _ => Err(refuse(format!( + "the unpacked image {} is not at {}: this Sandbox's root cannot be found, so it is \ + neither reopened nor deleted", + image.as_str(), + directory.display() + ))), + } +} + +/// The cache directory, made 0700 and owned by this (root) process when it is +/// not there yet, verified to be a real directory when it is. +fn beneath(cache: &Path) -> Result { + if !cache.is_absolute() { + return Err(RuntimeError::Protocol( + "image cache must be an absolute path", + )); + } + match fs::symlink_metadata(cache) { + Ok(metadata) if metadata.is_dir() => Ok(cache.to_owned()), + Ok(_) => Err(refuse(format!( + "the image cache {} is not a directory", + cache.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let parent = cache + .parent() + .ok_or(RuntimeError::Protocol("image cache parent"))?; + fs::symlink_metadata(parent)?; + DirBuilder::new().mode(0o700).create(cache)?; + Ok(cache.to_owned()) + } + Err(error) => Err(error.into()), + } +} + +/// The per-digest lock, held by the caller until the returned lock is dropped. +/// The file stays on disk: another process may be waiting on it. +fn lock(cache: &Path, image: &Image) -> Result, RuntimeError> { + let path = cache.join(format!("{}.lock", image.hex())); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(path)?; + Flock::lock(file, FlockArg::LockExclusive).map_err(|(_, error)| RuntimeError::Io(error.into())) +} + +/// Removes the temporary directories a crashed unpack of `image` left; only +/// this digest's, and only while holding its lock. +fn remove_temporaries(cache: &Path, image: &Image) -> Result<(), RuntimeError> { + let prefix = format!(".tmp-{}-", image.hex()); + for entry in fs::read_dir(cache)? { + let entry = entry?; + let name = entry.file_name(); + let Some(text) = name.to_str() else { continue }; + if !text.starts_with(&prefix) { + continue; + } + let path = entry.path(); + if fs::symlink_metadata(&path)?.file_type().is_dir() { + fs::remove_dir_all(&path)?; + } else { + fs::remove_file(&path)?; + } + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs new file mode 100644 index 0000000000..4d27e27743 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/layers.rs @@ -0,0 +1,220 @@ +//! What one layer does to the unpacked tree: a whiteout hides what the layers +//! below placed, and a regular file, directory, symbol link or hard link is +//! written inside the root — never through a symbolic link, and never outside +//! the root it belongs to. +use super::{ + refuse, + tar::{Entry, Kind}, +}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + ffi::OsStr, + fs::{self, DirBuilder, OpenOptions}, + io, + os::unix::{ + ffi::OsStrExt, + fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + }, + path::{Component, Path, PathBuf}, +}; + +/// One entry, applied to the unpacked tree: a whiteout, or a file, directory, +/// symbol link or hard link inside it. +pub(super) fn apply( + root: &Path, + entry: &Entry, + data: &mut dyn io::Read, +) -> Result<(), RuntimeError> { + let (Some(parent), Some(name)) = (entry.path.parent(), entry.path.file_name()) else { + return Err(refuse("the image archive's entry has no name")); + }; + let name = name.as_bytes(); + if entry.kind == Kind::Regular && name.starts_with(b".wh.") { + return whiteout(root, parent, &name[b".wh.".len()..]); + } + let target = beneath_root(root, &entry.path)?; + match &entry.kind { + Kind::Directory => make_directory(&target, entry.mode), + Kind::Regular => { + remove_any(&target)?; + write_file(&target, entry.mode, data) + } + Kind::Symlink(link) => { + remove_any(&target)?; + std::os::unix::fs::symlink(link, &target)?; + Ok(()) + } + Kind::HardLink(link) => { + let source = beneath_root(root, link)?; + if !fs::symlink_metadata(&source)?.file_type().is_file() { + return Err(refuse(format!( + "the image archive links {} to {}, which is not a file in it", + entry.path.display(), + link.display() + ))); + } + remove_any(&target)?; + fs::hard_link(&source, &target)?; + Ok(()) + } + } +} + +/// Applies `.wh.`/`.wh..wh..opq` to `root`: the named entry, or everything in +/// the directory, is hidden from the layers below. A whiteout with nothing +/// below it removes nothing. +fn whiteout(root: &Path, parent: &Path, name: &[u8]) -> Result<(), RuntimeError> { + // a whiteout hides what a lower layer placed; it never creates the + // directory it names, so a layer that only hides a path changes nothing + let Some(directory) = beneath_existing(root, parent)? else { + return Ok(()); + }; + match fs::symlink_metadata(&directory) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + return Err(refuse( + "the image archive's whiteout is outside a directory", + )) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + } + if name == b".wh..opq" { + for entry in fs::read_dir(&directory)? { + remove_any(&entry?.path())?; + } + return Ok(()); + } + if name.starts_with(b".wh..wh.") { + // aufs bookkeeping of an opaque directory's own metadata: not content + return Ok(()); + } + if name.is_empty() || name.contains(&b'/') { + return Err(refuse("the image archive's whiteout name")); + } + let hidden = directory.join(OsStr::from_bytes(name)); + match fs::symlink_metadata(&hidden) { + Ok(_) => remove_any(&hidden), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +/// `relative`'s place inside `root`, with every directory above it verified to +/// be one of this archive's own directories and never a symbolic link: an +/// entry can then only ever write inside the root it belongs to. +fn beneath_root(root: &Path, relative: &Path) -> Result { + let components: Vec = relative.components().collect(); + if components.is_empty() { + return Err(refuse("the image archive's entry has no name")); + } + let mut path = root.to_owned(); + for component in &components[..components.len() - 1] { + let Component::Normal(name) = component else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + path.push(name); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(refuse(format!( + "the image archive's entry {} passes through a symbolic link", + relative.display() + ))) + } + Ok(_) => { + return Err(refuse(format!( + "the image archive's entry {} passes through a non-directory", + relative.display() + ))) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + DirBuilder::new().mode(0o755).create(&path)?; + } + Err(error) => return Err(error.into()), + } + } + let Some(Component::Normal(name)) = components.last() else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + Ok(path.join(name)) +} + +/// `relative`'s place inside `root` when every component of it is already +/// there: `None` when one is not, and nothing is made. +fn beneath_existing(root: &Path, relative: &Path) -> Result, RuntimeError> { + let mut path = root.to_owned(); + for component in relative.components() { + let Component::Normal(name) = component else { + return Err(refuse( + "the image archive's entry is not a plain relative path", + )); + }; + path.push(name); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.is_dir() => {} + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(refuse( + "the image archive's entry passes through a symbolic link", + )) + } + Ok(_) => return Ok(None), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + } + Ok(Some(path)) +} + +/// A directory at `target`, with `mode` (0755 for a mode of zero). +fn make_directory(target: &Path, mode: u32) -> Result<(), RuntimeError> { + let mode = if mode & 0o777 == 0 { + 0o755 + } else { + mode & 0o777 + }; + match fs::symlink_metadata(target) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + remove_any(target)?; + DirBuilder::new().mode(mode).create(target)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + DirBuilder::new().mode(mode).create(target)?; + } + Err(error) => return Err(error.into()), + } + fs::set_permissions(target, fs::Permissions::from_mode(mode))?; + Ok(()) +} + +/// A regular file at `target`, whose bytes are all of `data`. +fn write_file(target: &Path, mode: u32, data: &mut dyn io::Read) -> Result<(), RuntimeError> { + let mode = mode & 0o777; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(mode) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(target)?; + io::copy(data, &mut file)?; + fs::set_permissions(target, fs::Permissions::from_mode(mode))?; + Ok(()) +} + +/// Removes whatever is at `path` without following it: a symbolic link is +/// removed as the link it is, never as what it names. +fn remove_any(path: &Path) -> Result<(), RuntimeError> { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(metadata) if metadata.is_dir() => Ok(fs::remove_dir_all(path)?), + Ok(_) => Ok(fs::remove_file(path)?), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs new file mode 100644 index 0000000000..729621a071 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/mod.rs @@ -0,0 +1,85 @@ +//! The Sandbox image as a read-only root filesystem: one archive per bundle, +//! unpacked once per config digest and shared by every Sandbox that names it. +//! +//! A namespace Sandbox's root is an overlay of two things: this unpacked image +//! (the lower, read-only layer) and the Sandbox's own `upper` and `work` +//! directories (its writable layer, deleted with it). The image is what the +//! commands see as `/`; the host's own userland is not part of that root. +//! +//! # The archive +//! +//! The bundle ships one **docker-archive** (`docker save`'s output) per +//! Sandbox image, named `/images/<64 hex>.tar`, where `<64 hex>` is +//! `deploy.image` without its `sha256:` prefix. The bundle builder sets +//! `LAYERFS_IMAGE` (the digest) and `LAYERFS_IMAGE_ARCHIVE` (the absolute path +//! of that file) for the worker that runs this runtime. +//! +//! `docker-archive` is chosen over an OCI image layout because +//! - `deploy.image` is Docker's own local image identity on the Docker backend, +//! and a docker-archive's `manifest.json` names its config by exactly that +//! digest, so one value addresses the image on both backends; +//! - `docker save` is available everywhere the Docker backend already works, +//! which is where the image comes from; +//! - its layer blobs are uncompressed tars, so unpacking needs no decompressor +//! and this crate keeps its single dependency. +//! +//! A compressed layer blob (`podman save` and some other writers gzip them) is +//! refused before anything is written; the archive must be produced by +//! `docker save` or converted to that shape. +//! +//! # The digest +//! +//! `deploy.image` is `sha256:<64 lowercase hex>` and must be the archive's +//! config digest: the docker-archive names that blob `<64 hex>.json` or +//! `blobs/sha256/<64 hex>`, and [`archive`] compares that name with the +//! requested digest. An archive whose config digest is something else is +//! refused before anything is unpacked, so a Sandbox never runs a root other +//! than the one the deployment named. The config blob is not re-hashed here: +//! the archive arrives inside a sealed bundle that the provider copies in +//! whole, and this crate has no hashing dependency of its own. +//! +//! # One unpack per digest +//! +//! The unpacked image lives under `/images/<64 hex>`, named +//! by its digest only, so two Sandboxes that name one image share one copy. +//! The first Sandbox to need it takes a per-digest lock +//! (`images/<64 hex>.lock`, `flock`), unpacks into `images/.tmp-<64 hex>-…` +//! and renames that directory into place once it is complete, so a second +//! Sandbox either waits for that rename or finds the finished directory, and a +//! crashed unpack never leaves a half image under the real name. The lock file +//! stays: another process may still hold it. +//! +//! A command never gets write access to the image. The unpacked directory is +//! outside the Sandbox's mount tree — only the image's *contents* are composed +//! into its root, by the kernel, as a read-only lower layer — so no path a +//! command can name reaches the cache, and the cache directory itself is +//! root-owned and not writable by the command identity. +//! +//! # Extraction safety +//! +//! Every layer is applied in order, with the archive's own whiteouts +//! (`.wh.` removes that name, `.wh..wh..opq` empties its directory). +//! Entries are refused, not repaired, when they are +//! - an absolute path or one with a `.`/`..` component, +//! - reachable only through a symbolic link or a non-directory (so a symlink +//! in an early entry cannot redirect a later entry's write outside the root), +//! - a device node, a FIFO or any other entry type this runtime does not +//! extract (regular files, directories, symlinks and hard links only). +//! +//! File ownership becomes root (the archive's numeric owners are not applied, +//! and a rootfs layer is root's), permission bits are applied as `0o777` — +//! setuid, setgid, sticky and xattrs (and so file capabilities) are not +//! restored — and a command runs with no-new-privs anyway. +//! +//! The pieces are the cache ([`cache`]), one archive's own reading +//! ([`archive`]), the tar format ([`tar`]), what one layer does to the tree +//! ([`layers`]) and the types they share ([`types`]). +mod archive; +mod cache; +mod layers; +mod tar; +mod types; + +pub use cache::{cache, ensure, lookup}; +pub use types::ImageRoot; +pub(crate) use types::{refuse, PATH_MAX}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs new file mode 100644 index 0000000000..e480994601 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/tar.rs @@ -0,0 +1,366 @@ +//! The tar format as far as an image archive needs it: one header at a time, +//! the entry's data read through the same handle, everything else skipped by +//! seeking. Paths are checked here, so nothing downstream sees one that could +//! leave the root. +use super::{refuse, PATH_MAX}; +use crate::RuntimeError; +use std::{ + ffi::OsStr, + io::{self, Read, Seek, SeekFrom}, + os::unix::ffi::OsStrExt, + path::{Component, Path, PathBuf}, +}; + +/// A tar block, and the header's own size. +const BLOCK: u64 = 512; +/// The largest pax or GNU extended header accepted. +const EXTENDED_MAX: usize = 1 << 20; + +/// One entry of a tar archive, its path already checked to be a plain relative +/// path inside the root. +#[derive(Debug)] +pub(super) struct Entry { + pub(super) path: PathBuf, + pub(super) kind: Kind, + pub(super) mode: u32, + pub(super) size: u64, +} + +/// What an entry places in the root. Every other tar entry type (device +/// nodes, FIFOs, …) is refused when the header is read. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum Kind { + Regular, + Directory, + Symlink(PathBuf), + HardLink(PathBuf), +} + +/// A streaming tar reader. Nothing is collected: an entry's bytes are streamed +/// to where they go, and skipped by seeking when the entry is not wanted. +pub(super) struct Tape { + input: R, + /// Bytes read from the start of this archive. + position: u64, + /// Bytes of the current entry's data not read yet. + left: u64, + /// Padding after that data, to the next block. + padding: u64, + /// Where the current entry's data starts. + data_offset: u64, + /// A pax or GNU name waiting for the entry it belongs to. + path: Option, + /// A pax or GNU link target waiting for the same entry. + link: Option, +} + +impl Tape { + pub(super) fn new(input: R) -> Self { + Self { + input, + position: 0, + left: 0, + padding: 0, + data_offset: 0, + path: None, + link: None, + } + } + + /// Where the current entry's bytes start. + pub(super) fn data_offset(&self) -> u64 { + self.data_offset + } + + /// The next entry, or `None` at the archive's end. Extended headers (pax + /// `x`, GNU `L`/`K`) are applied to the entry that follows them; pax + /// global headers and unknown fixed fields are read past. + pub(super) fn next(&mut self) -> Result, RuntimeError> { + loop { + self.skip_data()?; + let Some(header) = self.header()? else { + return Ok(None); + }; + let kind = header[156]; + let size = number(&header[124..136])?; + self.left = size; + self.padding = (BLOCK - size % BLOCK) % BLOCK; + self.data_offset = self.position; + match kind { + b'x' => { + let (path, link) = pax(&self.data(EXTENDED_MAX)?)?; + if path.is_some() { + self.path = path; + } + if link.is_some() { + self.link = link; + } + continue; + } + b'g' => continue, + b'L' => { + self.path = Some(long_name(&self.data(PATH_MAX)?)?); + continue; + } + b'K' => { + self.link = Some(long_name(&self.data(PATH_MAX)?)?); + continue; + } + _ => {} + } + let path = match self.path.take() { + Some(path) => path, + None => own_name(&header)?, + }; + let link = match self.link.take() { + Some(link) => link, + None => link_name(&header)?, + }; + let kind = kind_of(kind, link)?; + return Ok(Some(Entry { + path, + kind, + mode: u32::try_from(number(&header[100..108])?).unwrap_or(0o644), + size, + })); + } + } + + /// One block of header, or `None` at the end of the archive (an all-zero + /// block, or nothing left). + fn header(&mut self) -> Result, RuntimeError> { + let mut header = [0u8; 512]; + let read = self.fill(&mut header)?; + if read == 0 || header.iter().all(|byte| *byte == 0) { + return Ok(None); + } + if read < header.len() { + return Err(refuse("the image archive's header is truncated")); + } + if checksum(&header) != number(&header[148..156])? { + return Err(refuse("the image archive's header checksum")); + } + Ok(Some(header)) + } + + /// Reads the current entry's data (at most `max` bytes) and returns it. + fn data(&mut self, max: usize) -> Result, RuntimeError> { + if self.left > max as u64 { + return Err(refuse("the image archive's extended header is too long")); + } + let left = self.left; + let mut bytes = Vec::new(); + let mut source = Read::by_ref(self).take(left); + source.read_to_end(&mut bytes)?; + Ok(bytes) + } + + /// Skips whatever is left of the current entry and its padding. + fn skip_data(&mut self) -> Result<(), RuntimeError> { + let skip = (self.left) + .checked_add(self.padding) + .ok_or_else(|| refuse("the image archive's entry length"))?; + self.left = 0; + self.padding = 0; + if skip == 0 { + return Ok(()); + } + let distance = + i64::try_from(skip).map_err(|_| refuse("the image archive's entry length"))?; + self.input.seek(SeekFrom::Current(distance))?; + self.position += skip; + Ok(()) + } + + /// Fills `buffer` as far as the input allows, returning how many bytes were + /// read (zero only at the end). + fn fill(&mut self, buffer: &mut [u8]) -> Result { + let mut filled = 0; + while filled < buffer.len() { + let read = self.input.read(&mut buffer[filled..])?; + if read == 0 { + break; + } + filled += read; + self.position += read as u64; + } + Ok(filled) + } +} + +impl Read for Tape { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.left == 0 || buffer.is_empty() { + return Ok(0); + } + let wanted = (buffer.len() as u64).min(self.left) as usize; + let read = self.input.read(&mut buffer[..wanted])?; + self.left -= read as u64; + self.position += read as u64; + Ok(read) + } +} + +/// The entry's own name: its `name` and, in a ustar archive, its `prefix`. +fn own_name(header: &[u8; 512]) -> Result { + let name = text(&header[0..100]); + let prefix = if &header[257..262] == b"ustar".as_slice() { + text(&header[345..500]) + } else { + &[] + }; + let mut bytes = Vec::with_capacity(prefix.len() + 1 + name.len()); + if !prefix.is_empty() { + bytes.extend_from_slice(prefix); + bytes.push(b'/'); + } + bytes.extend_from_slice(name); + checked(&bytes) +} + +/// The entry's own link target, for the two kinds that carry one. +fn link_name(header: &[u8; 512]) -> Result { + Ok(PathBuf::from(OsStr::from_bytes(text(&header[157..257])))) +} + +/// The entry's kind, refusing every type this runtime does not extract. +fn kind_of(flag: u8, link: PathBuf) -> Result { + match flag { + b'0' | b'\0' | b'7' => Ok(Kind::Regular), + b'5' => Ok(Kind::Directory), + b'2' => Ok(Kind::Symlink(link)), + b'1' => Ok(Kind::HardLink(checked(link.as_os_str().as_bytes())?)), + b'3' | b'4' => Err(refuse( + "the image archive holds a device node, which a Sandbox root does not extract", + )), + b'6' => Err(refuse( + "the image archive holds a FIFO, which a Sandbox root does not extract", + )), + _ => Err(refuse( + "the image archive holds an entry type it does not extract", + )), + } +} + +/// A path checked to be plain and relative: no `/` at its start, no `.` or +/// `..` component, at most one kernel path's worth of bytes. +fn checked(bytes: &[u8]) -> Result { + let path = Path::new(OsStr::from_bytes(bytes)); + if bytes.len() > PATH_MAX || bytes.contains(&0) { + return Err(refuse( + "the image archive's path is not a safe relative path", + )); + } + let mut cleaned = PathBuf::new(); + for component in path.components() { + match component { + Component::Normal(name) => cleaned.push(name), + Component::CurDir => {} + Component::RootDir | Component::ParentDir | Component::Prefix(_) => { + return Err(refuse( + "the image archive's path is absolute or holds a `..` component", + )) + } + } + } + if cleaned.as_os_str().is_empty() { + return Err(refuse("the image archive's path is empty")); + } + Ok(cleaned) +} + +/// A manifest entry's path, from the UTF-8 text the archive's JSON holds. +pub(super) fn path_of(bytes: &[u8]) -> Result { + std::str::from_utf8(bytes) + .map_err(|_| refuse("the image archive's manifest is not UTF-8")) + .and_then(|text| checked(text.as_bytes())) +} + +/// A pax extended header's `path` and `linkpath` records, if it has them. +fn pax(data: &[u8]) -> Result<(Option, Option), RuntimeError> { + let (mut path, mut link) = (None, None); + let mut rest = data; + while !rest.is_empty() { + let space = rest + .iter() + .position(|byte| *byte == b' ') + .ok_or_else(|| refuse("the image archive's pax record"))?; + let length: usize = std::str::from_utf8(&rest[..space]) + .ok() + .and_then(|text| text.parse().ok()) + .filter(|length: &usize| *length > space + 1 && *length <= rest.len()) + .ok_or_else(|| refuse("the image archive's pax record length"))?; + let record = &rest[space + 1..length - 1]; + if let Some(equals) = record.iter().position(|byte| *byte == b'=') { + let (key, value) = (&record[..equals], &record[equals + 1..]); + match key { + b"path" => path = Some(checked(value)?), + b"linkpath" => link = Some(checked(value)?), + _ => {} + } + } + rest = &rest[length..]; + } + Ok((path, link)) +} + +/// A GNU long name's data: the name, up to its first NUL. +fn long_name(data: &[u8]) -> Result { + let name = match data.iter().position(|byte| *byte == 0) { + Some(end) => &data[..end], + None => data, + }; + checked(name) +} + +/// A header's NUL- or space-padded octal (or GNU base-256) number. +fn number(field: &[u8]) -> Result { + if let Some(first) = field.first() { + if first & 0x80 != 0 { + let mut value = u64::from(first & 0x7f); + for byte in &field[1..] { + value = (value) + .checked_mul(256) + .and_then(|value| value.checked_add(u64::from(*byte))) + .ok_or_else(|| refuse("the image archive's number"))?; + } + return Ok(value); + } + } + let mut value = 0u64; + for byte in field { + match byte { + b'0'..=b'7' => { + value = (value) + .checked_mul(8) + .and_then(|value| value.checked_add(u64::from(*byte - b'0'))) + .ok_or_else(|| refuse("the image archive's number"))?; + } + b' ' | 0 => {} + _ => return Err(refuse("the image archive's number")), + } + } + Ok(value) +} + +/// A header's checksum: the sum of its bytes with the checksum field itself +/// read as spaces. +fn checksum(header: &[u8; 512]) -> u64 { + (header.iter().enumerate()) + .map(|(index, byte)| match index { + 148..=155 => u64::from(b' '), + _ => u64::from(*byte), + }) + .sum() +} + +/// `field` up to its first NUL, trimmed of trailing spaces. +fn text(field: &[u8]) -> &[u8] { + let end = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = &field[..end]; + let end = (text.iter().rposition(|byte| *byte != b' ')).map_or(0, |index| index + 1); + &text[..end] +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs new file mode 100644 index 0000000000..eaa2b36215 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/rootfs/types.rs @@ -0,0 +1,36 @@ +//! The image's unpacked root handle and errors: [`ImageRoot`] and [`refuse`]. +use crate::{Image, RuntimeError}; +use std::{io, path::PathBuf}; + +/// The kernel's own limit on one path. +pub(crate) const PATH_MAX: usize = 4096; + +/// One image's unpacked root, and the digest it is the root of. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ImageRoot { + image: Image, + directory: PathBuf, +} + +impl ImageRoot { + /// The digest this root is the unpacked image of. + pub fn image(&self) -> &Image { + &self.image + } + + /// The read-only directory the Sandbox's overlay takes as its lower layer. + pub fn directory(&self) -> &std::path::Path { + &self.directory + } + + /// The root of `image` at `directory`, which is already unpacked there. + pub(super) fn new(image: Image, directory: PathBuf) -> Self { + Self { image, directory } + } +} + +/// A refusal with the exact reason, as the runtime's I/O cause: the runtime's +/// protocol causes are static and these name paths and digests. +pub(crate) fn refuse(message: impl Into) -> RuntimeError { + RuntimeError::Io(io::Error::new(io::ErrorKind::InvalidData, message.into())) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs new file mode 100644 index 0000000000..4b650396e7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs @@ -0,0 +1,365 @@ +//! One namespace Sandbox: its directory, the init stage that owns its +//! namespaces, its cgroup and its exact attempts. Nothing is retried and no +//! Drop cleans up; a Drop only reaps an init process this process started +//! that has already exited, so it never stays a zombie of this process. +use super::{cgroup, host, paths::Paths, rootfs::ImageRoot}; +use crate::backend::native::LocalProcess; +use crate::provider::{LocalRecord, NamespaceRecord, SandboxRecord}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::Path, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The recorded process facts of a Sandbox, as a live process to check. +pub(super) fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} + +/// The facts a record carries into a reopen: the init stage it saw, the port +/// it listened on, its labels and the image its root is an overlay of. +pub(crate) struct Recorded { + pub(crate) init: LocalProcess, + pub(crate) port: u16, + pub(crate) labels: Vec<(String, String)>, + pub(crate) image: ImageRoot, +} + +/// Pause between two looks at a killed init process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for the group's processes to be gone after `cgroup.kill`. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged namespace Sandbox: the init stage of its namespaces, its +/// listener and the cgroup that owns everything it started. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) paths: Paths, + pub(super) cgroup: std::path::PathBuf, + pub(super) labels: Vec<(String, String)>, + /// The helper executable this backend's init and exec stages run. + pub(super) helper: std::path::PathBuf, + /// The unpacked image its root is an overlay of, and its digest. + pub(super) image: ImageRoot, + /// The init stage this process started, kept so its exit is reaped here. + /// `None` after a reopen. + pub(super) child: Mutex>, + pub(super) init: Option, + pub(super) port: u16, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// The owner a start returns once its init stage is spawned and its + /// cgroup exists; the listener port follows. + pub(super) fn unstarted( + id: ContainerId, + paths: Paths, + cgroup: std::path::PathBuf, + labels: Vec<(String, String)>, + helper: std::path::PathBuf, + image: ImageRoot, + ) -> Self { + Self { + id, + paths, + cgroup, + labels, + helper, + image, + child: Mutex::new(None), + init: None, + port: 0, + stop_attempted: false, + stopped: false, + delete_attempted: false, + deleted: false, + } + } + + /// Reopens a recorded Sandbox whose init stage still runs: the same + /// process (id, start time and boot), in its own cgroup, and its loopback + /// port. It starts nothing. + pub(crate) fn reopen( + paths: Paths, + cgroup: std::path::PathBuf, + id: ContainerId, + helper: std::path::PathBuf, + recorded: Recorded, + ) -> Result> { + let Recorded { + init, + port, + labels, + image, + } = recorded; + let mut owner = Self::unstarted(id, paths, cgroup, labels, helper, image); + owner.init = Some(init); + owner.port = port; + if !owner.paths.local.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved namespaces Sandbox")); + } + if !init.alive() || !host::in_cgroup(init.pid, &owner.cgroup) { + let mut failure = owner.invalid(crate::types::STOPPED); + failure.observed_container = Some(owner.id); + return Err(failure); + } + owner.endpoint()?; + Ok(owner) + } + + /// A recorded Sandbox whose init stage is gone + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub(crate) fn stopped( + paths: Paths, + cgroup: std::path::PathBuf, + id: ContainerId, + helper: std::path::PathBuf, + recorded: Recorded, + ) -> Self { + let mut owner = Self::unstarted(id, paths, cgroup, recorded.labels, helper, recorded.image); + owner.init = Some(recorded.init); + owner.port = recorded.port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + + /// Its private directory (setup, executable, overlay, mounts, log). + pub fn directory(&self) -> &Path { + &self.paths.local.directory + } + + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> std::path::PathBuf { + self.paths.local.tmp.clone() + } + + /// Its init stage process, once started. + pub fn process(&self) -> Option { + self.init + } + + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + + /// The image its root is an overlay of. + pub fn image(&self) -> &ImageRoot { + &self.image + } + + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let init = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: init.pid, + start_time: init.start, + boot: init.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + + /// Its record, naming this backend so another one never reopens it, and + /// the image digest so a reopen finds the same root again. + pub fn record(&self) -> Option { + let local = self.local_record()?; + Some(SandboxRecord::Namespaces(NamespaceRecord { + local, + image: self.image.image().clone(), + })) + } + + /// The daemon's endpoint while its init stage runs. A Sandbox whose init + /// stage exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(init) = self.init else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !init.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + + /// Ends the Sandbox once: one write to `cgroup.kill` asks the kernel to + /// `SIGKILL` every process of its group, descendants and commands + /// included, and no graceful drain exists to wait for. A Sandbox whose + /// init stage already ended is the same stopped state. + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original namespaces Stop admission")); + } + self.stop_attempted = true; + cgroup::kill(&self.cgroup).map_err(|error| self.io(error))?; + let wait = Duration::from_secs(u64::from(timeout_seconds)).max(KILL_WAIT); + let deadline = Instant::now() + wait; + loop { + self.reap(); + if self.init.is_none_or(|init| !init.alive()) { + match cgroup::populated(&self.cgroup) { + Ok(0) => break, + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => break, + Err(error) => return Err(self.io(error)), + } + } + if Instant::now() >= deadline { + return Err( + self.invalid("a process of the namespaces Sandbox runs after cgroup.kill") + ); + } + std::thread::sleep(PAUSE); + } + self.stopped = true; + Ok(()) + } + + /// Deletes its cgroup and its directory once, after a known stop: a + /// group that still holds a process is refused, so nothing is reported + /// gone while it runs. The private root it made is a mount of the + /// Sandbox's own namespaces and is gone with them. The shared Store is + /// outside the directory and is never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.init.is_some() && !self.stopped) { + return Err(self.invalid("original namespaces Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.init.is_some_and(LocalProcess::alive) { + return Err(self.invalid("namespaces Sandbox init runs again; nothing was deleted")); + } + cgroup::remove(&self.cgroup).map_err(|error| self.io(error))?; + match fs::remove_dir_all(self.directory()) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + // the private root holds nothing once its namespaces ended + match fs::remove_dir(&self.paths.root) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(&self.paths.local.log)?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + + /// Nothing to name: `cgroup.kill` owns every process the Sandbox started, + /// including one a command left in a session of its own. + pub fn track(&self, _pid: u32, _start_time: u64) -> Result<(), Box> { + Ok(()) + } + + /// Reaps the init stage this process started, if it exited. True when no + /// init stage this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// An init stage this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running one is + /// left alone: ending it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/create.rs b/engine/crates/layerfs-sandbox/src/backend/native/create.rs new file mode 100644 index 0000000000..c814869456 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/create.rs @@ -0,0 +1,284 @@ +//! One local daemon start: the Sandbox's private directory, its 0600 setup, +//! one spawn and its listener line. Nothing is retried. +use super::{ + process::LocalProcess, + sandbox::{Sandbox, LOG}, + search::search_path, +}; +use crate::backend::{identity, listener}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::Duration, +}; + +/// A local runtime root: one directory per Sandbox under it. +#[derive(Clone, Debug)] +pub struct Local { + root: PathBuf, + name: Option, + /// Variables the daemon starts with besides `PATH`. + environment: Vec<(String, String)>, +} + +/// Where one Sandbox keeps its parts, all under its directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalPaths { + /// The Sandbox directory, `/`, 0755. + pub directory: PathBuf, + /// Its daemon setup, `config/daemon.setup`, 0600 in a 0700 directory. + pub config: PathBuf, + /// Its daemon overlay template, `overlay/overlay.sqlite`, in a 0700 directory. + pub overlay: PathBuf, + /// Its Workspace mount home, `workspaces`, made 0755 by the daemon. + pub mounts: PathBuf, + /// Its daemon's output, `daemon.log`, 0600. + pub log: PathBuf, + /// Its commands' scratch directory, `tmp`, 1777 like a container's + /// `/tmp`, so what they leave there goes with the Sandbox. + pub tmp: PathBuf, +} + +/// The boundary one failed create reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LocalPhase { + /// Local checks; nothing made. + Validate, + /// The directory, setup and log file. + Create, + /// The daemon spawn. + Start, + /// The wait for the daemon's listener line. + Listener, +} + +/// A failed create. `sandbox` holds what it made (its directory, and its +/// daemon once spawned), for the caller to stop and delete once. +#[derive(Debug)] +pub struct LocalCreateFailure { + pub phase: LocalPhase, + pub sandbox: Option, + pub cause: RuntimeError, +} + +impl Local { + /// Selects `root`: an existing absolute directory owned by this user, + /// writable by it alone and traversable by others, as is every directory + /// above it (commands enter their Workspace below it only after taking + /// another identity, so one closed directory on the way, such as a 0700 + /// `/root`, would fail every command at spawn). + pub fn new(root: impl AsRef) -> Result { + let local = Self::native(root.as_ref())?; + let root = root.as_ref(); + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } + Ok(local) + } + /// Selects `root` for the native route, whose commands run as this user + /// (no other identity): the root's own checks as in [`Self::new`], but + /// the directories above it need not be traversable by others (a macOS + /// home is 0750). + pub fn native(root: impl AsRef) -> Result { + let root = root.as_ref(); + if !root.is_absolute() + || root + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return Err(RuntimeError::Protocol("absolute local runtime root")); + } + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() + || metadata.uid() != nix::unistd::Uid::effective().as_raw() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & 0o001 == 0 + { + return Err(RuntimeError::Protocol( + "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", + )); + } + Ok(Self { + root: root.to_owned(), + name: None, + environment: Vec::new(), + }) + } + /// Starts the next daemon with `environment` besides its search path (the + /// native route's `LAYERFS_KERNEL=nfs` and its mount helpers). Nothing + /// else of this process's environment reaches it. + pub fn environment(mut self, environment: Vec<(String, String)>) -> Self { + self.environment = environment; + self + } + /// Names the next Sandbox's directory `name`, under the Docker name rule + /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at + /// most 128), so it is one plain path component. + pub fn naming(mut self, name: &str) -> Result { + let bytes = name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 128 + || !bytes[0].is_ascii_alphanumeric() + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) + { + return Err(RuntimeError::Protocol("local Sandbox name")); + } + self.name = Some(name.to_owned()); + Ok(self) + } + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self) -> Result { + let name = self + .name + .as_deref() + .ok_or(RuntimeError::Protocol("local Sandbox name"))?; + let directory = self.root.join(name); + Ok(LocalPaths { + config: directory.join("config").join("daemon.setup"), + overlay: directory.join("overlay").join("overlay.sqlite"), + mounts: directory.join("workspaces"), + log: directory.join(LOG), + tmp: directory.join("tmp"), + directory, + }) + } + /// Makes the named Sandbox's directory, writes `config` (the encoded + /// daemon setup naming [`Self::paths`]) 0600, spawns `executable + /// --config ` in its own process group with no inherited + /// environment but the search path, and waits up to `wait` for its + /// listener line. Once anything is made, a failure returns it. + pub fn create( + &self, + executable: &Path, + config: &[u8], + wait: Duration, + labels: Vec<(String, String)>, + ) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(LocalCreateFailure { + phase, + sandbox, + cause, + }) + }; + let paths = self + .paths() + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let checked = (|| { + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("local daemon executable")); + } + if config.is_empty() || config.len() > 8192 || wait.is_zero() { + return Err(RuntimeError::Protocol("local daemon setup/startup wait")); + } + match fs::symlink_metadata(&paths.directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(_) => Err(RuntimeError::Protocol("local Sandbox directory exists")), + } + })(); + checked.map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted(id, paths.directory.clone(), labels); + let log = match prepare(&paths, config) { + Ok(log) => log, + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + }; + let spawned = log.try_clone().and_then(|output| { + Command::new(executable) + .arg("--config") + .arg(&paths.config) + .env_clear() + .env("PATH", search_path()) + .envs(self.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + let process = match LocalProcess::observe(child.id()) { + Ok(process) => process, + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + }; + sandbox.process = Some(process); + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.log, wait, &|| { + sandbox.reap(); + sandbox.process.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } + } +} + +/// The setup and overlay directories (0700), the scratch directory (1777), +/// the setup file (0600, written once) and the empty log file (0600) it +/// returns. +fn prepare(paths: &LocalPaths, config: &[u8]) -> io::Result { + // the process umask may narrow the create mode + fs::set_permissions(&paths.directory, Permissions::from_mode(0o755))?; + for file in [&paths.config, &paths.overlay] { + let parent = file + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "local path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.tmp)?; + fs::set_permissions(&paths.tmp, Permissions::from_mode(0o1777))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + private(&paths.config)?.write_all(config)?; + private(&paths.log) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/exec.rs b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs new file mode 100644 index 0000000000..1478054771 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs @@ -0,0 +1,133 @@ +//! One ordinary command to its end in a local Sandbox: the command identity, +//! the Workspace directory and a search path (on macOS this process's own); +//! nothing else of this process's environment, no shell of its own and no +//! implicit timeout. Each command leads a process group of its own, named in +//! the Sandbox until it is empty, so what it leaves running ends with the +//! Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox, search::search_path}; +use crate::backend::streams; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use nix::sys::signal::{killpg, Signal}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +/// One command's end: its exit code (128 + the signal for one a signal +/// ended, as the Engine reports it), its process id (diagnostic only) and the +/// bytes each stream delivered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalExit { + pub code: i32, + pub pid: u32, + pub stdout: u64, + pub stderr: u64, +} + +impl Sandbox { + /// Runs `request` once while this Sandbox's daemon runs: as its uid/gid + /// with no supplementary groups, in its directory, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. Standard input is not offered. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + self.endpoint()?; + let mut command = Command::new(&request.arguments[0]); + command + .args(&request.arguments[1..]) + .env_clear() + .env("PATH", search_path()) + .current_dir(Path::new(&request.directory)) + // std drops supplementary groups (setgroups 0) when it sets a uid as root + .uid(request.identity.uid) + .gid(request.identity.gid) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + // named before anything is waited for: the leader is not reaped + // before the wait below, so it is observable even once it exited + let named = LocalProcess::observe(pid) + .and_then(|leader| groups::name(self.directory(), leader).map(|()| leader)); + let leader = match named { + Ok(leader) => leader, + Err(error) => { + // an unnamed group would outlive the Sandbox: it ends here, once + if let Ok(group) = process::pid(pid) { + let _ = killpg(group, Signal::SIGKILL); + } + let _ = child.wait(); + let mut failure = self.io(error); + failure.attempted = true; + return Err(failure); + } + }; + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + // a name left behind is harmless: its group is checked again when ended + let _ = groups::forget_if_empty(self.directory(), leader); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("local command exit status"))?; + Ok(LocalExit { + code, + pid, + stdout: out_bytes, + stderr: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/groups.rs b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs new file mode 100644 index 0000000000..6f18869381 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs @@ -0,0 +1,149 @@ +//! The process groups a local Sandbox's processes run in. A container's +//! processes end with it; a local Sandbox's would outlive it on this kernel, +//! so each command starts its own process group, named in the Sandbox's +//! `groups` directory while it may have members, and every named group (and +//! the daemon's own) is killed when the Sandbox stops or is deleted. +//! +//! A group is named by its leader: id, start time and boot. Its id stays the +//! group's while any member lives, so a group whose leader is gone is still +//! known by it; a leader found with another start time means the id was +//! reused and the group is not this Sandbox's. A process that leaves its +//! group (`setsid`) is not followed unless it is named here too +//! ([`super::Sandbox::track`]). +use super::platform; +use super::process::{self, LocalProcess}; +use nix::{fcntl::OFlag, sys::signal::Signal}; +use std::{ + fs::{self, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::Path, + time::{Duration, Instant}, +}; + +/// The directory of a Sandbox that names its groups, one empty file each. +pub(super) const GROUPS: &str = "groups"; +/// Pause between two looks at a killed group. +const PAUSE: Duration = Duration::from_millis(20); + +/// Names the group `leader` leads (its id is the group's), once. +pub(super) fn name(directory: &Path, leader: LocalProcess) -> io::Result<()> { + let groups = directory.join(GROUPS); + // a Sandbox started before groups were named has no directory yet + match fs::DirBuilder::new().mode(0o700).create(&groups) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(groups.join(entry(leader)?)) + .map(drop) +} + +/// Forgets the group `leader` led once no member of it is left. Called after +/// the leader was reaped; a group that still has members stays named. +pub(super) fn forget_if_empty(directory: &Path, leader: LocalProcess) -> io::Result<()> { + match nix::sys::signal::killpg(process::pid(leader.pid)?, None) { + Err(nix::errno::Errno::ESRCH) => {} + _ => return Ok(()), + } + match fs::remove_file(directory.join(GROUPS).join(entry(leader)?)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Kills every group the Sandbox at `directory` named, and the group of +/// `daemon` (its leader), once with KILL, as a container's end kills what +/// ran in it, then waits up to `wait` for their members to be gone. Groups +/// of another boot ended with it. +pub(super) fn end( + directory: &Path, + daemon: Option, + wait: Duration, +) -> io::Result<()> { + let mut leaders: Vec = daemon.into_iter().collect(); + match fs::read_dir(directory.join(GROUPS)) { + Ok(items) => { + for item in items { + let name = item?.file_name(); + let leader = name.to_str().and_then(parse).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "local Sandbox group name") + })?; + leaders.push(leader); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let this_boot = process::boot()?; + leaders.retain(|leader| leader.boot == Some(this_boot)); + let table = processes()?; + let mut killed = Vec::new(); + for leader in leaders { + if !members(&table, leader).is_empty() { + match nix::sys::signal::killpg(process::pid(leader.pid)?, Signal::SIGKILL) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => killed.push(leader), + Err(errno) => return Err(errno.into()), + } + } + } + let deadline = Instant::now() + wait; + loop { + let table = processes()?; + if killed + .iter() + .all(|leader| members(&table, *leader).is_empty()) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::other( + "a process of the local Sandbox still runs after KILL", + )); + } + std::thread::sleep(PAUSE); + } +} + +/// The running members of `leader`'s group: none when the leader's id now +/// names another process (the id was reused, so the group is another's). +fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { + let reused = (table.iter()).any(|(pid, stat)| *pid == leader.pid && stat.start != leader.start); + if reused { + return Vec::new(); + } + (table.iter()) + .filter(|(_, stat)| stat.group == leader.pid && !stat.ended()) + .map(|(pid, _)| *pid) + .collect() +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +fn processes() -> io::Result> { + platform::processes() +} + +/// A group's file name: `--`. +fn entry(leader: LocalProcess) -> io::Result { + let boot = leader + .boot + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "a group leader's boot"))?; + Ok(format!("{boot:032x}-{}-{}", leader.pid, leader.start)) +} + +fn parse(name: &str) -> Option { + let mut parts = name.split('-'); + let boot = parts.next().filter(|b| b.len() == 32)?; + let process = LocalProcess { + boot: Some(u128::from_str_radix(boot, 16).ok()?), + pid: parts.next()?.parse().ok()?, + start: parts.next()?.parse().ok()?, + }; + parts.next().is_none().then_some(process) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/mod.rs b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs new file mode 100644 index 0000000000..1f1fb4ad81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs @@ -0,0 +1,40 @@ +//! Native process runtime: each Sandbox is one daemon process on this kernel, +//! started from a private directory, with no container, image or Engine. +//! +//! It is for a machine that is itself the isolation boundary (a microVM, a +//! CI job, a developer's own Mac). It isolates nothing: the daemon and its +//! commands share this kernel's processes, network, users and filesystem with +//! everything else on it. Commands still run as their own nonroot identity +//! where the platform allows it, and what they leave running ends with the +//! Sandbox. One attempt per operation, as in the Docker runtime; nothing is +//! retried. +//! +//! The lifecycle here is one implementation for both platforms. What differs +//! is read where it lives: [`super::macos`] supplies Darwin process facts, the +//! `mount` table and `umount -f` for the loopback NFS the macOS daemon serves; +//! [`super::linux`] supplies `/proc` facts, the `mountinfo` table and a lazy +//! `MNT_DETACH` for the FUSE mount the root daemon serves. +//! +//! The daemon and its commands start with one search path and nothing else of +//! this process's environment: on Linux a container's, whatever this process +//! has; on macOS this process's own `PATH`, since a native command is this +//! user's own tool (Homebrew's `/opt/homebrew/bin`, `~/.local/bin`, a version +//! manager's `node`) and no image supplies it. +mod create; +mod exec; +mod groups; +pub(crate) mod process; +mod provider; +mod sandbox; +pub(crate) mod search; +#[cfg(target_os = "linux")] +pub(crate) use super::linux as platform; +#[cfg(target_os = "macos")] +pub(crate) use super::macos as platform; +pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; +pub use exec::LocalExit; +pub use process::LocalProcess; +pub use provider::{ + NativeLinuxInstance, NativeLinuxProvider, NativeMacosInstance, NativeMacosProvider, +}; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/native/process.rs b/engine/crates/layerfs-sandbox/src/backend/native/process.rs new file mode 100644 index 0000000000..74d58bf34d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/process.rs @@ -0,0 +1,85 @@ +//! A local process named by its id, kernel start time and boot, so a reused +//! id, or the same id and start time after a reboot or snapshot restore, is +//! never taken for the process that was started. The three facts come from +//! `/proc` on Linux and from `proc_pidinfo`/`sysctl` on macOS (see +//! [`super::macos`]). +pub(crate) use super::platform::{boot, stat}; +use nix::{sys::signal::Signal, unistd::Pid}; +use std::io; + +/// One started process: its id, its start time in clock ticks since boot +/// (`/proc//stat` field 22) and the boot it started in +/// (`/proc/sys/kernel/random/boot_id`). All must match for it to count as +/// alive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalProcess { + pub pid: u32, + pub start: u64, + /// The boot id as a number. `None` only for a record written before boot + /// ids were kept: it is matched by id and start time alone, as it was + /// then, and never signalled. + pub boot: Option, +} + +impl LocalProcess { + /// The process running now under `pid`, read once from this kernel, in + /// this boot. + pub fn observe(pid: u32) -> io::Result { + let stat = stat(pid)?; + Ok(Self { + pid, + start: stat.start, + boot: Some(boot()?), + }) + } + /// True while this exact process exists and has not exited: a zombie + /// (or a dead entry) is not alive, and another process under the same id + /// has another start time or another boot. + pub fn alive(self) -> bool { + let this_boot = self + .boot + .is_none_or(|recorded| boot().ok() == Some(recorded)); + match stat(self.pid) { + Ok(stat) => this_boot && stat.start == self.start && !stat.ended(), + Err(_) => false, + } + } + /// Sends `signal` once, only after this exact process was observed alive. + pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if self.boot.is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "a local process recorded without its boot is never signalled", + )); + } + if !self.alive() { + return Ok(()); + } + match nix::sys::signal::kill(pid(self.pid)?, signal) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), + Err(errno) => Err(errno.into()), + } + } +} + +/// What this kernel says of one process. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Stat { + pub(crate) state: u8, + pub(crate) group: u32, + pub(crate) start: u64, +} + +impl Stat { + /// A zombie or a dead entry: it runs nothing any more. + pub(crate) fn ended(self) -> bool { + matches!(self.state, b'Z' | b'X' | b'x') + } +} + +/// A process id as the kernel's signed type. +pub(super) fn pid(id: u32) -> io::Result { + i32::try_from(id) + .map(Pid::from_raw) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width")) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/provider.rs b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs new file mode 100644 index 0000000000..b5e48d9916 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs @@ -0,0 +1,178 @@ +//! The existing native runtime as a concrete backend; no alternate process engine. +use super::{Local, LocalPhase, LocalProcess, Sandbox}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, LocalRecord, MountDriver, + ProviderCapabilities, ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct NativeProvider { + local: Local, +} +/// Native macOS processes with loopback NFS and the caller's identity. +pub type NativeMacosProvider = NativeProvider; +/// Native Linux processes with FUSE and a separate nonroot command identity. +pub type NativeLinuxProvider = NativeProvider; +/// The existing local owner, using Darwin process facts. +pub type NativeMacosInstance = Sandbox; +/// The existing local owner, using Linux `/proc` process facts. +pub type NativeLinuxInstance = Sandbox; +impl NativeProvider { + /// Refuses other platforms before accessing the root or launching anything. + pub fn new(root: impl AsRef) -> Result { + if (LINUX && !cfg!(target_os = "linux")) || (!LINUX && !cfg!(target_os = "macos")) { + return Err(RuntimeError::Unsupported("native provider platform")); + } + if LINUX && !nix::unistd::Uid::effective().is_root() { + return Err(RuntimeError::Unsupported( + "native Linux provider requires root", + )); + } + Ok(Self { + local: if LINUX { + Local::new(root)? + } else { + Local::native(root)? + }, + }) + } + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: if LINUX { + "native-linux" + } else { + "native-macos" + }, + isolation: IsolationModel::HostProcess, + mount_driver: if LINUX { + MountDriver::Fuse + } else { + MountDriver::LoopbackNfs + }, + requires_image: false, + search_path_policy: if LINUX { + SearchPathPolicy::FixedContainer + } else { + SearchPathPolicy::InheritHost + }, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Local { + command_identity, + mut environment, + } = request.options + else { + return Err(fail(RuntimeError::Unsupported( + "native provider requires local launch options", + ))); + }; + command_identity.check().map_err(fail)?; + if (!LINUX + && (command_identity.uid != nix::unistd::Uid::effective().as_raw() + || command_identity.gid != nix::unistd::Gid::effective().as_raw())) + || environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != if LINUX { "fuse" } else { "nfs" }) + }) + { + return Err(fail(RuntimeError::Protocol( + "native command identity/environment", + ))); + } + environment.retain(|(key, _)| key != "LAYERFS_KERNEL"); + environment.push(( + "LAYERFS_KERNEL".into(), + if LINUX { "fuse" } else { "nfs" }.into(), + )); + let local = self + .local + .clone() + .naming(&request.name) + .map_err(fail)? + .environment(environment); + local + .create( + &request.daemon_executable, + &request.daemon_config, + request.startup_timeout, + request.labels, + ) + .map_err(|failure| { + Box::new(LaunchFailure { + phase: match failure.phase { + LocalPhase::Validate => LaunchPhase::Validate, + LocalPhase::Create => LaunchPhase::Create, + LocalPhase::Start => LaunchPhase::Start, + LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: failure + .sandbox + .map(|sandbox| Box::new(crate::backend::Sandbox::Native(sandbox))), + cause: failure.cause, + }) + }) + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Sandbox::reopen( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + ) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Ok(Sandbox::stopped( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + )) + } +} +fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs new file mode 100644 index 0000000000..8c67717882 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs @@ -0,0 +1,335 @@ +//! One local Sandbox: its directory, its daemon process and its exact +//! attempts. No Drop cleanup or replay, as for a container; a Drop only +//! reaps a daemon this process started that has already exited. A stop +//! returns only once a daemon this process started is reaped, so it never +//! stays a zombie of this process. +use super::{groups, platform, process::LocalProcess}; +use crate::provider::LocalRecord; +use crate::{ContainerId, RuntimeError, WireFailure}; +use nix::sys::signal::Signal; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The daemon's output file in a Sandbox directory, stdout and stderr together. +pub(super) const LOG: &str = "daemon.log"; +/// Pause between two looks at a stopping process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for a killed process to be gone. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged local Sandbox directory and, once started, its daemon. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) directory: PathBuf, + pub(super) labels: Vec<(String, String)>, + pub(super) process: Option, + pub(super) port: u16, + /// The daemon this process started, kept so its exit is reaped here and + /// it does not stay a zombie of this process. `None` after a reopen. + pub(super) child: Mutex>, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + /// Its process groups were killed and found empty (by its stop). + pub(super) ended: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// A directory made by a start that has not spawned its daemon yet. + pub(super) fn unstarted( + id: ContainerId, + directory: PathBuf, + labels: Vec<(String, String)>, + ) -> Self { + Self { + id, + directory, + labels, + process: None, + port: 0, + child: Mutex::new(None), + stop_attempted: false, + stopped: false, + ended: false, + delete_attempted: false, + deleted: false, + } + } + /// Reopens a recorded Sandbox whose daemon still runs: the same process + /// (id and start time) and its loopback port. It starts nothing. + pub fn reopen( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Result> { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + if !owner.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved local Sandbox")); + } + owner.endpoint()?; + Ok(owner) + } + /// A recorded Sandbox that [`Self::reopen`] found stopped + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub fn stopped( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Self { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + /// Its private directory (setup, overlay, Workspace mounts, daemon log). + pub fn directory(&self) -> &Path { + &self.directory + } + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + self.directory.join("tmp") + } + /// Its daemon process, once started. + pub fn process(&self) -> Option { + self.process + } + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let process = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: process.pid, + start_time: process.start, + boot: process.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + /// The daemon's endpoint while that exact process runs. A daemon that + /// exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(process) = self.process else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !process.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then + /// KILL (a daemon this process started has ended once reaped); then + /// KILL to every process group of its commands and its daemon, as a + /// container's end kills what ran in it. A daemon already gone is the + /// same stopped state. Crash teardown, not a graceful filesystem drain. + pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original local Stop admission")); + } + self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = platform::mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } + if let Some(process) = self.process { + self.end( + process, + Signal::SIGTERM, + Duration::from_secs(u64::from(seconds)), + )?; + if !self.gone(process) { + self.end(process, Signal::SIGKILL, KILL_WAIT)?; + } + if process.alive() { + return Err(self.invalid("local daemon still runs after KILL")); + } + } + self.stopped = true; + groups::end(&self.directory, self.process, KILL_WAIT).map_err(|error| self.io(error))?; + self.ended = true; + Ok(()) + } + /// Deletes its directory once, after a known stop: any process left in + /// its groups is killed first (when its stop did not run here, as for a + /// daemon found stopped), and any mount its daemon left is detached, so + /// nothing writes into it or is removed through a mount. The shared + /// Store is outside it and never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.process.is_some() && !self.stopped) { + return Err(self.invalid("original local Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.process.is_some_and(LocalProcess::alive) { + return Err(self.invalid("local daemon runs again; nothing was deleted")); + } + if !self.ended { + groups::end(&self.directory, self.process, KILL_WAIT) + .map_err(|error| self.io(error))?; + self.ended = true; + } + platform::mounts::detach(&self.directory).map_err(|error| self.io(error))?; + match fs::remove_dir_all(&self.directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(self.directory.join(LOG))?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + /// Names the process group `pid` leads or is about to lead: a process + /// started at `start` by a command of this Sandbox in a session of its + /// own (the SDK's `setsid` background process), so it ends with the + /// Sandbox too. One already gone needs nothing. + pub fn track(&self, pid: u32, start: u64) -> Result<(), Box> { + match LocalProcess::observe(pid) { + Ok(process) if process.start == start && process.alive() => { + groups::name(&self.directory, process).map_err(|error| self.io(error)) + } + _ => Ok(()), + } + } + /// Reaps the daemon this process started, if it exited. True when no + /// daemon this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + /// The daemon has ended and, if this process started it, is reaped. Its + /// main thread can end (a zombie id, so not alive) while its other + /// threads still run and it cannot be reaped yet: not gone. + fn gone(&self, process: LocalProcess) -> bool { + self.reap() && !process.alive() + } + /// Sends `signal` once and waits up to `wait` for the process to be gone. + /// A KILL also goes to the daemon this process started and has not + /// reaped: its id cannot be reused until reaped, so it is still the one + /// started, though its main thread may have ended already. + fn end( + &self, + process: LocalProcess, + signal: Signal, + wait: Duration, + ) -> Result<(), Box> { + process.signal(signal).map_err(|error| self.io(error))?; + if signal == Signal::SIGKILL { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + let _ = started.kill(); + } + } + let deadline = Instant::now() + wait; + loop { + if self.gone(process) || Instant::now() >= deadline { + return Ok(()); + } + std::thread::sleep(PAUSE); + } + } + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// A daemon this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running daemon + /// is left alone: stopping it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/search.rs b/engine/crates/layerfs-sandbox/src/backend/native/search.rs new file mode 100644 index 0000000000..3db862f089 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/search.rs @@ -0,0 +1,26 @@ +//! The search path a local daemon and its commands start with: a container's +//! on Linux, this process's own on macOS (the native route). +use std::ffi::OsString; + +/// A container's search path: what a local daemon and its commands start with +/// on Linux, where the local runtime stands in for a container. +const CONTAINER_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on Linux [`CONTAINER_PATH`], whatever this +/// process has. +#[cfg(not(target_os = "macos"))] +pub(crate) fn search_path() -> OsString { + OsString::from(CONTAINER_PATH) +} + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on macOS this process's own `PATH`; without +/// one, or an empty one, [`CONTAINER_PATH`] and Homebrew's `/opt/homebrew/bin`. +#[cfg(target_os = "macos")] +pub(crate) fn search_path() -> OsString { + match std::env::var_os("PATH") { + Some(path) if !path.is_empty() => path, + _ => OsString::from(format!("{CONTAINER_PATH}:/opt/homebrew/bin")), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/runtime.rs b/engine/crates/layerfs-sandbox/src/backend/runtime.rs new file mode 100644 index 0000000000..43ed8208a0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/runtime.rs @@ -0,0 +1,307 @@ +//! The concrete runtimes a Sandbox can run on: one enum names the deployment's +//! choice, and [`Sandbox`] is the launched owner over it. There is no provider +//! trait; each backend is a concrete type and dispatch is a `match` here. +use super::docker::{self, DockerProvider}; +#[cfg(target_os = "linux")] +use super::namespaces::{self, NamespaceProvider}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use super::native::{self, NativeLinuxProvider, NativeMacosProvider}; +use crate::provider::{ + ExecExit, LaunchFailure, LaunchRequest, LocalRecord, ProviderCapabilities, SandboxInspection, + SandboxRecord, +}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +/// What Windows would need and does not have yet: no host runtime exists. +const WINDOWS: &str = "windows native Sandbox runtime"; + +/// One deployment's runtime choice, kept for every Sandbox it starts. +/// +/// A Sandbox's record names the variant that made it; a platform without a +/// runtime is refused before any effect, never replaced by another backend. +#[derive(Clone, Debug)] +pub enum Runtime { + /// A local Docker Engine container; the default. + Docker(DockerProvider), + /// A local daemon process serving loopback NFS as this user (macOS). + #[cfg(any(target_os = "linux", target_os = "macos"))] + MacosNative(NativeMacosProvider), + /// A root local daemon process serving FUSE, commands as a nonroot + /// identity (Linux). + #[cfg(any(target_os = "linux", target_os = "macos"))] + LinuxNative(NativeLinuxProvider), + /// A root daemon in its own mount, PID and IPC namespaces and cgroup, + /// commands entering them as a nonroot identity (Linux). + #[cfg(target_os = "linux")] + Namespaces(NamespaceProvider), + /// Placeholder: selecting Windows refuses before any effect. + Windows, +} + +impl Runtime { + pub fn capabilities(&self) -> Result { + match self { + Self::Docker(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => Ok(provider.capabilities()), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => Ok(provider.capabilities()), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// Local resource ownership, checked by API before encoding daemon topology. + /// Nonlocal providers return None. No resource is created by this observation. + pub fn local_directory(&self, name: &str) -> Result, RuntimeError> { + match self { + Self::Docker(_) => Ok(None), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// The named native Sandbox's paths, for its daemon setup. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn native_paths(&self, name: &str) -> Result { + match self { + Self::MacosNative(provider) => provider.paths(name), + Self::LinuxNative(provider) => provider.paths(name), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name), + Self::Docker(_) | Self::Windows => Err(RuntimeError::Unsupported( + "native Sandbox paths need a native runtime", + )), + } + } + + /// Starts one Sandbox once; a failed launch keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + match self { + Self::Docker(provider) => provider.launch(request).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .launch(request) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(Box::new(LaunchFailure { + phase: crate::provider::LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(WINDOWS), + })), + } + } + + /// Reopen starts nothing and checks the recorded deployment or process identity. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen_stopped(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen_stopped(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } +} + +/// The launched owner of one Sandbox, on the runtime that started it. +#[derive(Debug)] +pub enum Sandbox { + Docker(docker::Sandbox), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Native(native::Sandbox), + #[cfg(target_os = "linux")] + Namespaces(Box), +} + +impl Sandbox { + pub fn identity(&self) -> ContainerId { + match self { + Self::Docker(sandbox) => sandbox.identity(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.identity(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.identity(), + } + } + + pub fn endpoint(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.endpoint(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.endpoint(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.endpoint(), + } + } + + /// One command to its end, its streams copied to the sinks as they arrive. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + match self { + Self::Docker(sandbox) => docker::run_command(sandbox, request, stdout, stderr), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + let exit = sandbox.run(request, stdout, stderr)?; + Ok(ExecExit { + code: exit.code, + pid: Some(exit.pid), + stdout_bytes: exit.stdout, + stderr_bytes: exit.stderr, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.run(request, stdout, stderr), + } + } + + /// Names a background process a command started in a session of its own, + /// so it ends with the Sandbox as a container's descendants do. + pub fn track(&self, pid: u32, start_time: u64) -> Result<(), Box> { + match self { + // Container teardown owns all descendants; no host process is adopted. + Self::Docker(_) => Ok(()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.track(pid, start_time), + // its cgroup owns every process the Sandbox started, whatever + // session it left, so nothing needs naming + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.track(pid, start_time), + } + } + + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + match self { + Self::Docker(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.log_tail(lines, cap), + } + } + + pub fn inspect(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.inspect(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + } + } + + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.stop(timeout_seconds), + } + } + + pub fn delete(&mut self) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.delete(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.delete(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.delete(), + } + } + + /// Where its commands keep scratch files: `/tmp` in a container, the + /// Sandbox's own directory for a native one, deleted with it. + pub fn scratch_directory(&self) -> String { + match self { + Self::Docker(_) => "/tmp".into(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.tmp().display().to_string(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.tmp().display().to_string(), + } + } + + /// Physical facts for the binding's existing record format, when this is native. + pub fn local_record(&self) -> Option { + match self { + Self::Docker(_) => None, + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.local_record(), + #[cfg(target_os = "linux")] + Self::Namespaces(_) => None, + } + } + + /// Persisted deployment facts for reopening this runtime instance. + pub fn record(&self) -> Option { + match self { + Self::Docker(sandbox) => Some(SandboxRecord::Docker(sandbox.request().clone().into())), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(_) => self.local_record().map(SandboxRecord::Local), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.record(), + } + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/streams.rs b/engine/crates/layerfs-sandbox/src/backend/streams.rs new file mode 100644 index 0000000000..6c6e011a17 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/streams.rs @@ -0,0 +1,23 @@ +//! An ordinary command's streams, copied to the caller's sinks as they +//! arrive. Both local backends read the same two pipes and count only what +//! each sink accepted. +use std::io::{self, Read, Write}; + +/// Copies one stream to its sink until end of stream; the bytes delivered. +pub(crate) fn copy(source: Option, sink: &mut (dyn Write + Send)) -> io::Result { + let Some(mut source) = source else { + return Ok(0); + }; + let mut total = 0u64; + let mut window = [0u8; 8192]; + loop { + let n = match source.read(&mut window) { + Ok(0) => return Ok(total), + Ok(n) => n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + sink.write_all(&window[..n])?; + total = total.saturating_add(n as u64); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs b/engine/crates/layerfs-sandbox/src/backend/strings.rs similarity index 86% rename from engine/crates/layerfs-sandbox/src/backend/docker/strings.rs rename to engine/crates/layerfs-sandbox/src/backend/strings.rs index 1d2afc32e8..fb22cf7c79 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs +++ b/engine/crates/layerfs-sandbox/src/backend/strings.rs @@ -41,6 +41,27 @@ impl Json { self.string_each(|c| text.push(c, capture))?; Ok(text) } + /// Reads a string value into `out` as UTF-8, refusing one longer than + /// `max`: the Engine's selected fields are short, and the longest string + /// this crate reads is an image archive's path. Its only caller unpacks + /// images, which this crate does on Linux only. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn text_into(&mut self, out: &mut Vec, max: usize) -> Result<(), RuntimeError> { + let mut too_long = false; + self.string_each(|c| { + let mut encoded = [0; 4]; + let bytes = c.encode_utf8(&mut encoded).as_bytes(); + if out.len() + bytes.len() > max { + too_long = true; + } else { + out.extend_from_slice(bytes); + } + })?; + if too_long { + return Err(RuntimeError::Protocol("JSON string length")); + } + Ok(()) + } pub fn string_equals(&mut self, expected: &str) -> Result { let mut expected = expected.chars(); let mut equal = true; diff --git a/engine/crates/layerfs-sandbox/src/backend/windows/README.md b/engine/crates/layerfs-sandbox/src/backend/windows/README.md new file mode 100644 index 0000000000..b667620c81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/windows/README.md @@ -0,0 +1,49 @@ +# Windows：占位（尚未实现） + +状态：占位。选到 `Runtime::Windows` 时返回既有的“不支持此平台”错误 +（`RuntimeError::Unsupported`），不 panic、不新建公开错误码、不产生任何副作用。 + +## 现在有什么 + +- `layerfs_sandbox::Runtime` 的四个取值：`Docker`、`MacosNative`、 + `LinuxNative`、`Windows`。前三个已有后端，Windows 只有这个变体。 +- `Runtime::{capabilities, local_directory, launch, reopen, reopen_stopped}` + 在 Windows 上全部返回 `Unsupported`；`Sandbox` 句柄没有 Windows 分支， + 因为 Windows 上不可能启动出实例。 +- `layerfs-sandbox` 的 `backend` 只在 unix 下编译，`layerfs-node` 也只用 + unix 系统调用，所以今天在 Windows 主机上选不到这个变体：它存在的意义是 + 把“没有 Windows 宿主运行时”写进类型，而不是让调用方自己判平台。 + +## 为什么不做“Windows 本地进程”后端 + +macOS 原生的前提是：daemon 以本用户身份跑 NFS 回环服务，命令以同一身份 +运行，一切在同一个内核里。Windows 上没有等价物：LayerFS 的 FUSE 服务端是 +Linux 内核特性，Store 的 WAL 与目录锁只在一个内核内协调（见 #352 直接 +Store 访问的安全条件），命令身份、挂载可见性和进程组语义都要另建一套。 +因此 Windows 的路线不是“再写一个本地后端”，而是把 Linux 侧的能力搬到一个 +Linux 宿主里。 + +## 计划方向 + +两条路线，都由 #515 的 placement 设置表达，而不是在 Windows 本机上跑 daemon： + +1. **WSL2 分发（近期）**：Windows 上的集成适配器把 SDK 调用转发给 + 每个 LayerFS home 一个的、仅回环且带密钥的 worker；worker 在一个选定的 + WSL2 Linux 分发里直接用真实 SDK 打开共享 Store（ext4/xfs/btrfs）， + 并用该分发的 Docker Engine 起 Sandbox daemon。Windows 只拥有 UI、绑定和 + trace，Linux 拥有本地句柄和项目发现。这是仓库根 `AGENTS.md` 已记录的 + Windows 例外，规则不变：只连回环、每次调用只尝试一次、丢失的回复不重放、 + 结果不确定的 Init 保留其 Linux home 和卷。 +2. **Hyper-V / VM 方向（后续）**：每个 Store 一个 Linux VM，Sandbox 是 VM 内 + 的命名空间；宿主侧只通过一条私有流连到 VM 内的 worker。见 #515 §6 + （`vm:` placement 与各 provider 的 VM/磁盘/私有流对照表）和 + #512 的实验结论。 + +## 实现时要先定的事 + +- placement 与后端选择的设置名（#515 V9 提议 `LAYERFS_PLACEMENT` 与 + `LAYERFS_SANDBOX_BACKEND`，只读 `LAYERFS_*`；集成自己的前缀由集成映射）。 +- Windows 上 `deploy.sandbox`、`deploy.localDir` 等现有设置的语义：#515 + 要求 product SDK 与 daemon 不为 Windows 改动，Windows 差异留在适配器里。 +- 到那时 `Runtime` 是否新增一个持有 worker 端点的变体（那时才需要数据）， + 以及记录里如何标注该 Sandbox 由哪个运行时创建（无字段即 Docker）。 diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs new file mode 100644 index 0000000000..6135b177e7 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/exec.rs @@ -0,0 +1,168 @@ +//! The exec stage: one command inside a running Sandbox. The helper joins the +//! Sandbox's cgroup first (after entering the Sandbox's mount namespace this +//! host path no longer resolves), opens the three namespace handles, then +//! enters them. Its child is created inside the Sandbox's PID namespace, runs +//! as the requested uid:gid and inherits no-new-privs, so file capabilities +//! and setuid bits cannot raise it again. +use nix::{ + sched::{setns, CloneFlags}, + sys::prctl::set_no_new_privs, +}; +use std::{ + ffi::OsString, + fs::{self, File}, + io::Write, + os::unix::process::{CommandExt, ExitStatusExt}, + path::{Path, PathBuf}, + process::Command, +}; + +/// One parsed exec stage. +struct Run { + init: u32, + cgroup: PathBuf, + uid: u32, + gid: u32, + directory: PathBuf, + arguments: Vec, +} + +/// Runs one command and exits with its code (128 + signal for one a signal +/// ended). A setup failure ends this process with a signal, so the caller +/// never reads it as a command's exit code. +pub(super) fn run(arguments: &[OsString]) -> Result { + let run = parse(arguments)?; + join(&run.cgroup)?; + let namespaces = Namespaces::open(run.init)?; + namespaces.enter()?; + set_no_new_privs().map_err(|error| format!("setting no-new-privs: {error}"))?; + let (program, rest) = run.arguments.split_first().ok_or("no command")?; + let mut command = Command::new(program); + command + .args(rest) + .uid(run.uid) + .gid(run.gid) + .current_dir(&run.directory) + .process_group(0); + let mut child = match command.spawn() { + Ok(child) => child, + // the command never started: this stage's own failure, which must + // never look like the command's status + Err(error) => fail(&format!("starting {}: {error}", program.to_string_lossy())), + }; + let status = child + .wait() + .map_err(|error| format!("waiting for the command: {error}"))?; + Ok(status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(super::EXIT_FAILURE)) +} + +/// The three namespace handles of a Sandbox's init stage, all opened before +/// any is entered: after entering the Sandbox's mount namespace this host's +/// `/proc//ns` no longer resolves. +struct Namespaces { + mount: File, + pid: File, + ipc: File, +} + +impl Namespaces { + fn open(init: u32) -> Result { + let directory = PathBuf::from(format!("/proc/{init}/ns")); + let open = |name: &str| { + File::open(directory.join(name)) + .map_err(|error| format!("{}: {error}", directory.join(name).display())) + }; + Ok(Self { + mount: open("mnt")?, + pid: open("pid_for_children").or_else(|_| open("pid"))?, + ipc: open("ipc")?, + }) + } + + fn enter(self) -> Result<(), String> { + let Self { mount, pid, ipc } = self; + setns(mount, CloneFlags::CLONE_NEWNS) + .map_err(|error| format!("entering the Sandbox's mount namespace: {error}"))?; + setns(pid, CloneFlags::CLONE_NEWPID) + .map_err(|error| format!("entering the Sandbox's PID namespace: {error}"))?; + setns(ipc, CloneFlags::CLONE_NEWIPC) + .map_err(|error| format!("entering the Sandbox's IPC namespace: {error}")) + } +} + +/// The Sandbox's own cgroup, so `cgroup.kill` ends this command too. +fn join(cgroup: &Path) -> Result<(), String> { + let pid = nix::unistd::getpid().as_raw(); + let mut file = fs::OpenOptions::new() + .write(true) + .open(cgroup.join("cgroup.procs")) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}"))?; + file.write_all(pid.to_string().as_bytes()) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}")) +} + +/// Ends this stage the way only a failure can: with a signal, never an exit +/// code, so the caller cannot read it as the command's status. +fn fail(message: &str) -> ! { + eprintln!("layerfs-sandbox-helper: {message}"); + let _ = nix::sys::signal::raise(nix::sys::signal::Signal::SIGKILL); + std::process::abort() +} + +fn parse(arguments: &[OsString]) -> Result { + let (mut init, mut uid, mut gid) = (0, 0, 0); + let (mut cgroup, mut directory) = (PathBuf::new(), PathBuf::new()); + let mut command = Vec::new(); + let mut index = 0; + while index < arguments.len() { + let flag = arguments[index].to_str(); + if flag == Some("--") { + command.extend(arguments[index + 1..].iter().cloned()); + break; + } + let value = arguments + .get(index + 1) + .ok_or_else(|| format!("{} needs a value", arguments[index].to_string_lossy()))?; + match flag { + Some("--init") => init = number(value)?, + Some("--uid") => uid = number(value)?, + Some("--gid") => gid = number(value)?, + Some("--cgroup") => cgroup = PathBuf::from(value), + Some("--cwd") => directory = PathBuf::from(value), + _ => { + return Err(format!( + "unknown argument {}", + arguments[index].to_string_lossy() + )) + } + } + index += 2; + } + if init == 0 + || uid == 0 + || gid == 0 + || cgroup.as_os_str().is_empty() + || !directory.is_absolute() + || command.is_empty() + { + return Err("missing or invalid exec argument".to_owned()); + } + Ok(Run { + init, + cgroup, + uid, + gid, + directory, + arguments: command, + }) +} + +fn number(value: &OsString) -> Result { + value + .to_str() + .and_then(|text| text.parse().ok()) + .ok_or_else(|| format!("{} is not a number", value.to_string_lossy())) +} diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs new file mode 100644 index 0000000000..1576db1163 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/init.rs @@ -0,0 +1,109 @@ +//! The two init stages. The outer one creates the namespaces and joins the +//! cgroup; its first child, the inner one, is PID 1 of the new PID namespace, +//! so it builds the private root, starts the daemon and reaps. Neither stage +//! retries anything: a failure ends the stage and is written to the Sandbox's +//! daemon log, which the start's error keeps. +use super::mounts; +use layerfs_sandbox::backend::HelperSpec; +use nix::{ + sched::{unshare, CloneFlags}, + sys::wait::{waitpid, WaitStatus}, + unistd::Pid, +}; +use std::{ + fs, + io::Write, + path::Path, + process::{Command, Stdio}, +}; + +/// The outer init stage: joins the Sandbox's cgroup, becomes the first +/// process of its mount, PID and IPC namespaces, and starts the inner stage. +/// Its exit code is the daemon's, so the Sandbox ends when the daemon does. +pub(super) fn outer(spec_path: &Path) -> Result { + let spec = read(spec_path)?; + join(&spec.cgroup)?; + unshare(CloneFlags::CLONE_NEWNS | CloneFlags::CLONE_NEWPID | CloneFlags::CLONE_NEWIPC) + .map_err(|error| format!("creating the Sandbox's namespaces: {error}"))?; + // this mount namespace is the Sandbox's alone: nothing it mounts + // propagates back to the VM's own mount tree + nix::mount::mount( + None::<&Path>, + Path::new("/"), + None::<&str>, + nix::mount::MsFlags::MS_REC | nix::mount::MsFlags::MS_PRIVATE, + None::<&str>, + ) + .map_err(|error| format!("making the Sandbox's mount tree private: {error}"))?; + let executable = + std::env::current_exe().map_err(|error| format!("this helper's own path: {error}"))?; + let mut inner = Command::new(executable) + .arg("init-inner") + .arg("--spec") + .arg(spec_path) + .spawn() + .map_err(|error| format!("starting the Sandbox's init stage: {error}"))?; + let status = inner + .wait() + .map_err(|error| format!("waiting for the Sandbox's init stage: {error}"))?; + Ok(code(status)) +} + +/// The inner init stage, PID 1 of the Sandbox's PID namespace: builds and +/// pivots to the private root, starts the daemon as root and reaps, so no +/// process of the Sandbox is left a zombie. +pub(super) fn inner(spec_path: &Path) -> Result { + let spec = read(spec_path)?; + mounts::build(&spec)?; + mounts::pivot(&spec.root)?; + let daemon = Command::new(&spec.daemon) + .arg("--config") + .arg(&spec.config) + .current_dir("/") + .stdin(Stdio::null()) + .spawn() + .map_err(|error| format!("starting the daemon: {error}"))?; + let daemon_pid = Pid::from_raw( + i32::try_from(daemon.id()).map_err(|_| "daemon process id width".to_owned())?, + ); + loop { + match waitpid(None, None) { + Ok(WaitStatus::Exited(pid, code)) if pid == daemon_pid => return Ok(code), + Ok(WaitStatus::Signaled(pid, signal, _)) if pid == daemon_pid => { + return Ok(128 + signal as i32) + } + // an orphan of a command: reaped here, as PID 1 does + Ok(_) => continue, + Err(nix::errno::Errno::ECHILD) => { + return Err("the daemon ended without a waitable status".to_owned()) + } + Err(error) => return Err(format!("reaping the Sandbox's processes: {error}")), + } + } +} + +/// The Sandbox's own cgroup, written by the Sandbox's own init stage: every +/// process it creates afterwards inherits it, so one write ends them all. +fn join(cgroup: &Path) -> Result<(), String> { + let pid = nix::unistd::getpid().as_raw(); + let mut file = fs::OpenOptions::new() + .write(true) + .open(cgroup.join("cgroup.procs")) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}"))?; + file.write_all(pid.to_string().as_bytes()) + .map_err(|error| format!("joining the Sandbox's cgroup: {error}")) +} + +fn read(spec_path: &Path) -> Result { + let bytes = + fs::read(spec_path).map_err(|error| format!("reading {}: {error}", spec_path.display()))?; + HelperSpec::parse(&bytes).map_err(|error| format!("reading {}: {error}", spec_path.display())) +} + +fn code(status: std::process::ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(super::EXIT_FAILURE) +} diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs new file mode 100644 index 0000000000..2e9b470c00 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/main.rs @@ -0,0 +1,65 @@ +//! The namespaces backend's helper: one single-threaded binary with two +//! stages, so the calls that act on the calling process (`unshare`, `setns`, +//! `pivot_root`, `PR_SET_NO_NEW_PRIVS`) run in a process of their own and the +//! product crate stays free of `unsafe`. +//! +//! - `init --spec `: joins the Sandbox's cgroup, creates its mount, PID +//! and IPC namespaces and starts the inner stage, which is PID 1 of them. +//! - `init-inner --spec `: builds the private root, pivots to it, starts +//! `layerfs-daemon` as root and reaps, as an init process does. +//! - `exec …`: enters a running Sandbox's namespaces, becomes the requested +//! nonroot identity with no-new-privs and runs one command. +#![deny(unsafe_code)] + +#[cfg(target_os = "linux")] +mod exec; +#[cfg(target_os = "linux")] +mod init; +#[cfg(target_os = "linux")] +mod mounts; + +#[cfg(target_os = "linux")] +fn main() { + match run() { + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("layerfs-sandbox-helper: {error}"); + std::process::exit(EXIT_FAILURE); + } + } +} + +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!("layerfs-sandbox-helper: the namespaces Sandbox runtime is Linux only"); + std::process::exit(125); +} + +/// A stage's own failure; a command's exit code is never this. +#[cfg(target_os = "linux")] +const EXIT_FAILURE: i32 = 125; + +#[cfg(target_os = "linux")] +fn run() -> Result { + let arguments: Vec = std::env::args_os().skip(1).collect(); + let (stage, rest) = arguments.split_first().ok_or(USAGE)?; + match stage.to_str() { + Some("init") => init::outer(spec(rest)?), + Some("init-inner") => init::inner(spec(rest)?), + Some("exec") => exec::run(rest), + _ => Err(USAGE.to_owned()), + } +} + +/// `--spec `, the only argument of either init stage. +#[cfg(target_os = "linux")] +fn spec(arguments: &[std::ffi::OsString]) -> Result<&std::path::Path, String> { + match arguments { + [flag, path] if flag == "--spec" => Ok(std::path::Path::new(path)), + _ => Err(USAGE.to_owned()), + } +} + +#[cfg(target_os = "linux")] +const USAGE: &str = "usage: layerfs-sandbox-helper init|init-inner --spec | exec --init \ + --cgroup --uid --gid --cwd -- …"; diff --git a/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs new file mode 100644 index 0000000000..426faba656 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/bin/layerfs-sandbox-helper/mounts.rs @@ -0,0 +1,201 @@ +//! The Sandbox's root: the unpacked image (a read-only lower layer) with the +//! Sandbox's own empty writable layer on top, so every command sees the image +//! as `/` and everything it writes belongs to that Sandbox alone. The +//! Sandbox's own directory and the Store's directory are bound at their own +//! paths so every path keeps one meaning on both sides of the pivot, and a +//! private `/proc`, `/dev` and `/tmp` are mounted over the image's own. +//! Nothing the Sandbox mounts reaches the VM's mount tree: the init stage made +//! this mount namespace private first. +use layerfs_sandbox::backend::HelperSpec; +use nix::{ + mount::{mount, umount2, MntFlags, MsFlags}, + unistd::chdir, +}; +use std::{ + fs, io, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +/// The devices a daemon and ordinary commands need, bound from this host's +/// `/dev`; nothing else of `/dev` is visible. +const DEVICES: [&str; 7] = ["fuse", "null", "zero", "full", "random", "urandom", "tty"]; + +/// Builds the Sandbox's root at `spec.root`: the image as the overlay's lower +/// layer, the Sandbox's own `upper` and `work` above it, and the private mounts +/// that must not come from the image. +pub(super) fn build(spec: &HelperSpec) -> Result<(), String> { + fs::set_permissions(&spec.root, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("{}: {error}", spec.root.display()))?; + overlay(spec)?; + // an overlay's own root takes its attributes from the empty upper + // directory, which is root's 0700 on this host: the root a command sees + // must instead be one it can traverse, as a container's `/` is + fs::set_permissions(&spec.root, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("{}: {error}", spec.root.display()))?; + let proc_directory = inside(&spec.root, Path::new("/proc"))?; + mount( + None::<&Path>, + &proc_directory, + Some("proc"), + MsFlags::MS_NOSUID | MsFlags::MS_NODEV | MsFlags::MS_NOEXEC, + None::<&str>, + ) + .map_err(|error| format!("mounting {}: {error}", proc_directory.display()))?; + devices(&inside(&spec.root, Path::new("/dev"))?)?; + let scratch = inside(&spec.root, Path::new("/tmp"))?; + tmpfs( + &scratch, + MsFlags::MS_NOSUID | MsFlags::MS_NODEV, + "mode=1777", + )?; + // the daemon's setup, overlay, Workspace mounts and `tmp` + bind(&spec.sandbox, &inside(&spec.root, &spec.sandbox)?, false)?; + // the shared Store's directory: SQLite's WAL and the writers' directory + // lock see the same inode as every other opener on this kernel + bind(&spec.store, &inside(&spec.root, &spec.store)?, false)?; + Ok(()) +} + +/// The image as the root's lower layer and the Sandbox's own upper and work +/// directories above it. The result is one mount, so a command's writes stay +/// in the upper directory and the image is never modified. +fn overlay(spec: &HelperSpec) -> Result<(), String> { + let data = format!( + "lowerdir={},upperdir={},workdir={}", + spec.lower.display(), + spec.upper.display(), + spec.work.display() + ); + mount( + Some("overlay"), + &spec.root, + Some("overlay"), + MsFlags::MS_NOSUID, + Some(data.as_str()), + ) + .map_err(|error| { + format!( + "mounting the image {} over {} with {}: {error}", + spec.lower.display(), + spec.root.display(), + data + ) + }) +} + +/// Pivots to `root`, so the VM's own tree is no longer reachable from the +/// Sandbox, and drops the old root. +pub(super) fn pivot(root: &Path) -> Result<(), String> { + // the private root must be a mount point of its own, which the overlay is + chdir(root).map_err(|error| format!("{}: {error}", root.display()))?; + nix::unistd::pivot_root(Path::new("."), Path::new(".")) + .map_err(|error| format!("pivoting to the Sandbox's root: {error}"))?; + umount2(Path::new("."), MntFlags::MNT_DETACH) + .map_err(|error| format!("dropping the VM's root: {error}"))?; + chdir("/").map_err(|error| format!("entering the Sandbox's root: {error}")) +} + +/// A private `/dev`: the basic devices, a `devpts` of its own for pseudo +/// terminals, and a `shm` directory for what expects one. +fn devices(dev: &Path) -> Result<(), String> { + tmpfs(dev, MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC, "mode=0755")?; + let host = Path::new("/dev"); + for name in DEVICES { + let source = host.join(name); + if fs::symlink_metadata(&source).is_err() { + continue; + } + // a bind needs its target to exist; the device then replaces this + // placeholder in the private tmpfs + let target = dev.join(name); + fs::write(&target, b"").map_err(|error| format!("{}: {error}", target.display()))?; + bind(&source, &target, false)?; + } + let pts = dev.join("pts"); + directory(&pts, 0o755)?; + mount( + None::<&Path>, + &pts, + Some("devpts"), + MsFlags::MS_NOSUID | MsFlags::MS_NOEXEC, + Some("newinstance,ptmxmode=0666,mode=0620"), + ) + .map_err(|error| format!("mounting {}: {error}", pts.display()))?; + let shm = dev.join("shm"); + directory(&shm, 0o755)?; + tmpfs(&shm, MsFlags::MS_NOSUID | MsFlags::MS_NODEV, "mode=1777")?; + for (name, target) in [ + ("ptmx", "pts/ptmx"), + ("fd", "/proc/self/fd"), + ("stdin", "/proc/self/fd/0"), + ("stdout", "/proc/self/fd/1"), + ("stderr", "/proc/self/fd/2"), + ] { + let link = dev.join(name); + let _ = fs::remove_file(&link); + std::os::unix::fs::symlink(target, &link) + .map_err(|error| format!("{}: {error}", link.display()))?; + } + Ok(()) +} + +/// `path`'s place inside `root`, made: `/a/b` becomes `/a/b`. +fn inside(root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix("/") + .map_err(|_| format!("{} is not an absolute runtime path", path.display()))?; + let target = root.join(relative); + fs::create_dir_all(&target).map_err(|error| format!("{}: {error}", target.display()))?; + Ok(target) +} + +/// A tmpfs at `target`, so nothing it holds outlives the namespaces. +fn tmpfs(target: &Path, flags: MsFlags, data: &str) -> Result<(), String> { + mount( + Some("tmpfs"), + target, + Some("tmpfs"), + flags | MsFlags::MS_NOSUID, + Some(data), + ) + .map_err(|error| format!("mounting a tmpfs at {}: {error}", target.display())) +} + +/// `source` at `target`, read-only when `read_only`. +fn bind(source: &Path, target: &Path, read_only: bool) -> Result<(), String> { + mount( + Some(source), + target, + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REC, + None::<&str>, + ) + .map_err(|error| { + format!( + "binding {} at {}: {error}", + source.display(), + target.display() + ) + })?; + if !read_only { + return Ok(()); + } + mount( + None::<&Path>, + target, + None::<&str>, + MsFlags::MS_BIND | MsFlags::MS_REMOUNT | MsFlags::MS_RDONLY, + None::<&str>, + ) + .map_err(|error| format!("making {} read-only: {error}", target.display())) +} + +fn directory(path: &Path, mode: u32) -> Result<(), String> { + if let Err(error) = fs::DirBuilder::new().mode(mode).create(path) { + if error.kind() != io::ErrorKind::AlreadyExists { + return Err(format!("{}: {error}", path.display())); + } + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..497fa2024b 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,9 +1,14 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. -#![forbid(unsafe_code)] +// unsafe only in backend/macos/darwin.rs: macOS process facts come from +// proc_pidinfo, a system call (`ps` is setuid, which Seatbelt refuses) +#![deny(unsafe_code)] #[cfg(unix)] pub mod backend; +pub mod provider; mod types; +#[cfg(unix)] +pub use backend::{Runtime, Sandbox}; pub use types::{ - CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, PendingRequest, - RuntimeError, WireFailure, + CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, ExecRequest, Image, + PendingRequest, RuntimeError, WireFailure, }; diff --git a/engine/crates/layerfs-sandbox/src/provider/capabilities.rs b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs new file mode 100644 index 0000000000..a11548c3eb --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs @@ -0,0 +1,30 @@ +//! Explicit runtime capabilities; these describe support, not qualification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProviderCapabilities { + pub name: &'static str, + pub isolation: IsolationModel, + pub mount_driver: MountDriver, + pub requires_image: bool, + pub search_path_policy: SearchPathPolicy, + pub supports_resource_limits: bool, + pub direct_store_access: bool, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IsolationModel { + Container, + HostProcess, + MicroVm, + RemoteCloud, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MountDriver { + Fuse, + LoopbackNfs, + SharedFs, + None, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchPathPolicy { + FixedContainer, + InheritHost, +} diff --git a/engine/crates/layerfs-sandbox/src/provider/launch.rs b/engine/crates/layerfs-sandbox/src/provider/launch.rs new file mode 100644 index 0000000000..0c654ad481 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/launch.rs @@ -0,0 +1,26 @@ +//! One-attempt launch failure: acknowledged partial ownership and the last phase reached. +use crate::backend::Sandbox; +use crate::RuntimeError; +use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LaunchPhase { + Validate, + Create, + Start, + Listener, +} +/// The caller retains the exact resource even when launch fails after creating it. +pub struct LaunchFailure { + pub phase: LaunchPhase, + pub instance: Option>, + pub cause: RuntimeError, +} +impl fmt::Debug for LaunchFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchFailure") + .field("phase", &self.phase) + .field("identity", &self.instance.as_ref().map(|i| i.identity())) + .field("cause", &self.cause) + .finish() + } +} diff --git a/engine/crates/layerfs-sandbox/src/provider/mod.rs b/engine/crates/layerfs-sandbox/src/provider/mod.rs new file mode 100644 index 0000000000..b9eee1e304 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/mod.rs @@ -0,0 +1,12 @@ +//! Runtime inputs and observations; concrete backends own their own lifecycle. +mod capabilities; +mod launch; +mod options; +pub use capabilities::{IsolationModel, MountDriver, ProviderCapabilities, SearchPathPolicy}; +pub use launch::{LaunchFailure, LaunchPhase}; +pub use options::{ + DockerRecord, ExecExit, LaunchRequest, LocalRecord, NamespaceRecord, ProviderLaunchOptions, + SandboxInspection, SandboxRecord, +}; +/// Compatible name for the existing exact 64-hex runtime identity. +pub type SandboxId = crate::ContainerId; diff --git a/engine/crates/layerfs-sandbox/src/provider/options.rs b/engine/crates/layerfs-sandbox/src/provider/options.rs new file mode 100644 index 0000000000..0ea23bd979 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/options.rs @@ -0,0 +1,193 @@ +//! Launch inputs and observations independent of the daemon control protocol. +use crate::{CommandIdentity, ContainerId}; +use std::{fmt, path::PathBuf, time::Duration}; +/// Configuration bytes may contain secrets; Debug only reports their length. +#[derive(Clone)] +pub struct LaunchRequest { + pub name: String, + pub daemon_executable: PathBuf, + pub daemon_config: Vec, + pub labels: Vec<(String, String)>, + pub startup_timeout: Duration, + pub options: ProviderLaunchOptions, +} +impl fmt::Debug for LaunchRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchRequest") + .field("name", &self.name) + .field("daemon_executable", &self.daemon_executable) + .field("config_bytes", &self.daemon_config.len()) + .field("startup_timeout", &self.startup_timeout) + .finish_non_exhaustive() + } +} +#[derive(Clone, Debug)] +pub enum ProviderLaunchOptions { + Local { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + }, + /// The Linux namespace backend: a namespaced daemon plus the Store file it + /// opens. `store` is the exact file; its directory is what the Sandbox's + /// private root binds, so SQLite's WAL and the writers' directory lock see + /// the same inode as every other opener on this kernel. `image` is the + /// digest its root is an overlay of, and `archive` the docker-archive in + /// the bundle that digest names, which is unpacked once per digest. + Namespaces { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + store: PathBuf, + image: crate::Image, + archive: PathBuf, + }, + Docker { + image: String, + store_volume: String, + port: u16, + connect_host: Option, + }, +} +/// Persisted physical facts supplied by the record owner; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalRecord { + pub name: String, + pub pid: u32, + pub start_time: u64, + pub boot: Option, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// A namespaced Linux Sandbox's record: the local facts (its directory name, +/// init process identity and listener port) and the image digest its root is +/// an overlay of, so a reopen finds the same unpacked image. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamespaceRecord { + pub local: LocalRecord, + pub image: crate::Image, +} +impl NamespaceRecord { + /// The physical local facts every record of this backend keeps. + pub fn local(&self) -> &LocalRecord { + &self.local + } + /// The image digest the Sandbox's root is an overlay of. + pub fn image(&self) -> &crate::Image { + &self.image + } +} +/// Persisted deployment facts for a Docker container; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DockerRecord { + pub image: String, + pub store_volume: String, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// Recorded runtime identity and deployment facts for reopen. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SandboxRecord { + Local(LocalRecord), + /// A namespaced Linux Sandbox; the physical facts kept are the same as a + /// local one (its directory name, init process identity and listener port), + /// plus the image digest its root is an overlay of. The variant itself + /// names the backend, so a record is never reopened by another one. + Namespaces(NamespaceRecord), + Docker(DockerRecord), +} +impl SandboxRecord { + pub fn local(&self) -> Option<&LocalRecord> { + match self { + Self::Local(record) => Some(record), + _ => None, + } + } + pub fn namespaces(&self) -> Option<&NamespaceRecord> { + match self { + Self::Namespaces(record) => Some(record), + _ => None, + } + } + pub fn docker(&self) -> Option<&DockerRecord> { + match self { + Self::Docker(record) => Some(record), + _ => None, + } + } +} +impl From for SandboxRecord { + fn from(record: LocalRecord) -> Self { + Self::Local(record) + } +} +impl From<&LocalRecord> for SandboxRecord { + fn from(record: &LocalRecord) -> Self { + Self::Local(record.clone()) + } +} +impl From for SandboxRecord { + fn from(record: DockerRecord) -> Self { + Self::Docker(record) + } +} +impl From<&DockerRecord> for SandboxRecord { + fn from(record: &DockerRecord) -> Self { + Self::Docker(record.clone()) + } +} +impl From for DockerRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image, + store_volume: request.store_volume, + port: request.port, + labels: request.labels, + } + } +} +impl From<&crate::backend::docker::SandboxRequest> for DockerRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image.clone(), + store_volume: request.store_volume.clone(), + port: request.port, + labels: request.labels.clone(), + } + } +} +impl From for crate::backend::docker::SandboxRequest { + fn from(record: DockerRecord) -> Self { + Self { + image: record.image, + store_volume: record.store_volume, + port: record.port, + labels: record.labels, + } + } +} +impl From for SandboxRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +impl From<&crate::backend::docker::SandboxRequest> for SandboxRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecExit { + pub code: i32, + /// Docker may not expose a process id after exit. + pub pid: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, +} +/// Unknown resource/timestamp fields stay None rather than invented zeroes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SandboxInspection { + pub identity: ContainerId, + pub running: bool, + pub exit_code: Option, + pub process_id: Option, + pub scratch_bytes: Option, +} diff --git a/engine/crates/layerfs-sandbox/src/types/failure.rs b/engine/crates/layerfs-sandbox/src/types/failure.rs index de4ddfe27d..26c3b3d915 100644 --- a/engine/crates/layerfs-sandbox/src/types/failure.rs +++ b/engine/crates/layerfs-sandbox/src/types/failure.rs @@ -3,6 +3,9 @@ use std::{fmt, io}; /// Original ordinary-runtime refusal or I/O cause. #[derive(Debug)] pub enum RuntimeError { + /// Original Docker provider stage failure, including retained transfer/stream owners. + #[cfg(unix)] + Docker(Box), Io(io::Error), Protocol(&'static str), /// Original HTTP status, with a separately retained bounded error-body diagnostic. @@ -18,6 +21,8 @@ impl From for RuntimeError { impl fmt::Display for RuntimeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + #[cfg(unix)] + Self::Docker(failure) => write!(f, "Docker provider: {failure:?}"), Self::Io(e) => write!(f, "runtime I/O: {e}"), Self::Protocol(e) => write!(f, "runtime protocol: {e}"), Self::Http(s) => write!(f, "runtime HTTP status {s}"), @@ -53,6 +58,24 @@ pub struct WireFailure { /// The cause [`WireFailure::container_stopped`] reports. pub(crate) const STOPPED: &str = "exact stopped container observation"; impl WireFailure { + /// A non-HTTP cause before an attempt, with no invented transport facts. + pub fn local(cause: RuntimeError) -> Box { + Box::new(Self { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: None, + requested_exec: None, + error_body: None, + }) + } /// The Engine still has the exact requested container, intact, but it is /// not running: its daemon exited, so nothing it held survives. pub fn container_stopped(&self) -> bool { diff --git a/engine/crates/layerfs-sandbox/src/types/image.rs b/engine/crates/layerfs-sandbox/src/types/image.rs new file mode 100644 index 0000000000..107ce35bae --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/image.rs @@ -0,0 +1,43 @@ +//! A Sandbox image's identity: the digest a deployment names, which is also +//! the name its unpacked root is cached under. It is a value type on every +//! platform, so a record written where the image backend runs can still be +//! read where it cannot. +use super::RuntimeError; + +/// The prefix every image digest carries. +const ALGORITHM: &str = "sha256:"; + +/// A Sandbox image's identity: `sha256:<64 lowercase hex>`, the same text the +/// Docker backend's `deploy.image` holds and a docker-archive names its config +/// blob by. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Image(String); + +impl Image { + /// Parses `sha256:<64 lowercase hex>`, the only form an archive's config + /// digest is compared with. + pub fn parse(text: &str) -> Result { + let hex = text.strip_prefix(ALGORITHM).unwrap_or(""); + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(RuntimeError::Protocol( + "Sandbox image must be an exact digest, sha256:<64 lowercase hex>", + )); + } + Ok(Self(text.to_owned())) + } + + /// The whole digest text, `sha256:<64 hex>`. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The 64 hex digits without the algorithm: the name this image's + /// unpacked root is cached under. + pub fn hex(&self) -> &str { + &self.0[ALGORITHM.len()..] + } +} diff --git a/engine/crates/layerfs-sandbox/src/types/mod.rs b/engine/crates/layerfs-sandbox/src/types/mod.rs index 69827343cb..2d8c1768ae 100644 --- a/engine/crates/layerfs-sandbox/src/types/mod.rs +++ b/engine/crates/layerfs-sandbox/src/types/mod.rs @@ -2,9 +2,13 @@ mod execution; mod failure; mod identity; +mod image; +mod request; mod transfer; pub use execution::{CommandIdentity, ExecInspection}; pub(crate) use failure::STOPPED; pub use failure::{ErrorDiagnostic, RuntimeError, WireFailure}; pub use identity::{ContainerId, ExecId}; +pub use image::Image; +pub use request::ExecRequest; pub use transfer::PendingRequest; diff --git a/engine/crates/layerfs-sandbox/src/types/request.rs b/engine/crates/layerfs-sandbox/src/types/request.rs new file mode 100644 index 0000000000..e14e795a9b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/request.rs @@ -0,0 +1,48 @@ +//! Provider-independent ordinary command metadata and validation. +use crate::{CommandIdentity, RuntimeError}; +use std::fmt; +/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. +#[derive(Clone)] +pub struct ExecRequest { + pub identity: CommandIdentity, + pub arguments: Vec, + pub environment: Vec, + pub directory: String, + pub stdin: bool, +} +impl fmt::Debug for ExecRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecRequest") + .field("identity", &self.identity) + .field("arguments_count", &self.arguments.len()) + .field("environment_count", &self.environment.len()) + .field("directory", &self.directory) + .field("stdin", &self.stdin) + .finish() + } +} +impl ExecRequest { + pub(crate) fn check(&self) -> Result<(), RuntimeError> { + self.identity.check()?; + if self.arguments.is_empty() + || self.arguments[0].is_empty() + || !std::path::Path::new(&self.directory).is_absolute() + || self + .arguments + .iter() + .chain(&self.environment) + .chain(std::iter::once(&self.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if self + .environment + .iter() + .any(|v| !v.contains('=') || v.starts_with('=')) + { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + Ok(()) + } +} diff --git a/engine/crates/layerfs-sandbox/tests/docker_provider.rs b/engine/crates/layerfs-sandbox/tests/docker_provider.rs new file mode 100644 index 0000000000..8022f295d6 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/docker_provider.rs @@ -0,0 +1,394 @@ +//! Docker provider contracts exercised through bounded Unix HTTP peers, without an Engine. +#![cfg(unix)] +#[allow(dead_code)] +#[path = "support/engine.rs"] +mod peer; +use layerfs_sandbox::{ + backend::docker::{DockerProvider, ProviderFailure, Sandbox, SandboxRequest}, + provider::{ + DockerRecord, LaunchPhase, LaunchRequest, LocalRecord, ProviderLaunchOptions, SandboxRecord, + }, + CommandIdentity, ContainerId, ExecRequest, Runtime, RuntimeError, +}; +use peer::{fixed, frame, upgrade, Peer}; +use std::{ + fs, io, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; +const CID: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const IMAGE: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +static NEXT: AtomicUsize = AtomicUsize::new(0); +fn deployment() -> SandboxRequest { + SandboxRequest { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + } +} +struct Binary(PathBuf); +impl Binary { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "h22-provider-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&path, b"sealed executable").unwrap(); + Self(path) + } + fn launch(&self) -> LaunchRequest { + LaunchRequest { + name: "h22-sandbox-1".into(), + daemon_executable: self.0.clone(), + daemon_config: b"private config".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: None, + }, + } + } +} +impl Drop for Binary { + fn drop(&mut self) { + fs::remove_file(&self.0).unwrap(); + } +} +fn logs(payload: &[u8]) -> Vec { + let mut bytes = format!("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.multiplexed-stream\r\nContent-Length: {}\r\n\r\n", payload.len()).into_bytes(); + bytes.extend(payload); + bytes +} +fn launch_responses() -> Vec> { + vec![ + fixed( + 200, + br#"{"Name":"provider-store","Driver":"local","Scope":"local","Options":null}"#, + ), + fixed(201, format!(r#"{{"Id":"{CID}"}}"#).as_bytes()), + fixed(200, b""), + fixed(204, b""), + logs(&frame(1, b"LAYERFS_DAEMON_LISTEN 0.0.0.0:30421\n")), + ] +} +fn exec_request() -> ExecRequest { + ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), "printf ordinary".into()], + environment: vec![], + directory: "/workspaces/1".into(), + stdin: false, + } +} +fn exec_inspection(running: bool, exit: &str) -> Vec { + fixed(200, format!(r#"{{"ID":"{EID}","ContainerID":"{CID}","Running":{running},"ExitCode":{exit},"Pid":42}}"#).as_bytes()) +} +#[test] +fn launch_stream_log_stop_delete_and_negotiated_versions_use_the_original_runtime() { + let binary = Binary::new(); + for minor in [48, 54] { + let mut responses = vec![fixed( + 200, + format!(r#"{{"ApiVersion":"1.{minor}"}}"#).as_bytes(), + )]; + responses.extend(launch_responses()); + let mut payload = frame(1, &vec![b'x'; 1024 * 1024 + 3]); + payload.extend(frame(2, b"err")); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&payload), + exec_inspection(false, "7"), + logs(&frame(1, b"daemon log\n")), + fixed(204, b""), + fixed(204, b""), + ]); + let (docker, peer) = Peer::scripted(responses, 8192); + let provider = Runtime::Docker(DockerProvider::new(docker)); + assert_eq!(provider.capabilities().unwrap().name, "docker"); + let mut instance = provider.launch(binary.launch()).unwrap(); + assert_eq!(instance.identity().to_string(), CID); + assert_eq!(instance.scratch_directory(), "/tmp"); + assert!(instance.local_record().is_none()); + assert_eq!( + instance.record(), + Some(SandboxRecord::Docker(DockerRecord { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + })) + ); + instance.track(42, 1).unwrap(); + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let exit = instance + .run(&exec_request(), &mut stdout, &mut stderr) + .unwrap(); + assert_eq!((exit.code, exit.pid), (7, Some(42))); + assert_eq!(exit.stdout_bytes, 1024 * 1024 + 3); + assert_eq!(stdout.len() as u64, exit.stdout_bytes); + assert_eq!(stderr, b"err"); + assert_eq!(instance.log_tail(20, 2048).unwrap(), b"daemon log\n"); + instance.stop(1).unwrap(); + instance.delete().unwrap(); + assert!(instance.stop(1).is_err()); + assert!(instance.delete().is_err()); + let calls = peer.finish(); + assert_eq!(calls.len(), 12); + assert!(calls[0].starts_with(b"GET /version ")); + for call in &calls[1..] { + assert!(String::from_utf8_lossy(call) + .split(' ') + .nth(1) + .unwrap() + .starts_with(&format!("/v1.{minor}/"))); + } + assert!(String::from_utf8_lossy(&calls[2]).contains("create?name=h22-sandbox-1")); + assert!(String::from_utf8_lossy(calls.last().unwrap()).contains("force=false&v=false")); + } +} +#[test] +fn uncertain_create_retains_partial_id_and_never_adopts_or_cleans_it() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.truncate(2); + responses[1] = fixed(201, format!(r#"{{"Id":"{CID}","Id":"{CID}"}}"#).as_bytes()); + let (docker, peer) = Peer::new(responses, 1); + let failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, LaunchPhase::Create); + assert!(failure.instance.is_none()); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed original cause") + }; + let ProviderFailure::Create(cause) = *cause else { + panic!("create custody") + }; + assert_eq!(cause.transfer.observed_container.unwrap().to_string(), CID); + assert!(cause.transfer.attempted); + assert_eq!(peer.finish().len(), 2); +} +#[test] +fn upload_start_and_listener_failures_keep_acknowledged_instance_and_original_receipts() { + let binary = Binary::new(); + for (stage, expected) in [ + (2, LaunchPhase::Create), + (3, LaunchPhase::Start), + (4, LaunchPhase::Listener), + ] { + let mut responses = launch_responses(); + responses.truncate(stage + 1); + responses[stage] = fixed(409, b"refused"); + let (docker, peer) = Peer::new(responses, 8192); + let mut failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, expected); + assert_eq!(failure.instance.take().unwrap().identity().to_string(), CID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed receipt") + }; + match (*cause, stage) { + (ProviderFailure::Upload(f), 2) => { + assert!(f.work.tar_bytes > 0); + assert_eq!(f.transfer.status, Some(409)); + } + (ProviderFailure::Start(f), 3) => assert_eq!(f.status, Some(409)), + (ProviderFailure::Listener(f), 4) => assert_eq!(f.transfer.status, Some(409)), + _ => panic!("original stage lost"), + } + assert_eq!(peer.finish().len(), stage + 1); + } +} +#[test] +fn validation_and_stdin_refuse_without_an_engine_attempt() { + let binary = Binary::new(); + let (docker, peer) = Peer::new(vec![], 1); + let provider = Runtime::Docker(DockerProvider::new(docker.clone())); + let mut request = binary.launch(); + request.name = "../escape".into(); + assert_eq!( + provider.launch(request).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut bad_host = binary.launch(); + bad_host.options = ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: Some("bad host\0".into()), + }; + assert_eq!( + provider.launch(bad_host).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = binary.launch(); + wrong_options.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + let foreign = SandboxRecord::Local(LocalRecord { + name: "test".into(), + pid: 1, + start_time: 1, + boot: None, + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(CID).unwrap(); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + assert!( + !provider + .reopen_stopped(id, &foreign) + .err() + .unwrap() + .attempted + ); + let stopped = Sandbox::stopped(docker, ContainerId::parse(CID).unwrap(), deployment()); + let mut request = exec_request(); + request.stdin = true; + assert!( + !layerfs_sandbox::Sandbox::Docker(stopped) + .run(&request, &mut Vec::new(), &mut Vec::new()) + .unwrap_err() + .attempted + ); + assert!(peer.finish().is_empty()); +} +#[test] +fn eof_with_unknown_exit_preserves_the_handle_without_polling_or_replay() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"ok")), + exec_inspection(true, "null"), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecExitUnknown { + handle, + inspection, + progress, + } = *cause + else { + panic!("EOF must not invent exit") + }; + assert_eq!(progress.stdout, 2); + assert_eq!(progress.stderr, 0); + assert!(inspection.running); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(peer.finish().len(), 8); +} +#[test] +fn exec_stage_failures_keep_deciding_wire_facts_and_delivered_progress() { + let binary = Binary::new(); + for stage in 0..3 { + let mut responses = launch_responses(); + let mut exec = vec![ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"delivered")), + exec_inspection(false, "0"), + ]; + exec.truncate(stage + 1); + exec[stage] = fixed(409, b"original refusal"); + responses.extend(exec); + let (docker, peer) = Peer::new(responses, 8192); + let owner = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = owner + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.status, Some(409)); + assert!(failure.sent_bytes > 0); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("original custody") + }; + match (*cause, stage) { + (ProviderFailure::ExecCreate(f), 0) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes) + } + (ProviderFailure::ExecStart(f), 1) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes); + assert_eq!(f.created.identity().1.to_string(), EID); + } + ( + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + 2, + ) => { + assert_eq!(failure.sent_bytes, f.sent_bytes); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(progress.stdout, 9); + assert_eq!(progress.stderr, 0); + } + _ => panic!("wrong stage"), + } + assert_eq!(peer.finish().len(), 6 + stage); + } +} +struct BrokenSink; +impl io::Write for BrokenSink { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn failed_sink_retains_pending_output_and_exec_identity_without_inspection_or_cleanup() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"undelivered")), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut BrokenSink, &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecOutput { handle, failure } = *cause else { + panic!("stream owner retained") + }; + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(failure.output.progress().stdout, 0); + assert_eq!(failure.output.progress().wire_payload, 11); + assert_eq!(peer.finish().len(), 7); +} diff --git a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs index 9c59dc4c25..e3d18209d7 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs @@ -405,6 +405,9 @@ fn reopened_from(body: &str) -> Box { fn only_the_exact_intact_container_not_running_is_a_stopped_container() { let stopped = reopened_from(&exited()); assert!(stopped.container_stopped()); + assert!(stopped.attempted); + assert_eq!(stopped.status, Some(200)); + assert!(stopped.sent_bytes >= stopped.request_header_bytes && stopped.request_header_bytes > 0); assert!(!stopped.container_gone()); let other = "c".repeat(64); assert!(!reopened_from(&exited().replace(CID, &other)).container_stopped()); @@ -414,6 +417,47 @@ fn only_the_exact_intact_container_not_running_is_a_stopped_container() { assert!(!reopened_from(&image).container_stopped()); } +#[test] +fn provider_inspects_and_reopens_exact_running_and_stopped_deployments() { + use layerfs_sandbox::backend::docker::DockerProvider; + use layerfs_sandbox::provider::SandboxRecord; + let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); + let running = inspect().replace( + r#""Running":true"#, + r#""Running":true,"Pid":42,"ExitCode":0"#, + ); + let stopped = exited().replace( + r#""Running":false"#, + r#""Running":false,"Pid":0,"ExitCode":7"#, + ); + let (docker, peer) = Peer::new( + vec![ + fixed(200, running.as_bytes()), + fixed(200, running.as_bytes()), + fixed(200, stopped.as_bytes()), + fixed(204, b""), + ], + 64, + ); + let provider = DockerProvider::new(docker); + let record = SandboxRecord::from(request()); + let mut owner = provider.reopen(id, &record).unwrap(); + let facts = owner.inspect().unwrap(); + assert!(facts.running); + assert_eq!(facts.process_id, Some(42)); + assert_eq!(facts.exit_code, None); + assert_eq!(facts.scratch_bytes, None); + let facts = owner.inspect().unwrap(); + assert!(!facts.running); + assert_eq!(facts.process_id, None); + assert_eq!(facts.exit_code, Some(7)); + // A read-only observation does not authorize Delete on the running owner. + assert!(!owner.delete().unwrap_err().attempted); + let mut stopped = provider.reopen_stopped(id, &record).unwrap(); + stopped.delete().unwrap(); + assert_eq!(peer.finish().len(), 4); +} + #[test] fn a_stopped_container_is_deleted_once_and_never_forced() { let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); diff --git a/engine/crates/layerfs-sandbox/tests/local_runtime.rs b/engine/crates/layerfs-sandbox/tests/local_runtime.rs new file mode 100644 index 0000000000..1db9a03b7c --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/local_runtime.rs @@ -0,0 +1,672 @@ +//! The native process runtime (`backend::native`): one daemon process per +//! Sandbox directory, observed by process id and start time, its commands run +//! as their own identity, and its stop and delete each attempted once. A +//! small shell script stands in for the daemon; nothing here needs root. +//! The runtime is Linux only, and so are these tests. +#![cfg(target_os = "linux")] +use layerfs_sandbox::{ + backend::{ + docker::ExecRequest, + native::{Local, LocalPhase, LocalProcess, Sandbox}, + }, + CommandIdentity, RuntimeError, +}; +use std::{ + fs, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; + +static NEXT: AtomicUsize = AtomicUsize::new(0); +const WAIT: Duration = Duration::from_secs(10); + +/// A fresh 0755 runtime root with a stand-in daemon `daemon.sh` beside it. +fn fixture(script: &str) -> (PathBuf, PathBuf) { + let base = std::env::temp_dir().join(format!( + "layerfs-local-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("root")).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + let daemon = base.join("daemon.sh"); + fs::write(&daemon, format!("#!/bin/sh\n{script}\n")).unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + (base.join("root"), daemon) +} + +/// Prints the listener line after checking its arguments, then waits. +const LISTENS: &str = r#"[ "$1" = --config ] && [ -f "$2" ] || exit 3 +env >&2 +echo "starting" +echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" +exec sleep 60"#; + +/// The command identity: this user, or nobody when the tests run as root. +fn me() -> CommandIdentity { + let meta = fs::metadata("/proc/self").unwrap(); + if meta.uid() == 0 { + CommandIdentity { + uid: 65534, + gid: 65534, + } + } else { + CommandIdentity { + uid: meta.uid(), + gid: meta.gid(), + } + } +} + +fn mode(path: &Path) -> u32 { + fs::symlink_metadata(path).unwrap().mode() & 0o7777 +} + +fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { + Local::new(root) + .unwrap() + .naming(name) + .unwrap() + .create( + daemon, + b"setup bytes", + WAIT, + vec![("layerfs.role".into(), "sandbox".into())], + ) + .unwrap() +} + +fn run(sandbox: &Sandbox, arguments: &[&str], environment: &[&str]) -> (i32, String, String) { + let request = ExecRequest { + identity: me(), + arguments: arguments.iter().map(|a| a.to_string()).collect(), + environment: environment.iter().map(|a| a.to_string()).collect(), + directory: sandbox.directory().display().to_string(), + stdin: false, + }; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + assert_eq!(exit.stdout, out.len() as u64); + assert_eq!(exit.stderr, err.len() as u64); + ( + exit.code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +#[test] +fn a_root_must_be_absolute_owned_private_to_write_and_traversable() { + let (root, _) = fixture("exit 0"); + assert!(matches!( + Local::new("relative/root"), + Err(RuntimeError::Protocol(_)) + )); + assert!(matches!( + Local::new(root.join("..").join("root")), + Err(RuntimeError::Protocol(_)) + )); + assert!(matches!( + Local::new(root.join("missing")), + Err(RuntimeError::Io(_)) + )); + fs::set_permissions(&root, fs::Permissions::from_mode(0o775)).unwrap(); + assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let local = Local::new(&root).unwrap(); + for bad in ["", ".hidden", "-x", "a/b", "..", "a b", &"x".repeat(129)] { + assert!(local.clone().naming(bad).is_err(), "{bad:?} admitted"); + } + assert!(local.paths().is_err(), "paths before a name"); + let paths = local.naming("layerfs-shop-1").unwrap().paths().unwrap(); + assert_eq!(paths.directory, root.join("layerfs-shop-1")); + assert_eq!(paths.config, paths.directory.join("config/daemon.setup")); + assert_eq!( + paths.overlay, + paths.directory.join("overlay/overlay.sqlite") + ); + assert_eq!(paths.mounts, paths.directory.join("workspaces")); + assert_eq!(paths.log, paths.directory.join("daemon.log")); + assert_eq!(paths.tmp, paths.directory.join("tmp")); +} + +#[test] +fn a_start_writes_a_private_setup_spawns_once_and_reads_the_listener() { + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-1"); + let directory = root.join("layerfs-shop-1"); + assert_eq!(sandbox.directory(), directory); + assert_eq!(sandbox.port(), 40404); + assert_eq!( + sandbox.endpoint().unwrap(), + "127.0.0.1:40404".parse().unwrap() + ); + assert_eq!( + sandbox.labels(), + [("layerfs.role".to_owned(), "sandbox".to_owned())] + ); + let process = sandbox.process().unwrap(); + assert!(process.alive()); + assert_eq!(LocalProcess::observe(process.pid).unwrap(), process); + assert!(!LocalProcess { + start: process.start + 1, + ..process + } + .alive()); + // private setup, private overlay home, traversable Sandbox directory + assert_eq!(mode(&directory), 0o755); + assert_eq!(mode(&directory.join("config")), 0o700); + assert_eq!(mode(&directory.join("config/daemon.setup")), 0o600); + assert_eq!( + fs::read(directory.join("config/daemon.setup")).unwrap(), + b"setup bytes" + ); + assert_eq!(mode(&directory.join("overlay")), 0o700); + assert_eq!(mode(&directory.join("daemon.log")), 0o600); + assert_eq!(mode(&directory.join("tmp")), 0o1777); + assert_eq!(sandbox.tmp(), directory.join("tmp")); + // the daemon inherits nothing of this process's environment but PATH + let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); + assert!(log.contains("PATH=/usr/local/sbin:"), "{log}"); + assert!(!log.contains("CARGO"), "{log}"); + assert_eq!( + String::from_utf8(sandbox.log_tail(1, 4096).unwrap()).unwrap(), + "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\n" + ); + assert_eq!(sandbox.log_tail(1, 5).unwrap(), b"0404\n"); + // the same name again is refused before anything is made + let again = Local::new(&root) + .unwrap() + .naming("layerfs-shop-1") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(again.phase, LocalPhase::Validate); + assert!(again.sandbox.is_none()); + // delete needs a known stop; stop ends the process; each is attempted once + assert!(sandbox.delete().is_err()); + let mut sandbox = Sandbox::reopen( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ) + .unwrap(); + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + assert!(sandbox.stop(1).is_err()); + assert!(sandbox.endpoint().is_err()); + sandbox.delete().unwrap(); + assert!(!directory.exists()); + assert!(sandbox.delete().is_err()); +} + +#[test] +fn a_command_runs_as_its_identity_in_its_directory_with_only_its_environment() { + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-2"); + let identity = me(); + let (code, out, err) = run( + &sandbox, + &[ + "bash", + "-c", + "id -u; id -g; pwd; echo \"$HOME|${CARGO:-none}\"; id -G >&2; echo oops >&2; exit 7", + ], + &["HOME=/tmp"], + ); + assert_eq!(code, 7); + assert_eq!( + out, + format!( + "{}\n{}\n{}\n/tmp|none\n", + identity.uid, + identity.gid, + sandbox.directory().display() + ) + ); + let (groups, err) = err.split_once('\n').unwrap(); + assert_eq!(err, "oops\n"); + // as root, the command keeps no supplementary group of this process + if fs::metadata("/proc/self").unwrap().uid() == 0 { + assert_eq!(groups, identity.gid.to_string()); + } + // a command a signal ended reports 128 + the signal, as the Engine does + let (code, _, _) = run(&sandbox, &["bash", "-c", "kill -9 $$"], &[]); + assert_eq!(code, 137); + // more output than one pipe holds, on both streams at once + let (code, out, err) = run( + &sandbox, + &[ + "bash", + "-c", + "head -c 300000 /dev/zero | tr '\\0' a; head -c 200000 /dev/zero | tr '\\0' b >&2", + ], + &[], + ); + assert_eq!((code, out.len(), err.len()), (0, 300000, 200000)); + // refused before any effect: root, stdin, a relative directory, a bad variable + let mut request = ExecRequest { + identity, + arguments: vec!["true".into()], + environment: Vec::new(), + directory: "/".into(), + stdin: false, + }; + for change in [ + (|r: &mut ExecRequest| r.identity = CommandIdentity { uid: 0, gid: 0 }) + as fn(&mut ExecRequest), + |r| r.stdin = true, + |r| r.directory = "relative".into(), + |r| r.environment = vec!["NOEQUALS".into()], + |r| r.arguments = Vec::new(), + ] { + let mut changed = request.clone(); + change(&mut changed); + let failure = sandbox + .run(&changed, &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(!failure.attempted, "{failure:?}"); + } + sandbox.stop(1).unwrap(); + // no command once the daemon is stopped + request.directory = "/".into(); + assert!(sandbox + .run(&request, &mut Vec::new(), &mut Vec::new()) + .is_err()); + sandbox.delete().unwrap(); +} + +#[test] +fn the_daemon_and_its_commands_start_with_a_containers_search_path() { + // on Linux, whatever this process's PATH is (macOS: tests/local_search_path.rs) + const CONTAINER: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-path"); + let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); + assert!( + log.lines().any(|line| line == format!("PATH={CONTAINER}")), + "{log}" + ); + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "printf %s \"$PATH\""], &[]); + assert_eq!((code, out.as_str()), (0, CONTAINER)); + // a command's own PATH still wins + let (code, out, _) = run( + &sandbox, + &["/bin/sh", "-c", "printf %s \"$PATH\""], + &["PATH=/opt/tools/bin:/usr/bin:/bin"], + ); + assert_eq!((code, out.as_str()), (0, "/opt/tools/bin:/usr/bin:/bin")); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); +} + +#[test] +fn a_daemon_that_ends_is_the_stopped_observation_and_only_its_delete_remains() { + let (root, daemon) = fixture(LISTENS); + let sandbox = started(&root, &daemon, "layerfs-shop-3"); + let process = sandbox.process().unwrap(); + let directory = sandbox.directory().to_owned(); + // killed from outside, as a crash or the OOM killer would + assert!(Command::new("kill") + .args(["-9", &process.pid.to_string()]) + .status() + .unwrap() + .success()); + let deadline = Instant::now() + WAIT; + let failure = loop { + match sandbox.endpoint() { + Err(failure) => break failure, + Ok(_) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(20)), + Ok(_) => panic!("killed daemon still observed running"), + } + }; + assert!(failure.container_stopped(), "{failure:?}"); + assert!(!failure.container_gone()); + // Stopped can be observed before the child is reapable. Keep observing + // through its owner so try_wait reaps it once all its threads have ended. + let deadline = Instant::now() + WAIT; + let stat = loop { + let failure = sandbox.endpoint().unwrap_err(); + assert!(failure.container_stopped(), "{failure:?}"); + assert!(!failure.container_gone()); + let stat = fs::read_to_string(format!("/proc/{}/stat", process.pid)); + if stat.as_ref().map_or(true, |stat| !stat.contains(") Z ")) || Instant::now() >= deadline { + break stat; + } + std::thread::sleep(Duration::from_millis(20)); + }; + // this process reaped its child: no zombie remains + assert!(stat.is_err() || !stat.unwrap().contains(") Z ")); + // a reopen from the record observes the same + let reopened = Sandbox::reopen( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ) + .unwrap_err(); + assert!(reopened.container_stopped()); + let mut stopped = Sandbox::stopped( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ); + assert!(stopped.endpoint().is_err()); + stopped.delete().unwrap(); + assert!(!directory.exists()); +} + +#[test] +fn a_failed_start_returns_what_it_made() { + // exits before its listener line + let (root, daemon) = fixture("echo broken >&2; exit 9"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-4") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + let mut sandbox = failure.sandbox.unwrap(); + assert_eq!(sandbox.log_tail(5, 100).unwrap(), b"broken\n"); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-4").exists()); + // never prints it: the wait ends + let (root, daemon) = fixture("exec sleep 60"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-5") + .unwrap() + .create(&daemon, b"x", Duration::from_millis(300), Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + assert!( + matches!(&failure.cause, RuntimeError::Io(e) if e.kind() == std::io::ErrorKind::TimedOut) + ); + let mut sandbox = failure.sandbox.unwrap(); + let process = sandbox.process().unwrap(); + assert!(process.alive()); + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + sandbox.delete().unwrap(); + // a listener that is not loopback is refused + let (root, daemon) = fixture("echo 'LAYERFS_DAEMON_LISTEN 0.0.0.0:5'; exec sleep 60"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-6") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + let mut sandbox = failure.sandbox.unwrap(); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + // a missing executable or an empty setup makes nothing + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-7") + .unwrap() + .create(&root.join("missing"), b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!( + (failure.phase, failure.sandbox.is_none()), + (LocalPhase::Validate, true) + ); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-7") + .unwrap() + .create(&daemon, b"", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!( + (failure.phase, failure.sandbox.is_none()), + (LocalPhase::Validate, true) + ); + assert!(!root.join("layerfs-shop-7").exists()); +} + +#[test] +fn a_zombie_is_not_alive() { + let mut child = Command::new("true").spawn().unwrap(); + let process = LocalProcess::observe(child.id()).unwrap(); + let deadline = Instant::now() + WAIT; + while process.alive() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process.alive(), "an exited, unreaped child counted alive"); + child.wait().unwrap(); +} + +#[test] +fn every_directory_above_a_root_must_be_traversable_by_others() { + let (root, _) = fixture("exit 0"); + let base = root.parent().unwrap().to_owned(); + // a root inside a directory others cannot pass, as `/root` (0700) is + let closed = base.join("closed"); + fs::create_dir(&closed).unwrap(); + fs::create_dir(closed.join("root")).unwrap(); + fs::set_permissions(closed.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(&closed, fs::Permissions::from_mode(0o700)).unwrap(); + let refused = |path: &Path| match Local::new(path) { + Err(RuntimeError::Io(error)) => { + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + error.to_string() + } + other => panic!("{} admitted: {other:?}", path.display()), + }; + let message = refused(&closed.join("root")); + assert!( + message.contains(&format!("{} is not traversable", closed.display())), + "{message}" + ); + // the same through a symlinked parent: the directory it resolves to counts + std::os::unix::fs::symlink(&closed, base.join("link")).unwrap(); + let message = refused(&base.join("link").join("root")); + assert!(message.contains("is not traversable"), "{message}"); + // traversable without being listable is enough + fs::set_permissions(&closed, fs::Permissions::from_mode(0o711)).unwrap(); + Local::new(closed.join("root")).unwrap(); + Local::new(base.join("link").join("root")).unwrap(); +} + +/// The process ids a command printed, one per line. +fn pids(out: &str) -> Vec { + out.lines() + .map(|line| line.trim().parse().unwrap()) + .collect() +} + +/// True once none of `pids` runs any more (a zombie has ended). +fn gone(pids: &[u32]) -> bool { + pids.iter().all( + |pid| match fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(stat) => matches!( + stat.rsplit_once(") ").map(|(_, rest)| &rest[..1]), + Some("Z" | "X") + ), + Err(_) => true, + }, + ) +} + +/// Leaves behind what an exec can: a `nohup` server writing into the +/// Sandbox's scratch directory, and a `setsid` one named with +/// [`Sandbox::track`] as the SDK names its background processes. +fn leave_running(sandbox: &Sandbox) -> Vec { + let home = format!("HOME={}", sandbox.tmp().display()); + let (code, out, err) = run( + sandbox, + &[ + "bash", + "-c", + "cd \"$HOME\"; nohup sh -c 'while :; do echo x >> \"$HOME/server.out\"; sleep 0.01; done' >/dev/null 2>&1 & echo $!", + ], + &[&home], + ); + assert_eq!(code, 0, "{err}"); + let mut left = pids(&out); + let (code, out, err) = run( + sandbox, + &[ + "bash", + "-c", + "setsid sleep 300 >/dev/null 2>&1 = 2, "{named}"); + let (code, _, _) = run(sandbox, &["true"], &[]); + assert_eq!(code, 0); + assert_eq!(groups(), named); + let deadline = Instant::now() + WAIT; + while !sandbox.tmp().join("server.out").exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!gone(&left)); + left +} + +#[test] +fn what_commands_leave_running_ends_with_the_sandbox() { + // stopped and deleted by its owner + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-8"); + let left = leave_running(&sandbox); + sandbox.stop(1).unwrap(); + assert!(gone(&left), "{left:?} outlived the stop"); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-8").exists()); + // its daemon killed from outside, then found stopped from its record + // and deleted, as a reclaim does + let sandbox = started(&root, &daemon, "layerfs-shop-9"); + let left = leave_running(&sandbox); + let process = sandbox.process().unwrap(); + assert!(Command::new("kill") + .args(["-9", &process.pid.to_string()]) + .status() + .unwrap() + .success()); + let deadline = Instant::now() + WAIT; + while sandbox.endpoint().is_ok() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!gone(&left), "a command ended with its daemon"); + let directory = sandbox.directory().to_owned(); + let mut stopped = Sandbox::stopped( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ); + stopped.delete().unwrap(); + assert!(gone(&left), "{left:?} outlived the delete"); + assert!(!directory.exists()); +} + +#[test] +fn a_process_of_another_boot_is_not_alive_and_one_without_a_boot_is_never_signalled() { + let (root, daemon) = fixture(LISTENS); + let sandbox = started(&root, &daemon, "layerfs-shop-10"); + let process = sandbox.process().unwrap(); + let boot = fs::read_to_string("/proc/sys/kernel/random/boot_id").unwrap(); + let boot = u128::from_str_radix(&boot.trim().replace('-', ""), 16).unwrap(); + assert_eq!(process.boot, Some(boot)); + // the same id and start time in another boot (a reboot or a restored + // snapshot) is another process + let other = LocalProcess { + boot: Some(boot ^ 1), + ..process + }; + assert!(!other.alive()); + assert!(Sandbox::reopen( + root.join("layerfs-shop-10"), + sandbox.identity(), + other, + 40404, + Vec::new() + ) + .unwrap_err() + .container_stopped()); + // a record from before boot ids is matched by id and start as then, and + // a stop through it signals nothing + let legacy = LocalProcess { + boot: None, + ..process + }; + assert!(legacy.alive()); + let mut reopened = Sandbox::reopen( + root.join("layerfs-shop-10"), + sandbox.identity(), + legacy, + 40404, + Vec::new(), + ) + .unwrap(); + assert!(reopened.stop(1).is_err()); + assert!(process.alive()); + let mut sandbox = sandbox; + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + sandbox.delete().unwrap(); +} + +/// A daemon whose main thread ends while another thread still runs: its +/// id is then a zombie that cannot be reaped yet, so it is not alive, but +/// its exit has not happened. A stop ends and reaps it before returning, +/// so it never stays a zombie of this process (the stand-in needs python3). +#[test] +fn a_stop_reaps_a_daemon_whose_main_thread_ended_first() { + let python = Command::new("python3").args(["-c", ""]).status(); + if !python.is_ok_and(|status| status.success()) { + eprintln!("skipped: the stand-in daemon needs python3"); + return; + } + let (root, daemon) = fixture( + r#"echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" +exec python3 -c 'import ctypes, threading, time; threading.Thread(target=time.sleep, args=(60,)).start(); ctypes.CDLL(None).pthread_exit(None)'"#, + ); + let mut sandbox = started(&root, &daemon, "layerfs-shop-11"); + let process = sandbox.process().unwrap(); + let deadline = Instant::now() + WAIT; + while process.alive() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process.alive(), "its main thread did not end"); + let tasks = fs::read_dir(format!("/proc/{}/task", process.pid)) + .unwrap() + .count(); + assert!(tasks >= 2, "its other thread ended already ({tasks})"); + sandbox.stop(1).unwrap(); + assert!( + fs::metadata(format!("/proc/{}", process.pid)).is_err(), + "the stopped daemon is still a process of this kernel (unreaped)" + ); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-11").exists()); +} diff --git a/engine/crates/layerfs-sandbox/tests/local_search_path.rs b/engine/crates/layerfs-sandbox/tests/local_search_path.rs new file mode 100644 index 0000000000..a6ff147a11 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/local_search_path.rs @@ -0,0 +1,113 @@ +//! A native (macOS) local Sandbox's daemon and commands start with this +//! process's own PATH, so this user's tools (`node` in `~/.local/bin`, +//! Homebrew's `/opt/homebrew/bin`) are found as in the caller's shell; with +//! no PATH, or an empty one, a container's search path and Homebrew's. +//! A small shell script stands in for the daemon. One test: it changes this +//! process's PATH, which no other test in this binary reads. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::{ + docker::ExecRequest, + native::{Local, Sandbox}, + }, + CommandIdentity, +}; +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const WAIT: Duration = Duration::from_secs(10); +const FALLBACK: &str = + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/homebrew/bin"; + +/// A fresh 0755 runtime root and a stand-in daemon that prints its +/// environment, then its listener line, then waits. +fn fixture() -> (PathBuf, PathBuf) { + let base = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("layerfs-search-path-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("root")).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + let daemon = base.join("daemon.sh"); + fs::write( + &daemon, + "#!/bin/sh\n[ \"$1\" = --config ] && [ -f \"$2\" ] || exit 3\n/usr/bin/env >&2\necho \"LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\"\nexec /bin/sleep 60\n", + ) + .unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + (base.join("root"), daemon) +} + +fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { + Local::native(root) + .unwrap() + .naming(name) + .unwrap() + .create(daemon, b"setup bytes", WAIT, Vec::new()) + .unwrap() +} + +/// The daemon's PATH line, from its log. +fn daemon_path(sandbox: &Sandbox) -> String { + let log = String::from_utf8(sandbox.log_tail(200, 16384).unwrap()).unwrap(); + log.lines() + .find_map(|line| line.strip_prefix("PATH=")) + .unwrap_or_else(|| panic!("no PATH line: {log}")) + .to_owned() +} + +/// A command's PATH: `printf` by its absolute path, so a lookup is not needed. +fn command_path(sandbox: &Sandbox, environment: &[&str]) -> String { + let request = ExecRequest { + identity: CommandIdentity { + uid: nix::unistd::Uid::effective().as_raw(), + gid: nix::unistd::Gid::effective().as_raw(), + }, + arguments: vec!["/bin/sh".into(), "-c".into(), "printf %s \"$PATH\"".into()], + environment: environment.iter().map(|e| e.to_string()).collect(), + directory: sandbox.directory().display().to_string(), + stdin: false, + }; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + assert_eq!(exit.code, 0, "{}", String::from_utf8_lossy(&err)); + String::from_utf8(out).unwrap() +} + +#[test] +fn a_native_daemon_and_its_commands_start_with_this_processs_path() { + let (root, daemon) = fixture(); + let own = "/Users/someone/.local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; + let saved = std::env::var_os("PATH"); + std::env::set_var("PATH", own); + let mut sandbox = started(&root, &daemon, "layerfs-path-1"); + assert_eq!(daemon_path(&sandbox), own); + assert_eq!(command_path(&sandbox, &[]), own); + // a command's own PATH still wins + assert_eq!( + command_path(&sandbox, &["PATH=/opt/x/bin:/bin"]), + "/opt/x/bin:/bin" + ); + // no PATH, then an empty one: a container's search path and Homebrew's + std::env::remove_var("PATH"); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + std::env::set_var("PATH", ""); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + let mut second = started(&root, &daemon, "layerfs-path-2"); + assert_eq!(daemon_path(&second), FALLBACK); + match saved { + Some(path) => std::env::set_var("PATH", path), + None => std::env::remove_var("PATH"), + } + for sandbox in [&mut sandbox, &mut second] { + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + } + let _ = fs::remove_dir_all(root.parent().unwrap()); +} diff --git a/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs b/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs new file mode 100644 index 0000000000..a1cb95089b --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs @@ -0,0 +1,48 @@ +//! Opt-in host proof against one caller-owned local Sandbox fixture. +//! Supply LAYERFS_TEST_STOP_DIRECTORY, _PID and _PORT. The fixture must be +//! disposable: this test stops its daemon and force-detaches its mounts. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::native::{LocalProcess, Sandbox}, + ContainerId, +}; +use std::{path::PathBuf, process::Command}; + +#[test] +#[ignore = "requires an explicitly supplied disposable mounted Sandbox"] +fn stop_detaches_mounts_before_returning_without_delete() { + let directory = PathBuf::from(std::env::var("LAYERFS_TEST_STOP_DIRECTORY").unwrap()); + let pid = std::env::var("LAYERFS_TEST_STOP_PID") + .unwrap() + .parse() + .unwrap(); + let port = std::env::var("LAYERFS_TEST_STOP_PORT") + .unwrap() + .parse() + .unwrap(); + assert!(directory.is_absolute()); + let mounts = || { + let out = Command::new("/sbin/mount").output().unwrap(); + assert!(out.status.success()); + String::from_utf8(out.stdout) + .unwrap() + .lines() + .filter(|line| line.contains(&format!(" on {}/", directory.display()))) + .map(str::to_owned) + .collect::>() + }; + assert!(!mounts().is_empty(), "fixture must start with a real mount"); + let process = LocalProcess::observe(pid).unwrap(); + let mut sandbox = Sandbox::reopen( + directory.clone(), + ContainerId::parse(&"1".repeat(64)).unwrap(), + process, + port, + Vec::new(), + ) + .unwrap(); + sandbox.stop(1).unwrap(); + assert!(!process.alive(), "fixture daemon survived Stop"); + assert!(directory.exists(), "Stop must not delete its directory"); + assert!(mounts().is_empty(), "Stop left mounts until Delete"); +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs new file mode 100644 index 0000000000..85987cbfe0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_lifecycle.rs @@ -0,0 +1,706 @@ +//! The namespace backend's Linux lifecycle and the #512 parity checks: a +//! Sandbox's daemon runs in its own mount, PID and IPC namespaces and its own +//! cgroup, on a root that is the Sandbox image with its own writable layer, +//! commands enter them as a nonroot identity with no capabilities and +//! no-new-privs, another Sandbox's processes, paths and writes are invisible, +//! and a close leaves no process, mount, writable layer or cgroup behind. +//! +//! It needs root, `/dev/fuse`, cgroup v2 and `user.max_user_namespaces=0`; it +//! starts a stub daemon (a shell script that prints the listener line), so no +//! Store or real daemon is involved. The image is this host's own userland in +//! a docker-archive, so the stub and the tools the checks run exist in it and +//! nowhere else in the Sandbox. Without those conditions it reports why and +//! returns, and the run is not proof. +#![cfg(target_os = "linux")] +#[allow(dead_code)] +mod support { + pub mod image; +} +use layerfs_sandbox::{ + backend::namespaces::{NamespaceProvider, Sandbox}, + provider::{LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ExecRequest, Image, RuntimeError, +}; +use std::{ + collections::BTreeSet, + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; +use support::image::{archive, hex, Tar, DIGEST}; + +/// The port the stub daemon claims; nothing listens on it. +const PORT: u16 = 40404; + +/// The programs the checks below run: each is copied into the image from this +/// host, with the libraries `ldd` names for it. +const PROGRAMS: [&str; 12] = [ + "/bin/sh", + "/bin/ls", + "/bin/cat", + "/bin/grep", + "/bin/test", + "/bin/echo", + "/bin/sleep", + "/bin/setsid", + "/bin/mkdir", + "/bin/mount", + "/bin/unshare", + "/bin/id", +]; + +/// One test's private directories: the runtime root, the Store's directory, +/// the stub daemon, a copy of the helper beside it, and the image archive. +struct Fixture { + base: PathBuf, + root: PathBuf, + store: PathBuf, + daemon: PathBuf, + archive: PathBuf, +} + +impl Fixture { + fn new(name: &str) -> Self { + let base = + std::env::temp_dir().join(format!("layerfs-namespaces-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let root = base.join("local"); + let store = base.join("store"); + for directory in [&root, &store] { + fs::create_dir(directory).unwrap(); + fs::set_permissions(directory, fs::Permissions::from_mode(0o755)).unwrap(); + } + fs::write(store.join("sealed.sqlite"), b"stub").unwrap(); + let daemon = base.join("stub-daemon"); + fs::write( + &daemon, + format!( + "#!/bin/sh\necho LAYERFS_DAEMON_LISTEN 127.0.0.1:{PORT}\nexec /bin/sleep 300\n" + ), + ) + .unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + let helper = helper(); + fs::copy(&helper, base.join("layerfs-sandbox-helper")).unwrap(); + let archive = image(&base); + Self { + base, + root, + store, + daemon, + archive, + } + } + + fn provider(&self) -> Result { + NamespaceProvider::new(&self.root) + } + + fn image(&self) -> Image { + Image::parse(DIGEST).unwrap() + } + + fn request(&self, name: &str) -> LaunchRequest { + LaunchRequest { + name: name.to_owned(), + daemon_executable: self.daemon.clone(), + daemon_config: b"stub setup".to_vec(), + labels: vec![("layerfs.role".to_owned(), "sandbox".to_owned())], + startup_timeout: Duration::from_secs(10), + options: ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())], + store: self.store.join("sealed.sqlite"), + image: self.image(), + archive: self.archive.clone(), + }, + } + } + + /// The Sandbox directory a launch names. + fn directory(&self, name: &str) -> PathBuf { + self.root.join(name) + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + if std::env::var_os("LAYERFS_H38_KEEP").is_some() { + eprintln!("kept {}", self.base.display()); + return; + } + let _ = fs::remove_dir_all(&self.base); + } +} + +/// The helper this build produced, beside the test binary's own directory. +fn helper() -> PathBuf { + let exe = std::env::current_exe().unwrap(); + let target = exe.parent().unwrap().parent().unwrap(); + let helper = target.join("layerfs-sandbox-helper"); + assert!( + helper.is_file(), + "build the helper first: cargo build -p layerfs-sandbox --bin layerfs-sandbox-helper" + ); + helper +} + +/// One docker-archive holding this host's own userland for the programs above, +/// plus a marker file a command can read and `/var/tmp`, the one directory a +/// command may write to. +fn image(base: &Path) -> PathBuf { + let mut tar = Tar::new() + .directory("etc/", 0o755) + .file("etc/layerfs-image", 0o644, b"layerfs image\n") + .directory("var/", 0o755) + .directory("var/tmp/", 0o1777); + let mut copied = BTreeSet::new(); + for program in PROGRAMS { + let Ok(source) = std::fs::canonicalize(program) else { + continue; + }; + tar = copy(tar, &mut copied, &source, Path::new(program)); + for library in libraries(&source) { + tar = copy(tar, &mut copied, &library, &library); + } + } + assert!( + copied.contains(Path::new("/bin/sh")) && copied.contains(Path::new("/bin/ls")), + "the host has no userland to build an image from" + ); + archive(base, "image", DIGEST, vec![tar.finish()]) +} + +/// The same bytes at `path` in the image, with the source's mode. +fn copy(mut tar: Tar, copied: &mut BTreeSet, source: &Path, path: &Path) -> Tar { + if !copied.insert(path.to_path_buf()) { + return tar; + } + let Ok(body) = fs::read(source) else { + return tar; + }; + let mode = fs::metadata(source).map_or(0o755, |metadata| metadata.permissions().mode() & 0o777); + // inside the image the path is relative: the tar carries `/bin/sh` as + // `bin/sh`, and the root's own `/` is the image itself + let text = path.to_str().expect("a UTF-8 host path"); + tar = tar.file(text.trim_start_matches('/'), mode, &body); + tar +} + +/// The libraries `ldd` names for `program`: every absolute path it prints. +fn libraries(program: &Path) -> Vec { + let Ok(output) = std::process::Command::new("ldd").arg(program).output() else { + return Vec::new(); + }; + let text = String::from_utf8_lossy(&output.stdout); + let mut libraries = Vec::new(); + for line in text.lines() { + if let Some(path) = (line.split_whitespace()).find(|token| token.starts_with('/')) { + libraries.push(PathBuf::from(path)); + } + } + libraries +} + +/// One command in `sandbox`, its streams captured. +fn run(sandbox: &Sandbox, arguments: &[&str], environment: &[&str]) -> (i32, String, String) { + let directory = sandbox.tmp().display().to_string(); + let request = ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: arguments.iter().map(|value| (*value).to_owned()).collect(), + environment: environment + .iter() + .map(|value| (*value).to_owned()) + .collect(), + directory: directory.clone(), + stdin: false, + }; + let mut out = Vec::new(); + let mut err = Vec::new(); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + let text = |bytes: Vec| String::from_utf8_lossy(&bytes).into_owned(); + (exit.code, text(out), text(err)) +} + +/// Why this host cannot run the lifecycle test, if it cannot. +fn unavailable() -> Option { + if !nix::unistd::Uid::effective().is_root() { + return Some("not root".into()); + } + match fs::read_to_string("/proc/sys/user/max_user_namespaces") { + Ok(value) if value.trim() == "0" => {} + Ok(value) => return Some(format!("user.max_user_namespaces is {}", value.trim())), + Err(error) => return Some(format!("user.max_user_namespaces: {error}")), + } + if !Path::new("/dev/fuse").exists() { + return Some("no /dev/fuse".into()); + } + if !Path::new("/sys/fs/cgroup/cgroup.controllers").exists() { + return Some("no cgroup v2".into()); + } + None +} + +#[test] +fn a_sandbox_is_its_own_namespaces_and_nothing_of_it_is_left_behind() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces lifecycle: {reason}"); + return; + } + let fixture = Fixture::new("lifecycle"); + let provider = fixture.provider().unwrap(); + let mut sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + assert_eq!(sandbox.port(), PORT); + assert!(sandbox.endpoint().is_ok()); + assert!(sandbox.process().unwrap().alive()); + assert!(matches!( + sandbox.record(), + Some(SandboxRecord::Namespaces(_)) + )); + let directory = fixture.directory("sandbox-1"); + assert!(directory.is_dir()); + // its root is the image, with this Sandbox's own empty writable layer + assert!(directory.join("overlay/upper").is_dir()); + assert_eq!( + fs::read_to_string( + fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + ) + .unwrap(), + "layerfs image\n" + ); + + // identity, capabilities and no-new-privs: a command is uid 501, gid 20, + // with no effective capability and no way to gain one + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "id -u; id -g"], &[]); + assert_eq!((code, out.as_str()), (0, "501\n20\n")); + let (code, out, _err) = run( + &sandbox, + &[ + "/bin/grep", + "-E", + "^(CapEff|NoNewPrivs|CapPrm):", + "/proc/self/status", + ], + &[], + ); + assert_eq!(code, 0); + assert!(out.contains("CapEff:\t0000000000000000"), "{out}"); + assert!(out.contains("CapPrm:\t0000000000000000"), "{out}"); + assert!(out.contains("NoNewPrivs:\t1"), "{out}"); + + // user namespaces are blocked, so a command cannot copy the mount + let (code, _, _) = run(&sandbox, &["/bin/sh", "-c", "unshare -U /bin/true"], &[]); + assert_ne!(code, 0, "unshare -U succeeded"); + // and without capabilities it cannot mount either + let (code, _, _) = run( + &sandbox, + &["/bin/sh", "-c", "mount -t tmpfs tmpfs /tmp/escape"], + &[], + ); + assert_ne!(code, 0, "a command mounted a filesystem"); + + // the root is the image: its own /proc, /dev/fuse, userland and marker + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "pwd"], &[]); + assert_eq!( + (code, out.trim_end().to_owned()), + (0, sandbox.tmp().display().to_string()) + ); + let (code, out, _) = run(&sandbox, &["/bin/ls", "/dev"], &[]); + assert_eq!(code, 0); + assert!(out.lines().any(|line| line == "fuse"), "{out}"); + let (code, out, _) = run(&sandbox, &["/bin/cat", "/proc/1/comm"], &[]); + assert_eq!(code, 0); + assert!(!out.trim().is_empty()); + let (code, out, _) = run(&sandbox, &["/bin/cat", "/etc/layerfs-image"], &[]); + assert_eq!((code, out.as_str()), (0, "layerfs image\n")); + // and it is read-only: a command cannot write the image, only its own + // writable layer above it + let (code, _, _) = run( + &sandbox, + &["/bin/sh", "-c", "echo changed > /etc/layerfs-image"], + &[], + ); + assert_ne!(code, 0, "a command wrote the image"); + let (code, _, _) = run(&sandbox, &["/bin/test", "-w", "/etc"], &[]); + assert_ne!(code, 0, "a command may write the image's directory"); + let (code, _, _) = run(&sandbox, &["/bin/test", "-w", "/var/tmp"], &[]); + assert_eq!(code, 0, "a command may not write its own writable layer"); + let (code, _, _) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "/bin/mkdir /var/tmp/written && echo x > /var/tmp/written/x", + ], + &[], + ); + assert_eq!(code, 0, "a command could not write its own layer"); + + // streams, environment and exit status + let (code, out, err) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "printf out; printf err >&2; printf %s \"$MARK\"; exit 7", + ], + &["MARK=value"], + ); + assert_eq!((code, out.as_str(), err.as_str()), (7, "outvalue", "err")); + + // the Store's directory is bound, the VM's own tree is not + let (code, _, _) = run( + &sandbox, + &["/bin/test", "-d", fixture.store.to_str().unwrap()], + &[], + ); + assert_eq!(code, 0); + let (code, _, _) = run(&sandbox, &["/bin/sh", "-c", "test -e /var/lib"], &[]); + assert_ne!(code, 0, "the VM's own tree is visible"); + + // close: one write ends every process, and nothing is left behind + sandbox.stop(5).unwrap(); + sandbox.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); + assert!(!fixture.root.join("sandbox-1.root").exists()); + assert!(!Path::new("/sys/fs/cgroup/layerfs") + .join(sandbox.identity().as_str()) + .exists()); + let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap(); + assert!( + !mounts.contains("sandbox-1"), + "a mount of the Sandbox remains" + ); + // the image stays: it is the cache every other Sandbox of it reuses + assert!(fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + .is_file()); +} + +#[test] +fn two_sandboxes_on_one_store_cannot_see_each_other() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces isolation: {reason}"); + return; + } + let fixture = Fixture::new("isolation"); + let provider = fixture.provider().unwrap(); + let first = Arc::new(provider.launch(fixture.request("sandbox-1")).unwrap()); + let second = Arc::new(provider.launch(fixture.request("sandbox-2")).unwrap()); + let other = fixture.directory("sandbox-2").display().to_string(); + + // the other Sandbox's directory is not in this mount namespace + let (code, _, _) = run(&first, &["/bin/test", "-e", &other], &[]); + assert_ne!(code, 0, "the other Sandbox's directory is visible"); + // and none of its processes is in this PID namespace: the other Sandbox's + // host PID does not exist here, and neither PID 1 nor PID 2 names it + let second_pid = second.process().unwrap().pid; + let (code, _, _) = run( + &first, + &["/bin/test", "-e", &format!("/proc/{second_pid}")], + &[], + ); + assert_ne!( + code, 0, + "the other Sandbox's PID is visible in this PID namespace" + ); + let (code, out, _) = run( + &first, + &[ + "/bin/sh", + "-c", + "grep -F -- \"$OTHER\" /proc/1/cmdline /proc/2/cmdline 2>/dev/null || true", + ], + &[&format!("OTHER={other}")], + ); + assert_eq!(code, 0); + assert!( + out.is_empty(), + "the other Sandbox's process is visible: {out}" + ); + // each cgroup holds only its own Sandbox + let (code, out, _) = run(&first, &["/bin/cat", "/proc/self/cgroup"], &[]); + assert_eq!(code, 0); + assert!(out.contains(first.identity().as_str()), "{out}"); + + // one image, two writable layers: what one Sandbox writes the other never + // sees, and each keeps its own + let (code, _, _) = run( + &first, + &["/bin/sh", "-c", "echo first > /var/tmp/mine"], + &[], + ); + assert_eq!(code, 0); + let (code, _, _) = run( + &second, + &["/bin/sh", "-c", "echo second > /var/tmp/mine"], + &[], + ); + assert_eq!(code, 0); + let (code, out, _) = run(&first, &["/bin/cat", "/var/tmp/mine"], &[]); + assert_eq!((code, out.as_str()), (0, "first\n")); + let (code, out, _) = run(&second, &["/bin/cat", "/var/tmp/mine"], &[]); + assert_eq!((code, out.as_str()), (0, "second\n")); + let (code, _, _) = run(&first, &["/bin/test", "-e", "/var/tmp/only-second"], &[]); + assert_ne!(code, 0); + // both still see the image itself, unchanged + let (code, out, _) = run(&first, &["/bin/cat", "/etc/layerfs-image"], &[]); + assert_eq!((code, out.as_str()), (0, "layerfs image\n")); + // and the writes live in each Sandbox's own upper layer, never in the + // shared image + let image_root = fixture.root.join("images").join(hex(DIGEST)); + assert!( + fs::read_to_string(image_root.join("var/tmp/mine")).is_err(), + "a Sandbox's write reached the shared image" + ); + assert_eq!( + fs::read_to_string( + fixture + .directory("sandbox-1") + .join("overlay/upper/var/tmp/mine") + ) + .unwrap(), + "first\n" + ); + assert_eq!( + fs::read_to_string( + fixture + .directory("sandbox-2") + .join("overlay/upper/var/tmp/mine") + ) + .unwrap(), + "second\n" + ); + + // both run at the same time, neither disturbed by the other + let handles = [first.clone(), second.clone()]; + let codes = std::thread::scope(|scope| { + let started = handles + .iter() + .map(|sandbox| { + let sandbox = sandbox.clone(); + scope.spawn(move || { + let (code, out, _) = run( + &sandbox, + &["/bin/sh", "-c", "/bin/sleep 0.2; printf done"], + &[], + ); + (code, out) + }) + }) + .collect::>(); + started + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + assert_eq!(codes, vec![(0, "done".to_owned()), (0, "done".to_owned())]); + drop(handles); + + for (mut sandbox, name) in [ + (Arc::try_unwrap(first).ok().unwrap(), "sandbox-1"), + (Arc::try_unwrap(second).ok().unwrap(), "sandbox-2"), + ] { + sandbox.stop(5).unwrap(); + sandbox.delete().unwrap(); + assert!(!fixture.directory(name).exists()); + } +} + +#[test] +fn a_record_reopens_the_exact_sandbox_and_nothing_else() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces reopen: {reason}"); + return; + } + let fixture = Fixture::new("reopen"); + let provider = fixture.provider().unwrap(); + let sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + let id = sandbox.identity(); + let record = sandbox.record().unwrap(); + // the record names the image its root is an overlay of + assert_eq!( + record.namespaces().map(|record| record.image().as_str()), + Some(DIGEST) + ); + assert_eq!( + record + .namespaces() + .map(|record| record.local().name.as_str()), + Some("sandbox-1") + ); + // what a new process reads: the record alone, with the provider's root, + // and it finds the same image under the digest the record keeps + let reopened_provider = fixture.provider().unwrap(); + let mut reopened = reopened_provider.reopen(id, &record).unwrap(); + assert_eq!(reopened.identity(), id); + assert_eq!(reopened.port(), PORT); + assert_eq!(reopened.record().unwrap(), record); + assert_eq!(reopened.image().image().as_str(), DIGEST); + let (code, out, _) = run(&reopened, &["/bin/echo", "again"], &[]); + assert_eq!((code, out.as_str()), (0, "again\n")); + assert!(!reopened.log_tail(10, 4096).unwrap().is_empty()); + + // a record whose image is gone is refused, not given another root + let mut without_image = record.clone(); + if let SandboxRecord::Namespaces(record) = &mut without_image { + record.image = Image::parse(&format!("sha256:{}", "e".repeat(64))).unwrap(); + } + let refused = reopened_provider + .reopen(id, &without_image) + .expect_err("a record without its image was reopened"); + assert!( + format!("{:?}", refused.cause).contains("cannot be found"), + "{:?}", + refused.cause + ); + + // a Docker record is refused rather than reinterpreting it + let foreign = SandboxRecord::Docker(layerfs_sandbox::provider::DockerRecord { + image: "sha256:0".to_owned(), + store_volume: "volume".to_owned(), + port: 30421, + labels: Vec::new(), + }); + assert!(reopened_provider.reopen(id, &foreign).is_err()); + + // a stopped Sandbox admits only its one Delete + reopened.stop(5).unwrap(); + drop(sandbox); + let stopped = reopened_provider.reopen_stopped(id, &record).unwrap(); + assert!(stopped.endpoint().is_err()); + let mut stopped = stopped; + stopped.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); +} + +#[test] +fn a_request_is_refused_before_any_effect() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces refusals: {reason}"); + return; + } + let fixture = Fixture::new("refusals"); + let provider = fixture.provider().unwrap(); + // a root command identity + let mut request = fixture.request("sandbox-1"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 0, gid: 0 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(matches!( + failure.cause, + RuntimeError::Protocol("nonroot command uid/gid") + )); + assert!(failure.instance.is_none()); + assert!(!fixture.directory("sandbox-1").exists()); + // the other runtime's options + let mut request = fixture.request("sandbox-2"); + request.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none()); + // a Store directory that would expose every Sandbox + let mut request = fixture.request("sandbox-3"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.root.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(matches!(failure.cause, RuntimeError::Protocol(_))); + assert!(!fixture.directory("sandbox-3").exists()); + // a bundle without the image archive + let mut request = fixture.request("sandbox-4"); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: fixture.image(), + archive: fixture.base.join("no-such-image.tar"), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none(), "{:?}", failure.cause); + assert!(format!("{:?}", failure.cause).contains("no-such-image.tar")); + assert!(!fixture.directory("sandbox-4").exists()); + // an image archive that holds another image than the deployment named + let mut request = fixture.request("sandbox-5"); + let other = Image::parse(&format!("sha256:{}", "d".repeat(64))).unwrap(); + request.options = ProviderLaunchOptions::Namespaces { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: Vec::new(), + store: fixture.store.join("sealed.sqlite"), + image: other.clone(), + archive: fixture.archive.clone(), + }; + let failure = provider.launch(request).unwrap_err(); + assert!(failure.instance.is_none(), "{:?}", failure.cause); + let message = format!("{:?}", failure.cause); + assert!(message.contains(other.hex()), "{message}"); + assert!(message.contains(hex(DIGEST)), "{message}"); + assert!(!fixture.directory("sandbox-5").exists()); +} + +#[test] +fn a_running_command_ends_with_its_sandbox() { + if let Some(reason) = unavailable() { + eprintln!("SKIPPED namespaces teardown: {reason}"); + return; + } + let fixture = Fixture::new("teardown"); + let provider = fixture.provider().unwrap(); + let mut sandbox = provider.launch(fixture.request("sandbox-1")).unwrap(); + // a command that outlives its own process group and keeps running + let (code, out, _) = run( + &sandbox, + &[ + "/bin/sh", + "-c", + "setsid /bin/sleep 300 >/dev/null 2>&1 & echo started", + ], + &[], + ); + assert_eq!((code, out.as_str()), (0, "started\n")); + sandbox.stop(5).unwrap(); + // cgroup.kill ended it: nothing of the Sandbox is left in its group + let events = fs::read_to_string( + Path::new("/sys/fs/cgroup/layerfs") + .join(sandbox.identity().as_str()) + .join("cgroup.events"), + ) + .unwrap(); + assert!(events.contains("populated 0"), "{events}"); + // closing it removes the writable layer and the directory, and the shared + // image stays for the next Sandbox of it + assert!(fixture + .directory("sandbox-1") + .join("overlay/upper") + .is_dir()); + sandbox.delete().unwrap(); + assert!(!fixture.directory("sandbox-1").exists()); + assert!(fixture + .root + .join("images") + .join(hex(DIGEST)) + .join("etc/layerfs-image") + .is_file()); +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs b/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs new file mode 100644 index 0000000000..6a1af22a34 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_protocol.rs @@ -0,0 +1,145 @@ +//! The namespace backend's formats and records: the init stage's spec, the +//! recorded facts a reopen reads, and the refusals that keep one runtime's +//! records from another. Nothing here starts a process. +use layerfs_sandbox::{ + backend::HelperSpec, + provider::{DockerRecord, LocalRecord, NamespaceRecord, SandboxRecord}, + Image, +}; +use std::path::PathBuf; + +/// The image digest these records name. +fn image() -> Image { + Image::parse(&format!("sha256:{}", "a".repeat(64))).unwrap() +} + +fn spec() -> HelperSpec { + HelperSpec { + root: PathBuf::from("/var/lib/layerfs/local/sandbox-1.root"), + sandbox: PathBuf::from("/var/lib/layerfs/local/sandbox-1"), + store: PathBuf::from("/layerfs-store/project"), + daemon: PathBuf::from("/var/lib/layerfs/local/sandbox-1/bin/layerfs-daemon"), + config: PathBuf::from("/var/lib/layerfs/local/sandbox-1/config/daemon.setup"), + cgroup: PathBuf::from("/sys/fs/cgroup/layerfs/0000000000000000000000000000000000000000000000000000000000000001"), + lower: PathBuf::from( + "/var/lib/layerfs/local/images/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), + upper: PathBuf::from("/var/lib/layerfs/local/sandbox-1/overlay/upper"), + work: PathBuf::from("/var/lib/layerfs/local/sandbox-1/overlay/work"), + } +} + +#[test] +fn the_init_spec_roundtrips() { + let spec = spec(); + let encoded = spec.encode().unwrap(); + assert_eq!(HelperSpec::parse(&encoded).unwrap(), spec); + // one line per field, key first + let text = String::from_utf8(encoded).unwrap(); + assert_eq!(text.lines().count(), 9); + assert!(text.starts_with("root=/var/lib/layerfs/local/sandbox-1.root\n")); +} + +#[test] +fn a_value_may_hold_an_equals_sign() { + let mut spec = spec(); + spec.store = PathBuf::from("/layerfs-store/a=b"); + let encoded = spec.encode().unwrap(); + assert_eq!(HelperSpec::parse(&encoded).unwrap().store, spec.store); +} + +#[test] +fn a_path_the_init_stage_cannot_mount_is_refused() { + let relative = HelperSpec { + root: PathBuf::from("relative/root"), + ..spec() + }; + assert!(relative.encode().is_err()); + let parent = HelperSpec { + sandbox: PathBuf::from("/var/lib/../sandbox"), + ..spec() + }; + assert!(parent.encode().is_err()); + let newline = HelperSpec { + config: PathBuf::from("/var/lib/a\nb"), + ..spec() + }; + assert!(newline.encode().is_err()); + let bytes = newline.encode().unwrap_err(); + assert_eq!(format!("{bytes:?}"), r#"Protocol("helper spec path")"#); + // the three overlay paths are one mount option string: neither a comma nor + // a colon may appear in them + let comma = HelperSpec { + upper: PathBuf::from("/var/lib/a,b/upper"), + ..spec() + }; + assert!(comma.encode().is_err()); + let colon = HelperSpec { + lower: PathBuf::from("/var/lib/a:b"), + ..spec() + }; + assert_eq!( + format!("{:?}", colon.encode().unwrap_err()), + r#"Protocol("an overlay path holds a `,` or `:`")"# + ); +} + +#[test] +fn a_spec_is_refused_unless_every_field_appears_once() { + let encoded = spec().encode().unwrap(); + let text = String::from_utf8(encoded).unwrap(); + // one field missing + let missing = text + .lines() + .filter(|line| !line.starts_with("cgroup=")) + .collect::>(); + assert!(HelperSpec::parse(missing.join("\n").as_bytes()).is_err()); + // one field twice + let twice = format!("{text}cgroup=/sys/fs/cgroup/layerfs/2\n"); + assert!(HelperSpec::parse(twice.as_bytes()).is_err()); + // an unknown field + let unknown = format!("{text}kernel=nfs\n"); + assert!(HelperSpec::parse(unknown.as_bytes()).is_err()); + // a line without a key + assert!(HelperSpec::parse(format!("{text}/\n").as_bytes()).is_err()); + // the whole file is bounded + let huge = vec![b'a'; 8193]; + assert!(HelperSpec::parse(&huge).is_err()); +} + +#[test] +fn a_namespaces_record_is_its_own_variant_and_keeps_the_local_facts() { + let local = LocalRecord { + name: "layerfs-project-1".to_owned(), + pid: 4242, + start_time: 7, + boot: Some(1), + port: 41000, + labels: vec![("layerfs.role".to_owned(), "sandbox".to_owned())], + }; + let namespace = NamespaceRecord { + local: local.clone(), + image: image(), + }; + let record = SandboxRecord::Namespaces(namespace.clone()); + assert_eq!(record.namespaces(), Some(&namespace)); + // the image digest the record keeps is the one its root is an overlay of + assert_eq!( + record.namespaces().map(|record| record.image().as_str()), + Some(image().as_str()) + ); + assert!(record.local().is_none()); + assert!(record.docker().is_none()); + // the native record of the same facts stays a native record + let native = SandboxRecord::Local(local); + assert!(native.namespaces().is_none()); + // a Docker record written before this backend existed still reads as Docker + let docker = SandboxRecord::Docker(DockerRecord { + image: "sha256:0".to_owned(), + store_volume: "volume".to_owned(), + port: 30421, + labels: Vec::new(), + }); + assert!(docker.namespaces().is_none()); + assert!(docker.local().is_none()); +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs b/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs new file mode 100644 index 0000000000..556fc04226 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_rootfs.rs @@ -0,0 +1,340 @@ +//! The Sandbox image: one docker-archive in the bundle, unpacked once per +//! config digest. These tests need no root, no Engine and no namespaces: they +//! unpack archives into a scratch directory and read what a command would see. +//! What they prove is the contract the lifecycle tests build on — the digest +//! check, the archive's own whiteouts, one unpack per digest, and the entries +//! a Sandbox root refuses. +#![cfg(target_os = "linux")] +#[allow(dead_code)] +mod support { + pub mod image; +} +use layerfs_sandbox::{ + backend::namespaces::rootfs::{self, ImageRoot}, + Image, +}; +use std::{ + fs, io, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; +use support::image::{archive, hex, legacy_archive, Scratch, Tar, DIGEST, OTHER}; + +/// Two layers: the first holds the image, the second hides part of it and adds +/// to it, which is what a whiteout and an opaque directory mean. +fn two_layers() -> Vec> { + let first = Tar::new() + .directory("etc/", 0o755) + .file("etc/marker", 0o644, b"layer one\n") + .file("etc/gone", 0o644, b"replaced\n") + .file("etc/kept", 0o644, b"kept\n") + .directory("etc/opaque/", 0o755) + .file("etc/opaque/old", 0o644, b"below\n") + .finish(); + let second = Tar::new() + // `.wh.` hides the entry the layer below placed at the same level + .file("etc/.wh.gone", 0o644, b"") + // `.wh..wh..opq` empties its directory, then this layer refills it + .file("etc/opaque/.wh..wh..opq", 0o644, b"") + .file("etc/opaque/new", 0o644, b"above\n") + // and an ordinary file the first layer never had + .file("etc/added", 0o644, b"added\n") + .finish(); + vec![first, second] +} + +fn image(digest: &str) -> Image { + Image::parse(digest).unwrap() +} + +/// The unpacked root of a digest, which must not exist yet. +fn unpacked(scratch: &Scratch, archive: &Path, digest: &str) -> ImageRoot { + rootfs::ensure(archive, &image(digest), &scratch.cache).unwrap() +} + +fn read(path: &Path) -> String { + fs::read_to_string(path).unwrap_or_else(|error| panic!("{}: {error}", path.display())) +} + +#[test] +fn the_layers_unpack_in_order_and_their_whiteouts_apply() { + let scratch = Scratch::new("layers"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(root.image().as_str(), DIGEST); + assert_eq!(root.directory(), scratch.cache.join(hex(DIGEST)).as_path()); + let directory = root.directory(); + // the first layer's files, and the second layer's + assert_eq!(read(&directory.join("etc/marker")), "layer one\n"); + assert_eq!(read(&directory.join("etc/kept")), "kept\n"); + assert_eq!(read(&directory.join("etc/added")), "added\n"); + // `.wh.gone` hid what the layer below put there + assert!(!directory.join("etc/gone").exists()); + // `.wh..wh..opq` emptied the directory: only the second layer's entry is left + assert_eq!(read(&directory.join("etc/opaque/new")), "above\n"); + assert!(!directory.join("etc/opaque/old").exists()); + // the cache directory is not writable by the command identity + let mode = fs::symlink_metadata(directory) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o555, "the image cache is writable"); +} + +#[test] +fn a_whiteout_with_nothing_below_it_creates_nothing() { + let scratch = Scratch::new("whiteout"); + let layers = vec![ + Tar::new().file("etc/kept", 0o644, b"kept\n").finish(), + // a whiteout under a directory no layer made: it hides nothing and + // must not become the empty directory it names + Tar::new() + .file("etc/.wh.hidden", 0o644, b"") + .file("missing/.wh..wh..opq", 0o644, b"") + .finish(), + ]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(read(&root.directory().join("etc/kept")), "kept\n"); + assert!(!root.directory().join("etc/hidden").exists()); + assert!(!root.directory().join("missing").exists()); +} + +#[test] +fn a_digest_the_archive_is_not_refuses_the_start_and_unpacks_nothing() { + let scratch = Scratch::new("digest"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let error = rootfs::ensure(&path, &image(OTHER), &scratch.cache).unwrap_err(); + let message = format!("{error}"); + assert!(message.contains(hex(DIGEST)), "{message}"); + assert!(message.contains(hex(OTHER)), "{message}"); + assert!(message.contains("config digest"), "{message}"); + // nothing was published, and no half unpack was left behind + assert!(!scratch.cache.join(hex(OTHER)).exists()); + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn an_older_writers_config_name_is_the_same_digest() { + let scratch = Scratch::new("legacy"); + let path = legacy_archive(&scratch.base, "image", DIGEST, two_layers()); + let root = unpacked(&scratch, &path, DIGEST); + assert_eq!(read(&root.directory().join("etc/marker")), "layer one\n"); +} + +#[test] +fn one_unpack_per_digest_however_many_sandboxes_ask_for_it() { + let scratch = Scratch::new("once"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + let first = unpacked(&scratch, &path, DIGEST); + let published = fs::metadata(first.directory()).unwrap(); + // a second Sandbox of the same image finds the finished directory: the + // same inode, and no new unpack + let second = unpacked(&scratch, &path, DIGEST); + assert_eq!(second.directory(), first.directory()); + let observed = fs::metadata(second.directory()).unwrap(); + assert_eq!( + (observed.ino(), observed.mtime()), + (published.ino(), published.mtime()), + "the image was unpacked again" + ); + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn a_second_sandbox_waits_for_the_unpack_that_is_running() { + let scratch = Scratch::new("lock"); + let path = archive(&scratch.base, "image", DIGEST, two_layers()); + // the first unpack publishes the cache and its lock + let published = unpacked(&scratch, &path, DIGEST); + let directory = published.directory().to_owned(); + drop(published); + fs::remove_dir_all(&directory).unwrap(); + // holding the digest's lock is what an unpack in progress looks like: a + // second one neither starts nor publishes anything until it is released + let held = nix::fcntl::Flock::lock( + fs::OpenOptions::new() + .read(true) + .write(true) + .open(scratch.cache.join(format!("{}.lock", hex(DIGEST)))) + .unwrap(), + nix::fcntl::FlockArg::LockExclusive, + ) + .unwrap(); + let waiting = { + let (archive, cache) = (path.clone(), scratch.cache.clone()); + std::thread::spawn(move || { + rootfs::ensure(&archive, &image(DIGEST), &cache).map(|root| root.directory().to_owned()) + }) + }; + let deadline = Instant::now() + Duration::from_millis(300); + while Instant::now() < deadline { + assert!( + !directory.exists(), + "a second unpack published while the first held the lock" + ); + std::thread::sleep(Duration::from_millis(20)); + } + drop(held); + let waited = waiting.join().expect("the waiting unpack").unwrap(); + assert_eq!(waited, directory); + assert_eq!(read(&directory.join("etc/added")), "added\n"); +} + +#[test] +fn a_path_that_is_not_plain_and_relative_is_refused() { + let scratch = Scratch::new("escape"); + for (name, layer) in [ + ( + "parent", + Tar::new().file("../outside", 0o644, b"x").finish(), + ), + ( + "absolute", + Tar::new().file("/etc/outside", 0o644, b"x").finish(), + ), + ( + "nested", + Tar::new() + .directory("a/", 0o755) + .file("a/../../outside", 0o644, b"x") + .finish(), + ), + ] { + let path = archive(&scratch.base, name, DIGEST, vec![layer]); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("absolute or holds a `..` component"), + "{name}: {error}" + ); + assert!(!scratch.cache.join(hex(DIGEST)).exists(), "{name}"); + assert!( + !scratch.base.join("outside").exists(), + "{name} wrote outside" + ); + assert!(!Path::new("/etc/outside").exists(), "{name} wrote outside"); + } + assert_eq!(temporaries(&scratch.cache).len(), 0); +} + +#[test] +fn a_symlink_cannot_redirect_a_later_layers_write() { + let scratch = Scratch::new("symlink"); + let layers = vec![ + Tar::new() + .directory("etc/", 0o755) + .symlink("etc/escape", "/tmp") + .finish(), + // the parent of this entry is a symbolic link, so writing it would + // leave the root entirely + Tar::new().file("etc/escape/outside", 0o644, b"x").finish(), + ]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("passes through a symbolic link"), + "{error}" + ); + assert!(!Path::new("/tmp/outside").exists(), "the link was followed"); + assert!(!scratch.cache.join(hex(DIGEST)).exists()); +} + +#[test] +fn a_device_node_is_refused() { + let scratch = Scratch::new("device"); + let layers = vec![Tar::new() + .directory("dev/", 0o755) + .raw("dev/sda", b'3', 0o660, &[]) + .finish()]; + let path = archive(&scratch.base, "image", DIGEST, layers); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!( + format!("{error}").contains("device node"), + "a device node was extracted: {error}" + ); + assert!(!scratch.cache.join(hex(DIGEST)).exists()); +} + +#[test] +fn a_compressed_layer_is_refused_with_the_reason() { + let scratch = Scratch::new("gzip"); + let mut gzip = vec![0x1f, 0x8b]; + gzip.extend_from_slice(&[0u8; 64]); + let path = archive(&scratch.base, "image", DIGEST, vec![gzip]); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + let message = format!("{error}"); + assert!(message.contains("gzip-compressed"), "{message}"); + assert!(message.contains("docker save"), "{message}"); +} + +#[test] +fn an_archive_without_its_manifest_or_a_layer_is_refused() { + let scratch = Scratch::new("missing"); + let without_manifest = Tar::new() + .file("1/layer.tar", 0o644, &Tar::new().finish()) + .finish(); + let path = scratch.base.join("no-manifest.tar"); + fs::write(&path, without_manifest).unwrap(); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("manifest.json"), "{error}"); + // a manifest naming a layer the archive does not hold + let mut tar = Tar::new().file( + "manifest.json", + 0o644, + format!( + "[{{\"Config\":\"blobs/sha256/{}\",\"Layers\":[\"7/layer.tar\"]}}]", + hex(DIGEST) + ) + .as_bytes(), + ); + tar = tar.file(&format!("blobs/sha256/{}", hex(DIGEST)), 0o644, b"{}"); + let path = scratch.base.join("no-layer.tar"); + fs::write(&path, tar.finish()).unwrap(); + let error = rootfs::ensure(&path, &image(DIGEST), &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("7/layer.tar"), "{error}"); +} + +#[test] +fn two_digests_share_one_cache_directory() { + let scratch = Scratch::new("digests"); + let first = archive(&scratch.base, "first", DIGEST, two_layers()); + let second = archive( + &scratch.base, + "second", + OTHER, + vec![Tar::new().file("other", 0o644, b"other\n").finish()], + ); + let one = unpacked(&scratch, &first, DIGEST); + let two = unpacked(&scratch, &second, OTHER); + assert_ne!(one.directory(), two.directory()); + assert_eq!(read(&one.directory().join("etc/marker")), "layer one\n"); + assert_eq!(read(&two.directory().join("other")), "other\n"); + // a record's image is found without unpacking anything again + assert_eq!( + rootfs::lookup(&image(OTHER), &scratch.cache) + .unwrap() + .directory(), + two.directory() + ); + // and one that is not there is refused rather than invented + let missing = Image::parse(&format!("sha256:{}", "d".repeat(64))).unwrap(); + let error = rootfs::lookup(&missing, &scratch.cache).unwrap_err(); + assert!(format!("{error}").contains("cannot be found"), "{error}"); +} + +/// The temporary directories an unpack in progress leaves, if any. +fn temporaries(cache: &Path) -> Vec { + match fs::read_dir(cache) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Vec::new(), + Err(error) => panic!("{}: {error}", cache.display()), + Ok(entries) => (entries.filter_map(Result::ok)) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(".tmp-")) + }) + .collect(), + } +} diff --git a/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs b/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs new file mode 100644 index 0000000000..fcfa2b0347 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/namespaces_userns.rs @@ -0,0 +1,59 @@ +//! The user-namespace block is a precondition, not a setting: the namespace +//! runtime refuses to start while a command could create a user namespace and +//! hold its own copy of the Workspace mount. +//! +//! This test opens the host limit for its own duration and restores it, so it +//! runs in its own binary (no Sandbox runs beside it) and only as root in the +//! VM or CI host that owns this kernel. +#![cfg(target_os = "linux")] +use layerfs_sandbox::{backend::namespaces::NamespaceProvider, RuntimeError}; +use std::{fs, os::unix::fs::PermissionsExt, path::Path, path::PathBuf}; + +/// The kernel's limit on user namespaces. +const LIMIT: &str = "/proc/sys/user/max_user_namespaces"; + +/// Restores the limit however the test ends. +struct Restore(String); + +impl Drop for Restore { + fn drop(&mut self) { + let _ = fs::write(LIMIT, &self.0); + } +} + +fn root(name: &str) -> PathBuf { + let root = + std::env::temp_dir().join(format!("layerfs-namespaces-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + root +} + +#[test] +fn an_open_user_namespace_limit_is_refused_before_any_effect() { + if !nix::unistd::Uid::effective().is_root() { + eprintln!("SKIPPED namespaces user-namespace refusal: not root"); + return; + } + let previous = fs::read_to_string(LIMIT).unwrap().trim().to_owned(); + let restore = Restore(previous.clone()); + let root = root("userns"); + // open the limit: a Sandbox must not start, and nothing may be made + fs::write(LIMIT, "1\n").unwrap(); + let error = NamespaceProvider::new(&root).unwrap_err(); + assert!( + matches!(&error, RuntimeError::Unsupported(message) + if message.contains("user namespaces are not blocked")), + "{error}" + ); + assert_eq!(fs::read_dir(&root).unwrap().count(), 0); + // block it again: the same root is accepted, given the other host facts + fs::write(LIMIT, &previous).unwrap(); + if previous == "0" && Path::new("/dev/fuse").exists() { + assert!(NamespaceProvider::new(&root).is_ok()); + } + drop(restore); + assert_eq!(fs::read_to_string(LIMIT).unwrap().trim(), previous); + let _ = fs::remove_dir_all(&root); +} diff --git a/engine/crates/layerfs-sandbox/tests/native_platform.rs b/engine/crates/layerfs-sandbox/tests/native_platform.rs new file mode 100644 index 0000000000..46ffae1258 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_platform.rs @@ -0,0 +1,71 @@ +//! Platform refusal precedes filesystem access; a runtime never selects another backend. +#![cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ContainerId, Runtime, RuntimeError, +}; +use std::time::Duration; + +#[test] +fn wrong_platform_is_refused_before_root_access() { + #[cfg(target_os = "macos")] + let result = layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider", + ); + #[cfg(target_os = "linux")] + let result = layerfs_sandbox::backend::native::NativeMacosProvider::new( + "/does-not-exist/native-provider", + ); + assert!(matches!(result, Err(RuntimeError::Unsupported(_)))); +} + +#[test] +fn native_linux_requires_root_before_root_access() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(matches!( + layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider" + ), + Err(RuntimeError::Unsupported(_)) + )); + } +} + +/// Windows has no host runtime: the placeholder variant refuses every +/// operation before any effect, and never falls back to Docker or native. +#[test] +fn windows_is_refused_before_any_effect() { + let runtime = Runtime::Windows; + assert!(matches!( + runtime.capabilities(), + Err(RuntimeError::Unsupported(_)) + )); + assert!(matches!( + runtime.local_directory("one"), + Err(RuntimeError::Unsupported(_)) + )); + let request = LaunchRequest { + name: "one".into(), + daemon_executable: "/does-not-exist/daemon".into(), + daemon_config: b"setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }, + }; + let failure = runtime.launch(request).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Validate); + assert!(failure.instance.is_none()); + assert!(matches!(failure.cause, RuntimeError::Unsupported(_))); + let record = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-store".into(), + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(&"b".repeat(64)).unwrap(); + assert!(runtime.reopen(id, &record).is_err()); + assert!(runtime.reopen_stopped(id, &record).is_err()); +} diff --git a/engine/crates/layerfs-sandbox/tests/native_provider.rs b/engine/crates/layerfs-sandbox/tests/native_provider.rs new file mode 100644 index 0000000000..29cf512469 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_provider.rs @@ -0,0 +1,124 @@ +//! Real native runtime lifecycle and partial launch custody through the Runtime enum. +#![cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(target_os = "linux")] +use layerfs_sandbox::backend::native::NativeLinuxProvider as NativeProvider; +#[cfg(target_os = "macos")] +use layerfs_sandbox::backend::native::NativeMacosProvider as NativeProvider; +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, Runtime, +}; +use std::{fs, os::unix::fs::PermissionsExt, time::Duration}; + +/// The native runtime this host offers, over `root`. +#[cfg(target_os = "linux")] +fn native(root: &std::path::Path) -> Runtime { + Runtime::LinuxNative(NativeProvider::new(root).unwrap()) +} + +#[cfg(target_os = "macos")] +fn native(root: &std::path::Path) -> Runtime { + Runtime::MacosNative(NativeProvider::new(root).unwrap()) +} + +#[test] +fn runtime_preserves_partial_launch_owner_and_reopens_the_exact_process() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(NativeProvider::new("/no-access-required").is_err()); + return; + } + let base = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("layerfs-provider-{}", std::process::id())); + fs::create_dir(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let binary = base.join("daemon"); + fs::write( + &binary, + "#!/bin/sh\necho LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\nexec /bin/sleep 30\n", + ) + .unwrap(); + fs::set_permissions(&binary, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = native(&base); + assert_eq!( + provider.capabilities().unwrap().name, + if cfg!(target_os = "linux") { + "native-linux" + } else { + "native-macos" + } + ); + let request = |name: &str| LaunchRequest { + name: name.into(), + daemon_executable: binary.clone(), + daemon_config: b"private setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { + uid: if cfg!(target_os = "linux") { + 501 + } else { + nix::unistd::Uid::effective().as_raw() + }, + gid: if cfg!(target_os = "linux") { + 20 + } else { + nix::unistd::Gid::effective().as_raw() + }, + }, + environment: vec![], + }, + }; + let mut sandbox = provider.launch(request("success")).unwrap(); + let id = sandbox.identity(); + let local_record = sandbox.local_record().unwrap(); + let record = sandbox.record().unwrap(); + assert_eq!(record, SandboxRecord::Local(local_record)); + assert!(sandbox.inspect().unwrap().running); + let reopened = provider.reopen(id, &record).unwrap(); + assert_eq!(reopened.identity(), id); + assert_eq!(reopened.endpoint().unwrap(), sandbox.endpoint().unwrap()); + let foreign = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-foreign".into(), + port: 30421, + labels: vec![], + }); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + drop(reopened); + sandbox.stop(1).unwrap(); + assert!(!sandbox.inspect().unwrap().running); + assert!(provider + .reopen(id, &record) + .err() + .unwrap() + .container_stopped()); + sandbox.delete().unwrap(); + fs::write(&binary, "#!/bin/sh\nexit 3\n").unwrap(); + let mut failure = provider.launch(request("failure")).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Listener); + let mut partial = failure.instance.take().expect("created resource retained"); + partial.stop(1).unwrap(); + partial.delete().unwrap(); + let mut invalid = request("../escape"); + invalid.daemon_config = b"NEVER_PRINT_THIS_SECRET".to_vec(); + assert!(!format!("{invalid:?}").contains("NEVER_PRINT_THIS_SECRET")); + assert_eq!( + provider.launch(invalid).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = request("wrong-options"); + wrong_options.options = ProviderLaunchOptions::Docker { + image: format!("sha256:{}", "b".repeat(64)), + store_volume: "lfs-foreign".into(), + port: 30421, + connect_host: None, + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-sandbox/tests/runtime_enum.rs b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs new file mode 100644 index 0000000000..1f7c1ba78f --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs @@ -0,0 +1,15 @@ +//! The Runtime enum's Docker selection: no local directory, container capabilities. +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::Runtime; +use std::time::Duration; + +/// Selecting Docker names an Engine endpoint only; nothing is dialed here. +#[test] +fn docker_reports_container_capabilities_and_owns_no_local_directory() { + let docker = Docker::new("/does-not-exist/docker.sock", Duration::from_secs(1)).unwrap(); + let runtime = Runtime::Docker(DockerProvider::new(docker)); + let capabilities = runtime.capabilities().unwrap(); + assert_eq!(capabilities.name, "docker"); + assert!(capabilities.requires_image); + assert!(runtime.local_directory("anything").unwrap().is_none()); +} diff --git a/engine/crates/layerfs-sandbox/tests/support/image.rs b/engine/crates/layerfs-sandbox/tests/support/image.rs new file mode 100644 index 0000000000..48dba21eea --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/support/image.rs @@ -0,0 +1,150 @@ +//! A docker-archive as a bundle would carry it, built in the test: the format +//! `docker save` writes (`manifest.json`, a config blob named by the image's +//! digest, uncompressed layer tars). Written by hand so a test needs no +//! Engine, no Docker and no root. +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// The digest the fixtures' images use unless a test needs another. +pub const DIGEST: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +/// A second digest, for one cache that holds two images. +pub const OTHER: &str = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + +/// The 64 hex digits of a digest. +pub fn hex(digest: &str) -> &str { + digest.strip_prefix("sha256:").expect("a sha256 digest") +} + +/// One tar archive, built entry by entry. +pub struct Tar { + bytes: Vec, +} + +impl Tar { + pub fn new() -> Self { + Self { bytes: Vec::new() } + } + + /// A regular file with `mode` and `body`. + pub fn file(self, path: &str, mode: u32, body: &[u8]) -> Self { + self.entry(path, mode, b'0', body, "") + } + + /// A directory entry; `path` ends with `/` as tar writes it. + pub fn directory(self, path: &str, mode: u32) -> Self { + self.entry(path, mode, b'5', &[], "") + } + + pub fn symlink(self, path: &str, target: &str) -> Self { + self.entry(path, 0o777, b'2', &[], target) + } + + pub fn hard_link(self, path: &str, target: &str) -> Self { + self.entry(path, 0o644, b'1', &[], target) + } + + /// Any entry type the writer must be able to produce, including the ones + /// a Sandbox root refuses (a device node is type `3`). + pub fn raw(self, path: &str, kind: u8, mode: u32, body: &[u8]) -> Self { + self.entry(path, mode, kind, body, "") + } + + fn entry(mut self, path: &str, mode: u32, kind: u8, body: &[u8], link: &str) -> Self { + assert!( + path.len() < 100 && link.len() < 100, + "{path} or {link} needs a pax header" + ); + let mut header = [0u8; 512]; + header[..path.len()].copy_from_slice(path.as_bytes()); + octal(&mut header[100..108], u64::from(mode)); + octal(&mut header[108..116], 0); + octal(&mut header[116..124], 0); + octal(&mut header[124..136], body.len() as u64); + octal(&mut header[136..148], 0); + header[156] = kind; + header[157..157 + link.len()].copy_from_slice(link.as_bytes()); + header[257..263].copy_from_slice(b"ustar\0"); + header[263..265].copy_from_slice(b"00"); + header[148..156].fill(b' '); + let sum: u64 = header.iter().map(|byte| u64::from(*byte)).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + self.bytes.extend_from_slice(&header); + self.bytes.extend_from_slice(body); + let padding = (512 - body.len() % 512) % 512; + self.bytes.resize(self.bytes.len() + padding, 0); + self + } + + /// The archive's bytes, ending with the two zero blocks that end a tar. + pub fn finish(mut self) -> Vec { + self.bytes.resize(self.bytes.len() + 1024, 0); + self.bytes + } +} + +fn octal(field: &mut [u8], value: u64) { + let text = format!("{value:0width$o}\0", width = field.len() - 1); + field.copy_from_slice(text.as_bytes()); +} + +/// A docker-archive at `/.tar` holding one image whose config +/// digest is `digest`, with `layers` applied in the order given. +pub fn archive(directory: &Path, name: &str, digest: &str, layers: Vec>) -> PathBuf { + write( + directory, + name, + &format!("blobs/sha256/{}", hex(digest)), + layers, + ) +} + +/// The same image as older writers name it: the config blob is `.json`. +pub fn legacy_archive(directory: &Path, name: &str, digest: &str, layers: Vec>) -> PathBuf { + write(directory, name, &format!("{}.json", hex(digest)), layers) +} + +fn write(directory: &Path, name: &str, config: &str, layers: Vec>) -> PathBuf { + let mut manifest = + format!("[{{\"Config\":\"{config}\",\"RepoTags\":[\"layerfs-test:latest\"],\"Layers\":["); + for index in 0..layers.len() { + if index > 0 { + manifest.push(','); + } + manifest.push_str(&format!("\"{index}/layer.tar\"")); + } + manifest.push_str("]}]"); + let mut tar = Tar::new() + .file("manifest.json", 0o644, manifest.as_bytes()) + .file(config, 0o644, br#"{"architecture":"amd64","os":"linux"}"#); + for (index, layer) in layers.iter().enumerate() { + tar = tar.file(&format!("{index}/layer.tar"), 0o644, layer); + } + let path = directory.join(format!("{name}.tar")); + fs::write(&path, tar.finish()).unwrap(); + path +} + +/// A scratch directory one test owns and removes when it ends. +pub struct Scratch { + pub base: PathBuf, + pub cache: PathBuf, +} + +impl Scratch { + pub fn new(name: &str) -> Self { + let base = + std::env::temp_dir().join(format!("layerfs-rootfs-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + let cache = base.join("images"); + Self { base, cache } + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.base); + } +} diff --git a/engine/crates/layerfs-store-service/src/install.rs b/engine/crates/layerfs-store-service/src/install.rs index 850570a334..f150c5685c 100644 --- a/engine/crates/layerfs-store-service/src/install.rs +++ b/engine/crates/layerfs-store-service/src/install.rs @@ -21,6 +21,7 @@ use std::sync::Arc; /// The daemon's user inside its Sandbox (the runtime's `User` is `0:0`). A /// daemon opens only a Store file and directory it owns, so an installer /// must run as this user too. +#[cfg(not(target_os = "macos"))] const DAEMON_UID: u32 = 0; /// The destination and its exclusive temporary name. @@ -202,7 +203,15 @@ pub fn install_local( result.map_err(|cause: String| LocalInstallFailure { cause, ..left }) } +/// macOS has only the native route, whose daemon runs as the user who +/// installs its Store (there is no root daemon to match); the daemon's own +/// check that it owns the Store file still holds. +#[cfg(target_os = "macos")] +fn installer_user() -> Result<(), String> { + Ok(()) +} /// Refuses an installer that is not the daemons' user. +#[cfg(not(target_os = "macos"))] fn installer_user() -> Result<(), String> { use std::os::unix::fs::MetadataExt; // the owner of /proc/self is this process's effective user diff --git a/engine/crates/layerfs-store-service/src/kernel_guard.rs b/engine/crates/layerfs-store-service/src/kernel_guard.rs index 49489cac1e..e0e3945972 100644 --- a/engine/crates/layerfs-store-service/src/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/src/kernel_guard.rs @@ -56,7 +56,32 @@ pub fn check(store: &Path) -> Result<&'static str, Unsupported> { )) }) } - #[cfg(not(target_os = "linux"))] + // macOS native (LAYERFS_SANDBOX=native): the daemon, the SDK and every + // command share this one macOS kernel, so the Store's locks coordinate + // exactly as on Linux; only a local filesystem may hold the Store. + #[cfg(target_os = "macos")] + { + let directory = store + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let stat = nix::sys::statfs::statfs(directory).map_err(|error| { + Unsupported(format!( + "the Store's directory {} cannot be examined: {error}", + directory.display() + )) + })?; + match stat.filesystem_type_name() { + "apfs" => Ok("apfs"), + "hfs" => Ok("hfs"), + other => Err(Unsupported(format!( + "filesystem {other} at {} does not carry SQLite locks between kernels; \ + direct Store access needs apfs or hfs", + directory.display() + ))), + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { let _ = store; Err(Unsupported( diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index d9050f370d..e99d46ea85 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -1,11 +1,13 @@ //! Direct Store access is refused `Unsupported` where the Store's locks do -//! not reach every opener (#352 F-020): on macOS, and on Linux for a -//! filesystem outside the allowlist (here tmpfs). The refusal comes before any -//! file is opened: the Store file's bytes are unchanged and no overlay -//! template is created. +//! not reach every opener (#352 F-020): off Linux and macOS, and on Linux for +//! a filesystem outside the allowlist (here tmpfs). The refusal comes before +//! any file is opened: the Store file's bytes are unchanged and no overlay +//! template is created. On macOS a local apfs or hfs directory is allowed. +#[cfg(not(target_os = "macos"))] use layerfs_store_service::{OpenRefusal, StoreService}; use std::path::{Path, PathBuf}; +#[cfg(not(target_os = "macos"))] fn refused(directory: &Path) -> (String, Vec, Vec) { let store = directory.join("store.sqlite"); std::fs::write(&store, b"not opened").unwrap(); @@ -30,7 +32,16 @@ fn scratch(parent: &Path, label: &str) -> PathBuf { directory } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] +#[test] +fn direct_access_is_allowed_on_a_local_macos_filesystem() { + let directory = scratch(&std::env::temp_dir(), "apfs"); + let checked = layerfs_store_service::kernel_guard::check(&directory.join("store.sqlite")); + assert!(matches!(checked, Ok("apfs") | Ok("hfs")), "{checked:?}"); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] fn direct_access_is_unsupported_off_linux() { let directory = scratch(&std::env::temp_dir(), "host"); diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index c4159c3c12..97201c5102 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -17,12 +17,14 @@ # Crates whose `unsafe` surface is bounded. layerfs-storage keeps one audited # FFI module (encoding/codec.rs); Persistence has the owner-authorized macOS -# seal file-control module. Their siblings stay unsafe-free. The +# seal file-control module; Sandbox reads macOS process facts through +# proc_pidinfo in backend/macos/darwin.rs. Their siblings stay unsafe-free. The # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-sandbox": "src/backend/macos/darwin.rs", } UNSAFE_FREE_CRATES = ( "layerfs-content", @@ -32,10 +34,10 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", - "layerfs-sandbox", ) UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", @@ -47,10 +49,11 @@ "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", + "layerfs-nfs": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", - "layerfs-sandbox": "#![forbid(unsafe_code)]", + "layerfs-sandbox": "#![deny(unsafe_code)]", } @@ -84,6 +87,8 @@ "layerfs-overlay": set(), "layerfs-workspace": {"layerfs-content", "layerfs-overlay", "layerfs-telemetry"}, "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, + # The NFS export reuses the kernel ports and operations; never daemon. + "layerfs-nfs": {"layerfs-content", "layerfs-overlay", "layerfs-workspace", "layerfs-fuse"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. "layerfs-store-service": { @@ -103,6 +108,7 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-bridge", "layerfs-content", "layerfs-storage", @@ -135,6 +141,7 @@ "layerfs-bridge": ENGINE, "layerfs-daemon": ENGINE, "layerfs-fuse": ENGINE, + "layerfs-nfs": ENGINE, "layerfs-overlay": ENGINE, "layerfs-persistence": ENGINE, "layerfs-project": ENGINE, @@ -196,6 +203,7 @@ def test_path_violations(path, source, packages): "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", diff --git a/tools/ci_tests.py b/tools/ci_tests.py index ac284bf549..b07ff4b0d5 100644 --- a/tools/ci_tests.py +++ b/tools/ci_tests.py @@ -40,6 +40,7 @@ # (package, test target): run seconds on the macOS host, 2026-10-23. SLOW = { + ("layerfs-nfs", "helper_deadline"): 60.1, # Six Linux FUSE teardown cases, including two bounded Join observations. ("layerfs-daemon", "stalled_unmount"): 21.28, # Linux native binding proof, owner-excluded from CI; run explicitly. diff --git a/tools/test_check_product_boundary.py b/tools/test_check_product_boundary.py index ec4196fbff..55f600b5f4 100644 --- a/tools/test_check_product_boundary.py +++ b/tools/test_check_product_boundary.py @@ -126,7 +126,15 @@ def test_sdk_source_edges_and_unsafe_boundary(self): def test_sandbox_active_edges_and_unsafe_boundary(self): path = Path("engine/crates/layerfs-sandbox/src/backend/docker/http.rs") self.assertTrue(unsafe_violations(path, "unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![deny(unsafe_code)]")) + self.assertTrue(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + darwin = Path("engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs") + self.assertFalse(unsafe_violations(darwin, "fn f() { unsafe { libc::proc_listallpids(); } }")) + self.assertTrue( + unsafe_violations( + Path("engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs"), "fn f() { unsafe { call(); } }" + ) + ) prefix = '[package]\nname="layerfs-sandbox"\n[dependencies]\n' for dependency in ( "layerfs-api-core", diff --git a/tools/worktree_build.py b/tools/worktree_build.py index 3af7c4bd28..cdabf8d1e8 100644 --- a/tools/worktree_build.py +++ b/tools/worktree_build.py @@ -219,6 +219,9 @@ def build(args): "layerfs-daemon", "-p", "layerfs-node", + # the namespaces Sandbox backend runs this beside the daemon (#515 §4.5) + "-p", + "layerfs-sandbox", "-j", str(args.jobs), ] @@ -230,8 +233,9 @@ def build(args): execute(docker_build(record, command), record["worktree"], log=receipt / "output.log") script = ( 'set -eu; cp "/work/target/$1/layerfs-daemon" /export/layerfs-daemon; ' + 'cp "/work/target/$1/layerfs-sandbox-helper" /export/layerfs-sandbox-helper; ' 'cp "/work/target/$1/liblayerfs_node.so" /export/layerfs.node; ' - 'chown "$2:$3" /export/layerfs-daemon /export/layerfs.node' + 'chown "$2:$3" /export/layerfs-daemon /export/layerfs-sandbox-helper /export/layerfs.node' ) export = docker_build( record,