diff --git a/Cargo.lock b/Cargo.lock index ca39be0e98..a9221a273c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -590,6 +590,7 @@ dependencies = [ "layerfs-content", "layerfs-fuse", "layerfs-history", + "layerfs-nfs", "layerfs-overlay", "layerfs-persistence", "layerfs-project", @@ -620,6 +621,23 @@ dependencies = [ "layerfs-content", ] +[[package]] +name = "layerfs-nfs" +version = "1.1.0" +dependencies = [ + "layerfs-bridge", + "layerfs-content", + "layerfs-daemon", + "layerfs-fuse", + "layerfs-history", + "layerfs-overlay", + "layerfs-persistence", + "layerfs-project", + "layerfs-storage", + "layerfs-telemetry", + "layerfs-workspace", +] + [[package]] name = "layerfs-node" version = "1.1.0" @@ -633,6 +651,7 @@ dependencies = [ "layerfs-storage", "layerfs-store-service", "layerfs-telemetry", + "libc", "napi", "napi-build", "napi-derive", diff --git a/api/agent/src/client.js b/api/agent/src/client.js index 7e1ae74a98..8455d1cb28 100644 --- a/api/agent/src/client.js +++ b/api/agent/src/client.js @@ -17,8 +17,9 @@ export const BACKEND = /^(backend|runtime)\./; /** Runs one operation once and returns its value or throws its refusal; * `options` are the client's settings, whose `trace` selects its trace file; - * `transport` is the client's binding (default: the one a new client would use). */ -export async function invoke(op, args, options, transport = binding()) { + * `transport` is the client's binding (default: the one a new client of + * `args.project` would use). */ +export async function invoke(op, args, options, transport = binding(args.project)) { return outcome(await traced(options, op, args, (scope) => { if (BACKEND.test(op)) throw new LayerFSError("UnknownMethod", "backend and runtime methods are @layerfs/backend's and @layerfs/runtime's", null); return transport.call(op, observed(args, scope)); @@ -26,7 +27,7 @@ export async function invoke(op, args, options, transport = binding()) { } /** One call on the binding's backend table (`backend.*`, `runtime.*`). */ -export async function invokeBackend(op, args, options, transport = binding()) { +export async function invokeBackend(op, args, options, transport = binding(args?.project)) { return outcome(await traced(options, op, args, (scope) => transport.backendCall(op, observed(args, scope)))); } @@ -60,9 +61,19 @@ export function sandboxArg(sandbox) { /** Deployment settings from the client options, else the environment * (LAYERFS_). A Workspace's Sandbox needs - * every one but the container prefix and daemon host. */ + * every one but the container prefix and daemon host. With `sandbox` + * (LAYERFS_SANDBOX) `native` (Linux or macOS), a Sandbox is a host daemon: + * Linux uses FUSE with a root daemon and nonroot commands (501:20); + * macOS uses loopback NFS and this user. It needs the daemon and + * `localDir` (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is + * Docker; any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || undefined; + const sandbox = options.sandbox ?? env("SANDBOX"); + if (sandbox === "native") return nativeDeployment(options, env, sandbox); + if (sandbox !== undefined && sandbox !== "docker") { + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not ${sandbox}`, null); + } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", image: options.image ?? env("IMAGE"), @@ -80,6 +91,35 @@ export function deployment(options = {}) { return deploy; } +/** A native Sandbox's settings: the daemon, the directory its Sandboxes + * live in, the optional name prefix, and the backend inside that directory + * (`sandboxBackend`, LAYERFS_SANDBOX_BACKEND: `native` when unset, or + * `namespaces` on Linux). The namespaces backend gives each Sandbox a root + * filesystem from an image, so it also needs `image` (LAYERFS_IMAGE, the + * same digest Docker deploys) and `imageArchive` (LAYERFS_IMAGE_ARCHIVE, the + * docker-archive the bundle carries). */ +function nativeDeployment(options, env, sandbox) { + const deploy = { + sandbox, + daemon: options.daemon ?? env("DAEMON"), + localDir: options.localDir ?? env("LOCAL_DIR"), + containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), + }; + const backend = options.sandboxBackend ?? env("SANDBOX_BACKEND"); + if (backend !== undefined) deploy.sandboxBackend = backend; + const required = [["daemon", "DAEMON"], ["localDir", "LOCAL_DIR"]]; + if (backend === "namespaces") { + deploy.image = options.image ?? env("IMAGE"); + deploy.imageArchive = options.imageArchive ?? env("IMAGE_ARCHIVE"); + required.push(["image", "IMAGE"], ["imageArchive", "IMAGE_ARCHIVE"]); + } + for (const [name, setting] of required) { + if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); + } + if (!deploy.containerPrefix) delete deploy.containerPrefix; + return deploy; +} + /** Refuses every call on a closed client (or one no factory opened). */ export function live(client) { if (client?.[STATE]?.closed !== false) { diff --git a/api/agent/src/index.js b/api/agent/src/index.js index 446b59a534..1dc96b3c7e 100644 --- a/api/agent/src/index.js +++ b/api/agent/src/index.js @@ -100,7 +100,15 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON; a * Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. + * container it starts `-`. On Linux or macOS, + * LAYERFS_SANDBOX=native starts each Sandbox as a host daemon in a directory + * under LAYERFS_LOCAL_DIR. Linux requires root for FUSE and runs commands + * as 501:20; macOS uses loopback NFS and the caller's identity. No Docker + * setting is read; native provides no isolation from the host. + * LAYERFS_SANDBOX_BACKEND picks the backend inside that directory: `native` + * (unset) or, on Linux, `namespaces` (namespaces, cgroups, nonroot commands). + * It is read for every call, so a Sandbox made with one backend is never + * reopened as another. * LAYERFS_DAEMON_HOST is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on @@ -129,13 +137,18 @@ export class LayerFS { volume: env("VOLUME"), daemon: env("DAEMON"), ...(env("CONTAINER_PREFIX") ? { containerPrefix: env("CONTAINER_PREFIX") } : {}), + ...(env("SANDBOX") ? { sandbox: env("SANDBOX") } : {}), + ...(env("SANDBOX_BACKEND") ? { sandboxBackend: env("SANDBOX_BACKEND") } : {}), + ...(env("LOCAL_DIR") ? { localDir: env("LOCAL_DIR") } : {}), + ...(env("IMAGE_ARCHIVE") ? { imageArchive: env("IMAGE_ARCHIVE") } : {}), ...(env("DAEMON_HOST") ? { daemonHost: env("DAEMON_HOST") } : {}), }, storePolicy: env("STORE_POLICY"), storePath: env("STORE_PATH"), }; - // not open until a factory has opened its Store; the binding is chosen now, once - this[STATE] = { closed: null, transport: binding() }; + // not open until a factory has opened its Store; the binding is chosen now, once, + // for this client's project (a VM placement has one worker per project) + this[STATE] = { closed: null, transport: binding(this.projectName) }; this.fs = bind(this, fsMethods); this.project = bind(this, { list_branches, list_sandboxes }); this.branch = bind(this, { list_commits, diff_commits, diff_branches, fork, merge, publish }); diff --git a/api/agent/test/native_deployment.mjs b/api/agent/test/native_deployment.mjs new file mode 100644 index 0000000000..4b0d951e30 --- /dev/null +++ b/api/agent/test/native_deployment.mjs @@ -0,0 +1,143 @@ +// Which runtime a Sandbox uses. No real addon, Store or Docker (the +// stand-in binding answers the client's calls): +// node api/agent/test/native_deployment.mjs +// LAYERFS_SANDBOX unset or `docker` gives exactly the Docker deployment it +// always gave; `native` (macOS) needs only the daemon and LAYERFS_LOCAL_DIR +// (no Docker setting); any other value, the former `local` included, is +// refused rather than taken as either. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { deployment } from "../src/client.js"; + +const FAKE = fileURLToPath(new URL("./fixture/fake-native.cjs", import.meta.url)); +const native = createRequire(import.meta.url)(FAKE); + +const clear = () => { + for (const name of Object.keys(process.env)) { + if (name.startsWith("LAYERFS_") || name.startsWith("DSH_LAYERFS_")) delete process.env[name]; + } + process.env.LAYERFS_NATIVE = FAKE; +}; +const docker = { docker: "/run/docker.sock", image: `sha256:${"a".repeat(64)}`, volume: "lfs-v", daemon: "/opt/layerfs-daemon" }; +const setDocker = () => { + process.env.LAYERFS_DOCKER = docker.docker; + process.env.LAYERFS_IMAGE = docker.image; + process.env.LAYERFS_VOLUME = docker.volume; + process.env.LAYERFS_DAEMON = docker.daemon; +}; +const store = () => { + process.env.LAYERFS_HOME = "/private/local-home"; + process.env.LAYERFS_STORE_PATH = "/fake/store.sqlite"; +}; + +// unset: the Docker deployment, field for field +clear(); +setDocker(); +assert.deepEqual(deployment(), docker); +// `docker` is the same deployment +process.env.LAYERFS_SANDBOX = "docker"; +assert.deepEqual(deployment(), docker); +// the Docker settings are still required there +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_IMAGE/.test(e.message)); + +// native: daemon + localDir, no Docker setting read or required +clear(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +assert.deepEqual(deployment(), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// an image (sandbox.create's option) does not change a native deployment +assert.deepEqual(deployment({ image: docker.image }), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// the backend inside the directory is native unless named; namespaces is +// passed through (the binding refuses it off Linux, and an unknown value) +assert.equal(deployment().sandboxBackend, undefined); +const archive = `/bundle/images/${"a".repeat(64)}.tar`; +process.env.LAYERFS_SANDBOX_BACKEND = "namespaces"; +process.env.LAYERFS_IMAGE = docker.image; +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// a namespaces Sandbox's root is an image: the same digest Docker deploys +// and the docker-archive the bundle carries +assert.deepEqual(deployment(), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "namespaces", + image: docker.image, + imageArchive: archive, +}); +// each of those is named when it is missing +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE\)/.test(e.message)); +process.env.LAYERFS_IMAGE = docker.image; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /\(LAYERFS_IMAGE_ARCHIVE\)/.test(e.message)); +process.env.LAYERFS_IMAGE_ARCHIVE = archive; +// the native backend inside the same directory reads neither of them +assert.deepEqual(deployment({ sandboxBackend: "native" }), { + sandbox: "native", + daemon: "/opt/layerfs-daemon", + localDir: "/Users/me/lfs", + sandboxBackend: "native", +}); +delete process.env.LAYERFS_SANDBOX_BACKEND; +delete process.env.LAYERFS_IMAGE; +delete process.env.LAYERFS_IMAGE_ARCHIVE; +// the container prefix names native Sandbox directories too +process.env.LAYERFS_CONTAINER_PREFIX = "ci-"; +assert.equal(deployment().containerPrefix, "ci-"); +delete process.env.LAYERFS_CONTAINER_PREFIX; +// each required setting is named when missing +delete process.env.LAYERFS_LOCAL_DIR; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_LOCAL_DIR/.test(e.message)); +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +delete process.env.LAYERFS_DAEMON; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_DAEMON/.test(e.message)); +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +// a DSH_LAYERFS_NFS_ name is not mapped (an integration maps its own names) +process.env.DSH_LAYERFS_NFS_MOUNT = "/bin/true"; +deployment(); +assert.equal(process.env.LAYERFS_NFS_MOUNT, undefined); +// since #428 the SDK reads LAYERFS_ names only: a DSH_LAYERFS_ twin selects nothing +clear(); +setDocker(); +process.env.DSH_LAYERFS_SANDBOX = "native"; +process.env.DSH_LAYERFS_LOCAL_DIR = "/l"; +assert.deepEqual(deployment(), docker); + +// anything else, the former `local` included, is refused, never silently Docker +for (const typo of ["local", "Native", "nativ", "NATIVE", "podman"]) { + clear(); + setDocker(); + process.env.LAYERFS_SANDBOX = typo; + assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && e.message.includes(`not ${typo}`)); +} + +// a client reads LAYERFS_SANDBOX and LAYERFS_LOCAL_DIR once, and +// sandbox.create sends the native deployment (no Docker setting) +clear(); +store(); +process.env.LAYERFS_SANDBOX = "native"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +const { LayerFS } = await import("../src/index.js"); +const client = await LayerFS.open("website"); +assert.equal(client.options.deploy.sandbox, "native"); +assert.equal(client.options.deploy.localDir, "/Users/me/lfs"); +const before = native.deploys.length; +await client.sandbox.create(); +assert.deepEqual(native.deploys.slice(before), [{ sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }]); +// an image given to sandbox.create does not turn it into a Docker deployment +await client.sandbox.create({ image: docker.image }); +assert.deepEqual(native.deploys.at(-1), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +await client.close(); +// unset: the client carries no new key +clear(); +store(); +setDocker(); +const plain = await LayerFS.open("website"); +assert.equal("sandbox" in plain.options.deploy, false); +assert.equal("localDir" in plain.options.deploy, false); +await plain.close(); +console.log("native_deployment: ok"); diff --git a/api/agent/test/verify/native_unmount_real.mjs b/api/agent/test/verify/native_unmount_real.mjs new file mode 100644 index 0000000000..d3c6b71b6a --- /dev/null +++ b/api/agent/test/verify/native_unmount_real.mjs @@ -0,0 +1,138 @@ +// Real native SDK regression. Explicit binaries + isolated root are required. +// node api/agent/test/verify/native_unmount_real.mjs /abs/root /abs/addon.node /abs/daemon +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, writeFile, readFile, access, readdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +const [base, addon, daemon, childMode, serialized] = process.argv.slice(2); +assert.ok(base?.startsWith('/') && addon?.startsWith('/') && daemon?.startsWith('/'), 'explicit absolute fixture paths'); +const repo = fileURLToPath(new URL('../../../..', import.meta.url)); +Object.assign(process.env, { + LAYERFS_SANDBOX: 'native', LAYERFS_NATIVE: addon, LAYERFS_DAEMON: daemon, + LAYERFS_HOME: base+'/home', LAYERFS_LOCAL_DIR: base+'/local', LAYERFS_STORE_PATH: base+'/store/store.sqlite', + LAYERFS_NFS_UNMOUNT: `sh ${base}/helper.sh {dir}`, +}); +const { LayerFS } = await import(repo+'/api/agent/src/index.js'); +const { LayerFSBackend } = await import(repo+'/api/backend/src/index.js'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function deadline(promise, ms) { + let timer; + try { return await Promise.race([promise, new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error('deadline '+ms)),ms);})]); } + finally { clearTimeout(timer); } +} +async function waitFile(path) { + const end=Date.now()+5000; + while(Date.now()=0);return error.detail;} +} +function child(mode, ws) { + const p=spawn(process.execPath,[fileURLToPath(import.meta.url),base,addon,daemon,mode,JSON.stringify(ws)],{env:process.env,stdio:['ignore','pipe','pipe']}); + let output='';p.stdout.on('data',d=>output+=d);p.stderr.on('data',d=>output+=d); + const ended=new Promise(resolve=>p.once('exit',(code,signal)=>resolve({code,signal,output}))); + return {p,ended}; +} +if(childMode){ + const lf=await LayerFS.open('unmount');const ws=JSON.parse(serialized); + if(childMode==='probe')await fast('cross-process-same-workspace',()=>refused(()=>lf.workspace.unmount(ws,true))); + if(childMode==='owner')await lf.workspace.unmount(ws,true); + await lf.close(); +}else{ + assert.equal(process.platform,'darwin','this fixture uses host mount_nfs'); + await mkdir(base+'/local',{recursive:true});await mkdir(base+'/seed',{recursive:true});await mkdir(base+'/store',{recursive:true,mode:0o700}); + await writeFile(base+'/seed/file','base\n'); + await writeFile(base+'/helper.sh',`#!/bin/sh\nprintf '%s' "$1" > '${base}/started'\ni=0\nwhile [ ! -f '${base}/release' ] && [ "$i" -lt 200 ]; do sleep 0.1; i=$((i+1)); done\nif [ -f '${base}/fail' ]; then echo injected-helper-failure >&2; exit 1; fi\nexec /sbin/umount "$1"\n`); + const lf=await LayerFS.init('unmount',base+'/seed');let owner;let active; + try{ + await lf.branch.fork('a');await lf.branch.fork('b'); + const sb=await lf.sandbox.create();const a=await lf.workspace.mount(sb.sandbox,'a');const b=await lf.workspace.mount(sb.sandbox,'b'); + active=lf.workspace.unmount(a,true);active.catch(()=>{}); + await waitFile(base+'/started'); + await fast('sandbox-log-needs-discovery-lock',()=>LayerFSBackend.of(lf).daemon.sandbox.get_log(sb.sandbox,{lines:2})); + await fast('other-workspace-status',()=>LayerFSBackend.of(lf).daemon.workspace.status(b)); + await fast('other-workspace-exec',()=>lf.workspace.exec(b,'printf ready'),r=>assert.equal(r.stdout,'ready')); + await fast('same-workspace-status',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.status(a))); + await fast('same-workspace-exec',()=>refused(()=>lf.workspace.exec(a,'true'))); + await fast('same-workspace-commit',()=>refused(()=>lf.workspace.commit(a,'must not run'))); + await fast('same-workspace-list-changes',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.list_changes(a))); + await fast('repeat-unmount',()=>refused(()=>lf.workspace.unmount(a,true))); + await fast('sandbox-close',()=>refused(()=>lf.sandbox.close(sb.sandbox))); + const probe=child('probe',a);assert.equal((await deadline(probe.ended,5000)).code,0); + const another=await fast('new-workspace-mount',()=>lf.workspace.mount(sb.sandbox,'b')); + const locker=spawn('python3',['-c','import fcntl,sys; f=open(sys.argv[1],"r+"); fcntl.flock(f,fcntl.LOCK_EX); print("LOCKED",flush=True); sys.stdin.read(1)',base+'/home/unmount/sandboxes/'+sb.sandbox+'/ownership.lock'],{stdio:['pipe','pipe','pipe']}); + try { + await deadline(new Promise((resolve,reject)=>{locker.stdout.once('data',resolve);locker.once('error',reject);}),2000); + await writeFile(base+'/release','go');await sleep(200); + // The terminal result is visible before cleanup can reacquire this lock. + await fast('published-success-before-cleanup-lock',()=>lf.workspace.unmount(a,true)); + } finally { locker.stdin.end('x');await deadline(new Promise(r=>locker.once('exit',r)),2000); } + await deadline(active,5000);active=null; + await fast('repeat-completed-unmount',()=>lf.workspace.unmount(a,true)); + // Busy and Dirty are reversible: release only this attempt's intent. + await lf.workspace.exec(b,'printf dirty > dirty'); + await assert.rejects(lf.workspace.unmount(b),e=>e.code==='Dirty'); + await lf.workspace.exec(b,'printf still-usable'); + const holder=spawn('/bin/sleep',['5'],{cwd:b.path,stdio:'ignore'}); + try { await assert.rejects(lf.workspace.unmount(b,true),e=>e.code==='Busy'); } + finally { holder.kill('SIGTERM');await deadline(new Promise(r=>holder.once('exit',r)),2000); } + await lf.workspace.unmount(b,true);await lf.workspace.unmount(another,true);await lf.sandbox.close(sb.sandbox); + // A killed binding loses its lease, not its intent. No subsequent caller + // may send a second unmount when the daemon may still own the first. + const {unlink}=await import('node:fs/promises');await unlink(base+'/release');await unlink(base+'/started'); + const sb2=await lf.sandbox.create();const lost=await lf.workspace.mount(sb2.sandbox,'a'); + owner=child('owner',lost);await waitFile(base+'/started');owner.p.kill('SIGKILL');await deadline(owner.ended,5000); + await fast('lost-owner-no-replay',()=>refused(()=>lf.workspace.unmount(lost,true),'Retained')); + await writeFile(base+'/release','go'); + await sleep(300); + // Explicit close takes over the orphan while its daemon is healthy; + // it never resends the unknown Workspace unmount. + await fast('lost-owner-close-takeover',()=>lf.sandbox.close(sb2.sandbox)); + // A failed helper leaves daemon custody and the original SDK error. + // Repeated calls read that terminal result rather than launching again. + const sb3=await lf.sandbox.create();const kept=await lf.workspace.mount(sb3.sandbox,'a'); + await writeFile(base+'/fail','fail'); + const error=await lf.workspace.unmount(kept,true).then(()=>null,e=>e);assert.ok(error); + await fast('repeat-retained-result',async()=>{ + const repeated=await lf.workspace.unmount(kept,true).then(()=>null,e=>e); + assert.equal(repeated.code,error.code);assert.equal(repeated.message,error.message);assert.equal(repeated.detail.state,'Retained'); + }); + await fast('retained-close-takeover',()=>lf.sandbox.close(sb3.sandbox)); + console.log('UNMOUNT_CONCURRENCY PASS'); + }finally{ + await writeFile(base+'/release','cleanup'); + if(owner?.p.exitCode===null&&owner?.p.signalCode===null)owner.p.kill('SIGKILL'); + if(active)await deadline(active.catch(()=>{}),5000); + // Always give the fixture helpers permission to finish. A failed test + // must not leave a stopped helper or a held client request behind. + await lf.close(); + // Failure cleanup is scoped by the explicit fixture root. Detach while + // its daemon is still alive; never remove directories through mounts. + const {spawnSync}=await import('node:child_process'); + const table=spawnSync('/sbin/mount',[],{encoding:'utf8',timeout:5000}); + for(const line of (table.stdout||'').split('\n')){ + const point=line.match(/ on (.*) \(/)?.[1]; + if(point?.startsWith(base+'/local/')){ + const off=spawnSync('/sbin/umount',['-f',point],{encoding:'utf8',timeout:15000,killSignal:'SIGKILL'}); + assert.equal(off.status,0,'fixture detach: '+off.stderr); + } + } + const processes=spawnSync('/bin/ps',['-axo','pid=,command='],{encoding:'utf8',timeout:5000}); + for(const line of (processes.stdout||'').split('\n')){ + if(line.includes(daemon+' --config '+base+'/local/')){ + const pid=Number(line.trim().split(/\s+/)[0]); + try{process.kill(pid,'SIGTERM');}catch(e){if(e.code!=='ESRCH')throw e;} + } + } + } +} diff --git a/api/docs/agent.md b/api/docs/agent.md index 330fcab914..919fcc934b 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -45,7 +45,7 @@ async and return a ready client, and `new LayerFS()` is refused `InvalidInput`. **There is no options argument:** every setting comes from the environment, read once when the client is made, by its `LAYERFS_` name only: `LAYERFS_HOME`, `_STORE_PATH`, `_DOCKER`, `_IMAGE`, -`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_TRACE`, +`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_SANDBOX`, `_LOCAL_DIR`, `_TRACE`, `_STORE_POLICY` (and `_TRACE_STEP`, read at each call). An object passed where the options went (`LayerFS.open(name, {…})`, or an object as `LayerFS.init`'s third argument) is refused `InvalidInput`, never ignored. `open` opens the project's Store and makes one project lookup @@ -105,7 +105,9 @@ the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere else opening fails `Unsupported`, and without a `storePath` it fails `InvalidInput`. On macOS and Windows, any process that names its LayerFS home's worker (`LAYERFS_WORKER_ENDPOINT`, or the endpoint `@layerfs/runtime` -hands over with `useWorker`) sends the calls there instead. The route is open to any integration (owner, +hands over with `useWorker` — or that project's endpoint, on a +`vm:` home where each project's Store is in its own VM) sends the +calls there instead. The route is open to any integration (owner, 2026-10-24); LayerFS names none. A client keeps the binding it was made with: the in-process addon, or the worker if one was set when it was made, so a client made before `@layerfs/runtime` hands over its worker stays in-process and @@ -141,10 +143,13 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional path of the real addon; defaults to `engine/crates/layerfs-node/layerfs.node`. | -| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows: the endpoint file of the LayerFS home's worker, which then makes every binding call. Any integration may set it. | +| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows, and a `vm:` project VM: the endpoint file of the worker that makes this project's binding calls. Any integration may set it. | | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | +| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `native` (macOS only): a Sandbox is a daemon process of this user, with no container, no root and no Docker setting read; Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Elsewhere `sandbox.create` refuses `native` as `Unsupported`. Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=native`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each native Sandbox gets its own directory under it. | +| `LAYERFS_SANDBOX_BACKEND` | With `LAYERFS_SANDBOX=native` on Linux, optional: `native` (unset) or `namespaces`. `namespaces` puts each Sandbox's daemon in its own mount, PID and IPC namespaces and its own cgroup v2 directory, and runs commands as the sandbox's uid:gid with no capabilities and no-new-privs; see [Namespace Sandboxes (Linux)](#namespace-sandboxes-linux). It needs root, cgroup v2, `/dev/fuse` and `user.max_user_namespaces=0`, and is refused `Unsupported` elsewhere or without them. Any other value is refused `InvalidInput` before anything is made. It is read when the client is made and sent with every call, so a Sandbox made with one backend is never reopened as the other. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | @@ -244,6 +249,130 @@ Daemon/container restart recovery and replay of uncertain operations are exclude Background process ownership is in the native executor, including persisted process identity; the filesystem daemon launches and supervises no commands. +### Native Sandboxes (macOS) + +`LAYERFS_SANDBOX=native` is the native runtime for macOS, with no root, no +Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon +runs as a process of this user and serves its Workspaces over NFSv3 on +127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them +(`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands +run as this user, in this machine's namespaces: **it isolates nothing** +beyond what the caller's own sandbox (for instance Seatbelt) imposes. A +command starts with this process's `PATH` (so `node`, Homebrew's tools and +the like are found as in the caller's own shell) and its own environment, +nothing else of this process's. Docker stays the default, and Linux keeps +the Docker route only. + +It is the in-process route: the client loads the macOS addon and opens the +Store itself, so no worker endpoint (`LAYERFS_WORKER_ENDPOINT`) is set. It +needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned by +this user, 0755; the directories above it need not be traversable by others) +and `LAYERFS_STORE_PATH` on a local apfs or hfs volume; a Store on any other +filesystem is refused `Unsupported`. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` +and `_DAEMON_HOST` are not read, and an `image` given to `sandbox.create` is +ignored. Everything else above holds: numbered Sandboxes of up to 16 +Workspaces, `sandbox.create` as the only start, `sandbox.close`, idle close +after 2 minutes, reclaim, records under `LAYERFS_HOME//sandboxes//` +and start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME`) and +`daemon.log` (the daemon's output, what `backend.daemon.sandbox.get_log` +reads; a failed start's report carries its last 20 lines). The daemon listens +on 127.0.0.1 only. Its records name the daemon by process id and start time, +so a reused id is never taken for it. `sandbox.create`, the mount result and +`project.list_sandboxes()` give a native Sandbox's `image` as `local`. A +record's `runtime` is `native`; it is used only by a client whose deployment +is `native` with the same `LAYERFS_LOCAL_DIR` and prefix, so it puts that +Sandbox exactly where the record says; any other is refused `InvalidInput` +and nothing is deleted. A record of the former `local` runtime (an earlier, +unmerged build) is never used: it is refused the same way and nothing it +names is deleted. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next `sandbox.create` +or `runtime.sandbox.reclaim` reclaims it, forcing off any mount the dead +daemon left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. `@layerfs/runtime`'s Docker-side +`runtime.sandbox.list` and `sweep` do not see native Sandboxes. Before a Commit the daemon +writes back what the client buffered; after a Commit or merge installs +another root, it makes the client drop the replaced files' cached attributes +at once. + +`LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's +helper commands (`{port}` and `{dir}` are filled in). The default mount is +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,noatime,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`noatime` suppresses client access-time updates, which can otherwise add a +SETATTR for each file read. The export keeps one timestamp, shows it as atime, +mtime and ctime, and drops the access time decoded from SETATTR. +`LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client +then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are +read; an integration that can set only its own prefixed names maps them +itself. + +Limits: background processes still read `/proc` and are not available on +macOS; many small writes cost about six times APFS's time; checked on Apple +silicon with macOS 27.2. Imported application SQLite databases in WAL mode +can fail on this NFS route with the system SQLite VFS; see +[SQLite database preparation and cache placement](../../docs/general/sqlite-on-network-filesystems.md). + +### Namespace Sandboxes (Linux) + +`LAYERFS_SANDBOX_BACKEND=namespaces` keeps the native deployment (no Docker, no +image) and adds isolation inside the project's VM: each Sandbox's daemon runs +in its own mount, PID and IPC namespaces and its own cgroup v2 directory +(`/sys/fs/cgroup/layerfs/`), so two Sandboxes on one Store cannot +see each other's mounts, processes or files, and closing one ends everything +it started. + +It needs root (mount and PID namespaces, `/dev/fuse`), the unified cgroup v2 +hierarchy with `cgroup.kill`, and `user.max_user_namespaces=0`: without the +last, a command could unshare a user namespace and keep its own copy of the +Workspace mount, outside the daemon's unmount. The runtime never sets that +limit; the VM's owner does (`sysctl -w user.max_user_namespaces=0`). A host +that fails any of these is refused `Unsupported` before anything is made, and +the message names the missing condition. + +Sandbox `n` lives where a native Sandbox does — `LAYERFS_LOCAL_DIR/-/` +with `config/daemon.setup`, `overlay/`, `workspaces/`, `tmp/` and +`daemon.log` — plus `bin/layerfs-daemon` (the daemon executable, copied in +0755) and its sibling `-.root/`, the private root its init +stage mounts a tmpfs on and pivots to. The private root holds: the VM's own +userland (`/usr`, `/bin`, `/sbin`, `/lib`, `/lib64`, `/etc`) bound read-only, +so there is no image and no package restoration; a private `/proc` (only the +Sandbox's processes), `/dev` (the basic devices, `/dev/fuse`, a private +`devpts` and `shm`), `/tmp`; the Sandbox's own directory bound at its own path, +so `config/`, `overlay/` and `workspaces/` keep the same names on both +sides; and the Store's directory bound read-write, so the daemon's SQLite WAL +and the writers' directory lock see the same inode as every other opener on +the kernel. The daemon listens on the VM's `127.0.0.1`, as a native one does, +and `127.0.0.1:0` in its setup means the same thing. + +Commands run through `layerfs-sandbox-helper`, which is looked up at +`LAYERFS_SANDBOX_HELPER` when set and otherwise beside the daemon executable. +Each command enters the Sandbox's namespaces and runs as its uid:gid with no +effective or permitted capabilities and no-new-privs; it cannot mount, cannot +unmount the Workspace and cannot create a user namespace. A background process +it leaves behind (even after `setsid`) is ended by the Sandbox's close, which +is one `cgroup.kill` write followed by waiting for the group to be empty; +Delete then removes the cgroup and the directory, so a closed Sandbox leaves no +process, mount or cgroup behind. No graceful filesystem drain exists. + +A record's `runtime` is `namespaces`, with the same fields as a native record; +it is used only by a client whose `LAYERFS_SANDBOX_BACKEND` is `namespaces` +with the same `LAYERFS_LOCAL_DIR` and prefix. Another backend's record — and a +`namespaces` record seen by a native deployment — is refused `InvalidInput` +and nothing is deleted. Reopen checks the recorded init process (id, start +time, boot) and its cgroup membership, so a reused id, or a process outside +the Sandbox's cgroup, is never taken for it. Everything else above holds: +numbered Sandboxes, up to 16 Workspaces each, one daemon per Sandbox, idle +close after 2 minutes, reclaim, and `image` reported as `local`. + +Limits: the network namespace is shared with the VM, so a command can reach +another Sandbox's daemon port on `127.0.0.1`; `/sys` is not mounted; the +userland is the VM's, not a pinned image; and the private root's bind mounts +are read-only at their top only, as this kernel's remount is not recursive. + ### Exec output `exec` returns `{ kind, exitCode, stdout, stderr, message, truncated }` when the diff --git a/core/docs/architecture/nfs-native-lifecycle.md b/core/docs/architecture/nfs-native-lifecycle.md new file mode 100644 index 0000000000..775159664e --- /dev/null +++ b/core/docs/architecture/nfs-native-lifecycle.md @@ -0,0 +1,142 @@ +# Native NFS helper and local Sandbox teardown + +Status: implemented on the macOS native adaptation branch. This describes the +NFS route in `layerfs-daemon`, not the upstream FUSE force-drain protocol in +[forced teardown](80-forced-teardown.md). No persistence profile, wire format, +mount option or Store interface changes here. + +## Helper ownership and deadlines + +The helper budget remains 60 seconds plus five seconds for post-KILL exit +observation. A 15-second cutoff is not qualified: a normal slow writeback may +exceed it, and interrupting a helper cannot prove its kernel effect was undone. +Removing Sandbox ownership from the wait resolves the serialization problem +without trading it for earlier unknown outcomes. + +[`control/export_helper.rs`](../../crates/layerfs-daemon/src/control/export_helper.rs) +owns each NFS mount, unmount, force and optional flush helper attempt. The +public `Helper` and `HELPER_WAIT` exports remain available from `control`. +One attempt gets 60 seconds of exit observation. At expiry or an observation +error it sends one KILL and polls `try_wait` for at most five more seconds. +It never calls an unconditional blocking `wait` after KILL: macOS can leave an +unmount helper in kernel wait where a signal does not establish process exit. + +An unobserved exit is an error with PID, KILL result and retained-child detail. +The process-wide export serving owner retains the `Child`; a later helper call +can reap an exit already observed with `try_wait`, but never signals or reruns +that attempt. No claim is made that the kernel released a mount or that KILL +rolled back a helper's effects. At daemon exit, process ownership passes to the +operating system as for any remaining child. + +Output pipes are nonblocking, drained in bounded chunks during observation and +capped at 2048 retained bytes. An exited helper's descendant can keep a pipe +open, so EOF is not a completion requirement. Output beyond the cap is discarded; +continuous output cannot postpone the exit deadline. A normal nonzero helper +answer containing `busy` keeps the existing reversible Busy disposition. An +interrupted attempt is never classified Busy from partial diagnostic output. + +The control layer keeps a timed-out unmount's export and engine mount at +`Retained/Detach`; subsequent requests report the same custody without replay. +A timed-out **mount** may already have reached the kernel, so Attach also retains +the serving export, engine mount and directory rather than revoking or deleting +through an unknown client mount. No Ready acknowledgement is issued for it. + +## Discovery ownership wait + +The Node binding registers a Workspace-scoped unmount intent under Sandbox +`ownership.lock`, then drops Sandbox ownership before waiting for the daemon. +The existing NAPI worker waits synchronously on one control connection: no +new background helper or client process is introduced. The daemon still owns +its helper, reap deadline and export custody. Another Workspace can use another +pooled control connection; Sandbox discovery reads also proceed during the wait. + +`api/binding/src/unmount.rs` owns a private `workspace-.unmount` result +record and an exclusive nonblocking `workspace-.unmount-lock` flock. The +lease spans registration, the lock-free exchange and publication. It is explicitly +unlocked on drop, including when another fork briefly inherited its descriptor. +Lock order is Sandbox guard → nonblocking Workspace lease at admission; on +completion it publishes the known result using only its Workspace lease, then +reacquires the Sandbox guard for shared-record cleanup. A cleanup failure adds +`cleanupError` to a known successful response rather than replacing success with +Busy. Close checks the lease even for a published result, so it cannot delete or +reuse the Sandbox directory while that cleanup is in progress. Observers use +nonblocking shared flock probes, so they neither wait on the Workspace lease nor +mistake another observer for the exclusive operation owner. +The shared discovery readiness deadline is back to 60 seconds; it no longer +needs to cover the 60+5-second helper wait. Other long discovery operations or +control-pool exhaustion can still cause waiting; this is not a global latency +promise for all calls. + +State transitions: + +- No intent → Unmounting (path and start time saved before sending a request). +- Unmounting → Unmounted after known daemon success (including an explicit + Missing answer); shared-record cleanup follows. Repeated unmount reads the + saved result without sending another request. +- Busy/Dirty → no intent, preserving their reversible semantics. +- A control connection failure before the unmount exchange removes the intent; + no operation was sent. Other failure → Retained with the original refusal + saved, never replayed. +- Unmounting with no lease holder → Unknown/Retained, including client crash + before send or after daemon completion. Observation cannot decide whether an + effect occurred. It neither removes the Workspace nor repeats the request. + +The public method signatures and daemon protocol are unchanged. During active +unmount, existing `Busy` refusals carry `detail.state = "Unmounting"` and +`elapsedMs`; owner loss uses existing `Unknown` with `state = "Retained"`. +Workspace token/directory consumers and backend reads check this state before +sending a call; Sandbox close refuses an active intent. A new mount creates a +different Workspace and remains admissible. This detail shape and completed +unmount replay behavior need maintainer alignment before upstream publication. + +Terminal result records remain until Sandbox deletion; they are bounded by +that Sandbox's lifetime rather than silently evicted. Known token lookups use a +direct filename, not a scan of completed records; unknown-path recovery and +Sandbox close scan the saved records. An explicit Sandbox close can take over an +orphaned or terminal Retained unmount while the daemon is still healthy, provided +no unrelated live Workspace remains. It performs the existing Stop/Delete, +never a replay of the uncertain Workspace operation. If the daemon dies, the +existing stopped-Sandbox reclaim path owns physical detach and record removal. +Stop still detaches before daemon termination, and Delete retains its checked +detach fence. A failed result-file write is reported as cleanupError on known +success; durable recovery of that lost result is not claimed. + +## Stop before daemon termination + +[`backend/native/sandbox.rs`](../../crates/layerfs-sandbox/src/backend/native/sandbox.rs) +attempts detach below that Sandbox's own directory before TERM/KILL, while its +NFS server can still answer client writeback. A detach failure is logged and +does not prevent the existing daemon and process-group stop. Stop remains crash +teardown, with no Commit or graceful daemon-wide drain. It does not delete the +Sandbox directory. Delete retains its checked detach before directory removal; +failed detach never authorizes removal through a mount. + +Linux continues using `MNT_DETACH`, deepest mount first; macOS uses one +`umount -f` per mount, deepest first. The macOS force helper likewise gets a +60-second observation and a five-second bounded post-KILL reap. Unreaped children +are retained, and failure includes the exact PID and observation rather than +claiming a successful detach. The mount table is checked after ordinary helper +completion. No global NFS operation or cleanup of another Sandbox is performed. + +## Evidence and limits + +- `layerfs-nfs/tests/helper_deadline.rs` drives the real control Service with + process helpers: descendant-held pipe, verbose Busy answer, stopped helper + crossing the real 60-second deadline and repeated Retained requests. +- `layerfs-sandbox/tests/local_stop_mount.rs` is an opt-in macOS test using an + explicitly supplied disposable Sandbox. It proves Stop removes the mount + before Delete and leaves the directory. It never selects an existing user + Sandbox automatically. The normal suite lists this test as ignored. +- `api/binding/tests/unmount-concurrency.mjs` requires explicitly supplied + disposable paths and binaries. It runs real NFS with a paused helper, checks + another Workspace and discovery read below one second, checks same-Workspace + refusals and cross-process progress, reversible Busy/Dirty, saved failures, + publication while discovery cleanup is blocked, and explicit close takeover + after client-owner crash or retained failure. +- These tests cannot simulate an unkillable kernel U-state process: userspace + cannot ignore SIGKILL. Bounded observation of an unreaped U-state helper is + source-checked; the original long kernel hang has not been reproduced. +- A retained helper is reaped opportunistically on a later helper invocation, + not by a background reaper. Helper cleanup and kernel resource recovery remain + distinct facts. Stop may take the sum of deadlines over several mounts; + there is no promise that a multi-mount stop completes in one helper budget. diff --git a/docs/README.md b/docs/README.md index 793b96abdd..0f8f57cf55 100644 --- a/docs/README.md +++ b/docs/README.md @@ -61,6 +61,7 @@ - [Core concepts](general/concepts.md) - [Sandbox cache design](general/sandbox-cache-design.md): cluster-two ownership, branch sharing, source structure and remaining qualification - [Workspace filesystem view](general/workspace-filesystem-view.md): global Store base, daemon overlay, lazy FUSE read path and ASCII diagrams +- [SQLite in macOS NFS Workspaces](general/sqlite-on-network-filesystems.md): imported WAL databases, local conversion and cache placement - [Workspace queue scheduling](general/workspace-queue-scheduling.md): archived; the shared-database queue and credits removed by #321 - [Optimization guide](general/optimization-guide.md): binding performance-engineering rules - [Optimization handbook](general/optimization-handbook.md): diagnostic groups, existing counters, scaling checks, procedure and the current candidate list diff --git a/docs/general/sqlite-on-network-filesystems.md b/docs/general/sqlite-on-network-filesystems.md new file mode 100644 index 0000000000..b3e90cc1af --- /dev/null +++ b/docs/general/sqlite-on-network-filesystems.md @@ -0,0 +1,72 @@ +# SQLite databases in macOS NFS Workspaces + +> **Status:** Current general guide. + +A SQLite database imported from a local filesystem can remain in WAL mode even +when its `-wal` and `-shm` files are absent. With macOS's system SQLite and its +default VFS, querying that database in an NFS Workspace can fail with +`unable to open database file` (`SQLITE_CANTOPEN`, SQLite result code 14). +Opening the connection alone can succeed; the first query exposes the failure. +This concerns application databases inside the Workspace. The LayerFS Store +stays on its supported local filesystem and keeps its configured profile. + +## Cause and scope + +SQLite's macOS automatic locking selection uses an NFS-specific I/O method table +for an `nfs` filesystem. That table has version 1 and no `xShmMap` callback, so it +cannot provide the shared-memory WAL index through the default route. This is a +SQLite VFS policy; a null callback does not establish a failed kernel `mmap` +system call. Result code 14 is not POSIX errno 14. + +A diagnostic on macOS 27.2 with system SQLite 3.54.0 and the native LayerFS NFSv3 +backend observed: + +| Application database | Local APFS | Imported or created in NFS Workspace | +| --- | --- | --- | +| Closed WAL database, querying a saved row | Success | `SQLITE_CANTOPEN` (14) | +| WAL database converted to DELETE before import | Success | Same saved row returned | +| New database requesting `journal_mode=WAL` | Returns `wal` | Returns `delete`; simple read/write succeeds | + +The LayerFS mount already used `locallocks`. Changing an NFS server's handling +of `-shm` files does not provide the callback disabled by the client's VFS. +SQLite builds or explicitly selected VFS implementations can behave differently; +this observation does not qualify every SQLite application or concurrent workload. +A separate macOS system-nfsd export was not tested because starting it required +administrator authentication unavailable to the diagnostic session. + +## Preparing application databases + +For a database the application permits in rollback-journal mode, stop every +process using it, keep a recoverable backup, then use SQLite on the original +local filesystem before import: + +```sh +sqlite3 -bail /absolute/local/path/application.db \ + 'PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE;' +``` + +Proceed only after the checkpoint reports no busy readers/writers and the mode +change returns `delete`. Keep the application stopped until import finishes. +The mode change persists in the database header. Do not edit that header by hand +or delete a `-wal` file: committed data may still be in the WAL. Copying an active +database and its sidecars separately does not guarantee a consistent snapshot. + +For disposable caches, use the application's supported cache reset while it is +stopped, or configure its cache on a local APFS directory outside the mount. +The application may request WAL again on startup; check its actual returned mode +and its own requirements. Simple DELETE-mode success does not establish general +network-filesystem database safety or multi-host lock correctness. + +Import preserves application bytes and does not convert databases automatically. +Checking only sidecar names misses closed WAL databases. A future advisory check +could inspect the SQLite signature and header bytes 18/19 (value 2 denotes WAL +read/write format), without opening or modifying application databases. It would +need an explicit diagnostic surface and a bounded read strategy; it must not +exclude files, rewrite content or claim that a live import is consistent. + +## References + +- [SQLite WAL overview and network-filesystem restrictions](https://www.sqlite.org/wal.html) +- [SQLite Unix VFS source (`nfsIoMethods` and `autolockIoFinderImpl`)](https://www.sqlite.org/src/doc/trunk/src/os_unix.c) +- [SQLite database header format](https://www.sqlite.org/fileformat2.html#the_database_header) +- [SQLite result codes](https://www.sqlite.org/rescode.html#cantopen) diff --git a/engine/crates/layerfs-api/PROVIDERS.md b/engine/crates/layerfs-api/PROVIDERS.md new file mode 100644 index 0000000000..16bd8b3cf9 --- /dev/null +++ b/engine/crates/layerfs-api/PROVIDERS.md @@ -0,0 +1,13 @@ +# Sandbox runtimes in the SDK + +Status: Current general guide. Native macOS is implemented and host-verified; Linux native is implemented, pending Linux host qualification; the Docker runtime is the default. `layerfs-sandbox` selects between them with the concrete `Runtime` enum; there are no provider traits. + +`layerfs-sandbox::backend` exposes `Runtime` (a deployment's choice: `Docker`, `MacosNative`, `LinuxNative`, `Windows`) and `Sandbox` (the launched owner of one Sandbox). `layerfs_sandbox::{Runtime, Sandbox}` re-export both; `layerfs_sandbox::provider` keeps the inputs and records they share. `ExecRequest` lives in the shared types layer; its Docker re-export remains compatible. `SandboxId` aliases the existing `ContainerId`, preserving encoded identities and records. The runtime layer has no daemon control dependency. + +`Runtime::MacosNative(..)` and `Runtime::LinuxNative(..)` wrap `NativeMacosProvider::new(root)` and `NativeLinuxProvider::new(root)`, the two aliases of the same native implementation whose instances share the existing local runtime owner. Each refuses the wrong platform before root access. macOS uses Darwin process facts, the `mount` table and `umount -f`; Linux uses the retained #402 `/proc` PID/start/boot checks, process groups and lazy `MNT_DETACH`. Linux requires a root daemon and a root directory traversable by command users; ordinary commands keep #402's nonroot identity (Node selects 501:20) and fixed container PATH. macOS keeps the caller identity, inherited PATH and loopback NFS. The provider explicitly selects `LAYERFS_KERNEL=fuse` on Linux and `nfs` on macOS and rejects conflicting launch overrides. Launch preserves the original single-attempt process lifecycle, native identity, inherited PATH, NFS environment and partial owner on failure. `inspect` reports exact process liveness; unknown exit/storage facts stay absent. `Runtime::Windows` refuses every operation with `RuntimeError::Unsupported` before any effect; no Windows host runtime exists. Records remain the Node binding's responsibility; runtimes neither scan directories nor infer project ownership. B8 list aggregation and doctor remain future work. + +`SandboxApi::with_runtime(Arc)` and `create_provider_file(ProviderCreate, store, manifest)` (or `create(ProviderCreate, project)`) launch through the deployment's runtime and return `ProviderSandbox`, whose `runtime` field is the `Sandbox` handle. Both Docker (via `Runtime::Docker(DockerProvider)`) and native runtimes route through this single unified API path. `WorkspaceApi::exec` delegates ordinary execution through `sandbox.runtime.run(...)` with streaming stdout/stderr buffers, and `api.close` drains and removes instances via `sandbox.runtime.stop(1)` and `sandbox.runtime.delete()`. Before launch, API validates authentication identities, selected Store, command identity and local resource ownership: overlay and mount home must belong to the instance, Store must live outside it, and native listen uses dynamic loopback. API encodes daemon configuration, then uses the shared TCP/Noise/Hello and Store-install pipeline in `sandbox/connection.rs`. Failures retain their phase, original cause, runtime, control, observed status, installation acknowledgement and handshake work. No automatic retry or cleanup occurs in API. + +`layerfs-node` builds a `Runtime` from `deploy.sandbox` (`docker` by default, `native` on Linux and macOS), keeps the launched `Sandbox` in its Sandbox state, and reads records, image, scratch directory and background-process tracking through it. Legacy deprecated forwarders (`SandboxApi::new(Docker)`, `SandboxCreate`, `ManagedSandbox`, `from_managed`, `internal::split/exec/control_managed`) remain removed. + +The Store remains explicitly Disposable/WAL/OFF. Native macOS keeps #427's direct Store access without a worker; this refactor does not change persistence authorization or filesystem semantics. diff --git a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs index c2e8f73671..cf2fd9aeed 100644 --- a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs +++ b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs @@ -4,8 +4,8 @@ //! registered with the daemon, read the complete root and mutate it through //! plain syscalls. use layerfs_api::{ - InitRequest, ManagedSandbox, MountedWorkspace, OperationCause, ProjectApi, SandboxApi, - SandboxCreate, WorkspaceApi, + InitRequest, MountedWorkspace, OperationCause, ProjectApi, ProviderCreate, ProviderSandbox, + SandboxApi, WorkspaceApi, }; use layerfs_bridge::{ control::{ControlCode, DaemonPhase, MountPhase, Reply, Request}, @@ -14,7 +14,8 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ @@ -22,6 +23,7 @@ use std::{ fs, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, + sync::Arc, time::{Duration, Instant}, }; @@ -100,59 +102,46 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -type Running = ( - layerfs_sandbox::backend::docker::RuntimeOutput, - layerfs_sandbox::backend::docker::ExecHandle, -); enum Directory<'a> { Mounted(&'a MountedWorkspace), Kernel(&'a str), } -/// Mounted commands use SDK exec; outside-mount commands retain ordinary Sandbox execution. -fn start( - owner: &ManagedSandbox, +fn run_dir( + runtime: &layerfs_sandbox::Sandbox, + identity: layerfs_sandbox::CommandIdentity, script: &str, - directory: Directory<'_>, -) -> Result> { - let arguments = vec!["bash".into(), "-c".into(), script.into()]; - let created = match directory { - Directory::Mounted(mounted) => WorkspaceApi::exec(owner, mounted, arguments, vec![], false), - Directory::Kernel(directory) => layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - arguments, - vec![], - directory.into(), - false, - ), - } - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - Ok((output, handle)) -} -fn finish((output, handle): Running) -> Result, Box> { - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - output - .copy_to(&mut stdout, &mut stderr) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - if inspected.known_root_exit() != Some(0) { + directory: &str, +) -> Result, String> { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = runtime + .run( + &layerfs_sandbox::ExecRequest { + identity, + arguments: vec!["bash".into(), "-c".into(), script.into()], + environment: vec![], + directory: directory.into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("exec {f:?}"))?; + if exit.code != 0 { return Err(format!( - "command {inspected:?}: {} {}", + "command exit {exit:?}: {} {}", String::from_utf8_lossy(&stdout), String::from_utf8_lossy(&stderr) - ) - .into()); + )); } Ok(stdout) } -fn run( - owner: &ManagedSandbox, - script: &str, - directory: Directory<'_>, -) -> Result, Box> { - finish(start(owner, script, directory)?) +fn run(owner: &ProviderSandbox, script: &str, directory: Directory<'_>) -> Result, String> { + let dir = match directory { + Directory::Mounted(mounted) => &mounted.ready.directory, + Directory::Kernel(dir) => dir, + }; + run_dir(owner.runtime.as_ref(), owner.command_identity, script, dir) } fn main() -> Result<(), Box> { let began = Instant::now(); @@ -191,22 +180,30 @@ fn main() -> Result<(), Box> { fs::remove_dir_all(directory.join("source"))?; let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; - let api = SandboxApi::new(Docker::new(&args[1], Duration::from_secs(3))?); + let docker = Docker::new(&args[1], Duration::from_secs(3))?; + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); let mut owner = api .create( - SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "kernel-mount-test".into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup: setup(daemon.private, keys.public), - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }, &project, ) @@ -323,45 +320,54 @@ fn main() -> Result<(), Box> { // Reversible normal unmount: a running command's working directory keeps // the kernel busy. The filesystem neither kills nor waits for the command. - let holder = start( - &owner, - "sleep 1.2", - Directory::Kernel(&format!("{root}/node_modules")), - )?; - run( - &owner, - &format!( - "until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{root}/node_modules'; do sleep 0.02; done" - ), - Directory::Kernel("/"), - )?; - let busy = WorkspaceApi::new(&mut owner) - .unmount(mounted.bound.token, false) - .unwrap_err(); - match &busy.cause { - OperationCause::Remote(refusal) => { - assert_eq!( - (refusal.code, refusal.phase.as_str()), - (ControlCode::Busy, "unmount:kernel") - ); + let node_modules_dir = format!("{root}/node_modules"); + let identity = owner.command_identity; + let (runtime, control) = layerfs_api::internal::provider_parts(&mut owner); + std::thread::scope(|s| { + let holder = s.spawn(|| run_dir(runtime, identity, "sleep 1.2", &node_modules_dir)); + run_dir( + runtime, + identity, + &format!("until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{node_modules_dir}'; do sleep 0.02; done"), + "/", + ) + .unwrap(); + let busy = WorkspaceApi::from_control(control) + .unmount(mounted.bound.token, false) + .unwrap_err(); + match &busy.cause { + OperationCause::Remote(refusal) => { + assert_eq!( + (refusal.code, refusal.phase.as_str()), + (ControlCode::Busy, "unmount:kernel") + ); + } + other => panic!("{other:?}"), } - other => panic!("{other:?}"), - } - let status = WorkspaceApi::new(&mut owner).status(mounted.bound.token)?; - let kernel = status.kernel.expect("kernel status block"); - assert_eq!(kernel.phase, MountPhase::Ready); - assert_eq!( - kernel.ready.as_ref(), - Some(&mounted.ready), - "same connection" - ); - assert_eq!( - run(&owner, "cat .git/index", Directory::Mounted(&mounted))?, - FILES[1].1 - ); - finish(holder)?; - println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); - mark("busy"); + let status = WorkspaceApi::from_control(control) + .status(mounted.bound.token) + .unwrap(); + let kernel = status.kernel.expect("kernel status block"); + assert_eq!(kernel.phase, MountPhase::Ready); + assert_eq!( + kernel.ready.as_ref(), + Some(&mounted.ready), + "same connection" + ); + assert_eq!( + run_dir( + runtime, + identity, + "cat .git/index", + &mounted.ready.directory + ) + .unwrap(), + FILES[1].1 + ); + holder.join().unwrap().unwrap(); + println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); + mark("busy"); + }); // Terminal unmount: detach, loop joins, drain, revocation and Close. WorkspaceApi::new(&mut owner) diff --git a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs index f54c059ff8..f6958361ab 100644 --- a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs +++ b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs @@ -1,6 +1,6 @@ //! Positive-path host Init -> protected owned Sandbox -> direct shared Store control proof. //! Typed failure custody is covered separately by public adapter tests; this example reports failures in stderr. -use layerfs_api::{InitRequest, ManagedSandbox, ProjectApi, SandboxApi, SandboxCreate}; +use layerfs_api::{InitRequest, ProjectApi, ProviderCreate, ProviderSandbox, SandboxApi}; use layerfs_bridge::{ control::{DaemonPhase, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, @@ -8,13 +8,15 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ fs, io::Write, path::PathBuf, + sync::Arc, time::{Duration, Instant}, }; fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { @@ -39,25 +41,26 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { - let command="set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; - let created = layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - vec!["bash".into(), "-c".into(), command.into()], - vec![], - "/".into(), - false, - ) - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - let progress = output - .copy_to(&mut std::io::stdout(), &mut std::io::stderr()) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - println!("ORDINARY_RESULT {:?} {:?}", progress, inspected); - assert_eq!(inspected.known_root_exit(), Some(0)); +fn ordinary(owner: &ProviderSandbox) -> Result<(), Box> { + let command = "set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; + let mut stdout = std::io::stdout(); + let mut stderr = std::io::stderr(); + let exit = owner + .runtime + .run( + &layerfs_sandbox::ExecRequest { + identity: layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), command.into()], + environment: vec![], + directory: "/".into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("run {f:?}"))?; + println!("ORDINARY_RESULT {:?}", exit); + assert_eq!(exit.code, 0); Ok(()) } fn main() -> Result<(), Box> { @@ -96,23 +99,30 @@ fn main() -> Result<(), Box> { let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; let docker = Docker::new(&args[1], Duration::from_secs(3))?; - let api = SandboxApi::new(docker); - let make = |setup: DaemonSetup| SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); + let make = |name: &str, setup: DaemonSetup| ProviderCreate { + launch: LaunchRequest { + name: name.into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup, - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }; let mut first = api - .create(make(setup(daemon.private, keys.public)), &project) + .create(make("first", setup(daemon.private, keys.public)), &project) .map_err(|f| format!("first {f:?}"))?; println!( "OWNED_FIRST_CONTAINER {} phase={:?}", @@ -131,7 +141,7 @@ fn main() -> Result<(), Box> { let mut second_setup = setup(daemon.private, keys.public); second_setup.existing_store = Some(installed.manifest.clone()); let second = api - .create(make(second_setup), &project) + .create(make("second", second_setup), &project) .map_err(|f| format!("second {f:?}"))?; println!( "OWNED_SECOND_CONTAINER {} phase={:?}", @@ -161,7 +171,7 @@ fn main() -> Result<(), Box> { let mut reopened_setup = setup(daemon.private, keys.public); reopened_setup.existing_store = Some(installed.manifest.clone()); let mut reopened = api - .create(make(reopened_setup), &project) + .create(make("reopened", reopened_setup), &project) .map_err(|f| format!("reopen {f:?}"))?; println!( "OWNED_REOPEN_CONTAINER {} phase={:?}", diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 7bffa705c0..108b2f0251 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -98,59 +98,40 @@ pub fn into_connection(control: Control) -> Connection { /// Original authenticated control owner, without a reconnect or reset. #[cfg(unix)] -pub fn control(sandbox: &mut crate::ManagedSandbox) -> &mut Control { +pub fn control(sandbox: &mut crate::sandbox::ProviderSandbox) -> &mut Control { &mut sandbox.control } -/// The runtime, control owner and command identity of one Sandbox, taken -/// apart for an owner that serves several Workspaces at once: commands run on -/// the runtime while the control owner answers one request at a time. +/// Transfers provider runtime, original channel and admitted command identity. #[cfg(unix)] -pub fn split( - sandbox: crate::ManagedSandbox, +pub fn split_provider( + sandbox: crate::sandbox::ProviderSandbox, ) -> ( - layerfs_sandbox::backend::docker::Sandbox, + Box, Control, layerfs_sandbox::CommandIdentity, ) { (sandbox.runtime, sandbox.control, sandbox.command_identity) } +/// Borrows the same provider instance and original channel for concurrent execution. +#[cfg(unix)] +pub fn provider_parts( + sandbox: &mut crate::sandbox::ProviderSandbox, +) -> (&layerfs_sandbox::Sandbox, &mut Control) { + (sandbox.runtime.as_ref(), &mut sandbox.control) +} /// Original one-attempt startup used by existing-Store deployments and lifecycle proofs. #[cfg(unix)] pub fn start( api: &crate::SandboxApi, - request: crate::SandboxCreate, -) -> Result> { - crate::sandbox::create::create(&api.docker, request) + request: crate::sandbox::ProviderCreate, +) -> Result> { + api.start(request) } /// Existing bounded startup observation for control-channel qualification. #[cfg(unix)] pub fn sandbox_status( - sandbox: &mut crate::ManagedSandbox, + sandbox: &mut crate::sandbox::ProviderSandbox, wait_for_store: bool, ) -> Result> { sandbox.status(wait_for_store) } - -/// Disjoint original runtime/control borrows for concurrent Workspace orchestration. -#[cfg(unix)] -pub fn parts( - sandbox: &mut crate::ManagedSandbox, -) -> (&layerfs_sandbox::backend::docker::Sandbox, &mut Control) { - (&sandbox.runtime, &mut sandbox.control) -} - -/// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. -#[cfg(unix)] -pub fn exec( - sandbox: &layerfs_sandbox::backend::docker::Sandbox, - identity: layerfs_sandbox::CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result< - layerfs_sandbox::backend::docker::ExecCreated, - Box, -> { - crate::sandbox::exec::launch(sandbox, identity, arguments, environment, directory, stdin) -} diff --git a/engine/crates/layerfs-api/src/lib.rs b/engine/crates/layerfs-api/src/lib.rs index e652d32640..1641980cd8 100644 --- a/engine/crates/layerfs-api/src/lib.rs +++ b/engine/crates/layerfs-api/src/lib.rs @@ -27,7 +27,7 @@ pub use layerfs_project::store::{ pub use layerfs_project::{InitCounters, InitCounts, InitStep, ProjectError, SourceRefusal}; #[cfg(unix)] pub use sandbox::{ - connect_address, ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, + connect_address, ProviderCreate, ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, }; pub use sandbox::{BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi}; diff --git a/engine/crates/layerfs-api/src/sandbox/close.rs b/engine/crates/layerfs-api/src/sandbox/close.rs index e0a57ec08a..5f55414a9e 100644 --- a/engine/crates/layerfs-api/src/sandbox/close.rs +++ b/engine/crates/layerfs-api/src/sandbox/close.rs @@ -1,10 +1,10 @@ //! Explicit stop then delete of the exact acknowledged Sandbox runtime owner. -use super::{ManagedSandbox, SandboxApi, SandboxCause, SandboxFailure, SandboxPhase}; +use super::{ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase}; impl SandboxApi { /// Stops once, then deletes once after acknowledged stop. No Workspace drain is inferred. /// A failed attempt retains the original runtime, control and deployment inputs; no replay. - pub fn close(&self, mut sandbox: ManagedSandbox) -> Result<(), Box> { + pub fn close(&self, mut sandbox: ProviderSandbox) -> Result<(), Box> { if let Err(cause) = sandbox.runtime.stop(1) { return Err(failure( sandbox, @@ -23,14 +23,14 @@ impl SandboxApi { } } fn failure( - sandbox: ManagedSandbox, + sandbox: ProviderSandbox, phase: SandboxPhase, cause: SandboxCause, -) -> Box { - Box::new(SandboxFailure { +) -> Box { + Box::new(ProviderFailure { request: sandbox.request, phase, - sandbox: Some(sandbox.runtime), + runtime: Some(sandbox.runtime), control: Some(sandbox.control), handshake: sandbox.handshake, observed: Some(sandbox.startup), diff --git a/engine/crates/layerfs-api/src/sandbox/connection.rs b/engine/crates/layerfs-api/src/sandbox/connection.rs new file mode 100644 index 0000000000..ba3a75df2d --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/connection.rs @@ -0,0 +1,108 @@ +//! One shared TCP/Noise/Hello pipeline, retaining progress through every failure. +use super::create::{hello, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest}, + duplex::{initiate_observed, ChannelWork}, +}; +use std::{ + net::{SocketAddr, TcpStream}, + time::Duration, +}; + +pub(super) struct Startup { + pub phase: SandboxPhase, + pub control: Option, + pub handshake: ChannelWork, + pub observed: Option, +} +impl Startup { + pub fn new() -> Self { + Self { + phase: SandboxPhase::Validate, + control: None, + handshake: ChannelWork::default(), + observed: None, + } + } + pub fn connect( + &mut self, + endpoint: SocketAddr, + host: Option<&str>, + wait: Duration, + secret: &[u8; 32], + peer: [u8; 32], + existing: bool, + ) -> Result { + self.phase = SandboxPhase::Connect; + let address = super::create::connect_address(endpoint, host).map_err(SandboxCause::Io)?; + let stream = TcpStream::connect_timeout(&address, wait).map_err(SandboxCause::Io)?; + stream + .set_read_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + stream + .set_write_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + self.phase = SandboxPhase::Authenticate; + let start = initiate_observed(stream, secret, peer); + self.handshake = start.work; + self.control = Some(Control::new(start.result.map_err(SandboxCause::Channel)?)); + self.phase = SandboxPhase::Policy; + self.control + .as_ref() + .expect("authenticated") + .clear_io_waits() + .map_err(SandboxCause::Channel)?; + self.phase = SandboxPhase::Hello; + let status = hello( + self.control.as_mut().expect("authenticated"), + HelloRequest { + expected_instance: None, + wait_for_store: existing, + }, + ) + .map_err(SandboxCause::Control)?; + self.observed = Some(status.clone()); + let expected = if existing { + DaemonPhase::ControlReady + } else { + DaemonPhase::InstallPending + }; + if status.phase != expected { + return Err(SandboxCause::Setup(ControlError("original startup phase"))); + } + Ok(status) + } +} +/// Installation and its Ready observation are identical for every runtime. +pub(super) fn install( + control: &mut Control, + startup: &mut DaemonStatus, + installed: &mut Option, + phase: &mut SandboxPhase, + store: &layerfs_persistence::SealedStore, + manifest: &layerfs_bridge::provision::StoreManifest, +) -> Result<(), SandboxCause> { + *phase = SandboxPhase::Install; + *installed = Some( + control + .install_file(store, manifest) + .map_err(SandboxCause::Install)?, + ); + *phase = SandboxPhase::Hello; + let status = hello( + control, + HelloRequest { + expected_instance: Some(startup.instance), + wait_for_store: true, + }, + ) + .map_err(SandboxCause::Control)?; + *startup = status; + if startup.phase != DaemonPhase::ControlReady { + return Err(SandboxCause::Setup(ControlError( + "original installed startup phase", + ))); + } + Ok(()) +} diff --git a/engine/crates/layerfs-api/src/sandbox/create.rs b/engine/crates/layerfs-api/src/sandbox/create.rs index 2146783582..61837c6693 100644 --- a/engine/crates/layerfs-api/src/sandbox/create.rs +++ b/engine/crates/layerfs-api/src/sandbox/create.rs @@ -1,48 +1,10 @@ //! Ordinary Sandbox startup, original deployment inputs and failure custody. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::{ - control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest, Reply, Request}, - daemon_setup::DaemonSetup, - duplex::{initiate_observed, public_key, ChannelError, ChannelWork}, -}; -use layerfs_sandbox::backend::docker::{ - Docker, ListenerFailure, Sandbox, SandboxCreateFailure, SandboxRequest, UploadFailure, -}; -use std::{fmt, fs::File, io, net::TcpStream, path::PathBuf, time::Duration}; -/// One actual stopped-container deployment, private archive and authenticated startup. -#[derive(Clone)] -pub struct SandboxCreate { - /// Exact immutable image and borrowed shared named volume. - pub deployment: SandboxRequest, - /// Explicit private daemon resources and Disposable install/open selection. - pub setup: DaemonSetup, - /// Sealed Linux executable supplied by the caller; streamed once, never read as Store data. - pub executable: PathBuf, - /// Controller Noise secret; redacted from Debug and never sent in env/argv. - pub controller_private: [u8; 32], - /// Expected actual daemon identity, checked against private deployment config. - pub daemon_peer: [u8; 32], - /// Explicit listener event wait; does not apply to ordinary commands. - pub startup_wait: Duration, - /// Host to dial for the daemon's loopback-published port, when this - /// process does not share the Docker host's loopback (a client in a - /// container: `host.docker.internal`). `None` dials the observed - /// 127.0.0.1 endpoint. The publish and its check are unchanged. - pub connect_host: Option, -} -impl fmt::Debug for SandboxCreate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxCreate") - .field("deployment", &self.deployment) - .field("setup", &self.setup) - .field("executable", &self.executable) - .field("controller_private", &"[redacted]") - .field("daemon_peer", &self.daemon_peer) - .field("startup_wait", &self.startup_wait) - .field("connect_host", &self.connect_host) - .finish() - } -} +use layerfs_bridge::control::{ControlError, DaemonStatus, HelloRequest, Reply, Request}; +use layerfs_bridge::duplex::ChannelError; +use layerfs_sandbox::Runtime; +use std::{io, sync::Arc}; + /// The address to dial for a daemon `endpoint` observed on the Docker host's /// loopback: the endpoint itself, or the port on `host` (one resolution, its /// first address; nothing is tried twice). @@ -62,6 +24,7 @@ pub fn connect_address( ) }) } + /// Concrete boundary reached by one original lifecycle operation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SandboxPhase { @@ -94,6 +57,7 @@ pub enum SandboxPhase { /// One delete attempt after acknowledged stop. Delete, } + /// Exact original failure, independent of owned resource disposition. #[derive(Debug)] pub enum SandboxCause { @@ -101,14 +65,10 @@ pub enum SandboxCause { Io(io::Error), /// Local config or correlated startup validation. Setup(ControlError), - /// Actual original runtime Create including partial decoded ID. - Create(Box), - /// Actual incremental archive extraction failure and work. - Upload(Box), - /// Original listener wait/progress/fence failure. - Listener(Box), /// Original Engine transfer, no replay. Runtime(Box), + /// Original provider launch cause; partial runtime ownership is retained separately. + Provider(Box), /// Original authentication failure; handshake work remains in the receipt. Channel(ChannelError), /// Original correlated daemon control exchange/refusal. @@ -116,181 +76,13 @@ pub enum SandboxCause { /// Original streaming installation failure and exact publication knowledge. Install(Box), } -/// Caller retains exact acknowledged owner, connection and original deployment inputs. -pub struct SandboxFailure { - /// Original inputs with secrets retained privately. - pub request: SandboxCreate, - /// Original reached boundary. - pub phase: SandboxPhase, - /// Exact acknowledged owned container; no automatically adopted partial ID. - pub sandbox: Option, - /// Original authenticated control owner, if established. - pub control: Option, - /// Actual original handshake work, including failure. - pub handshake: ChannelWork, - /// Original correlated Hello, even if its phase failed selection validation. - pub observed: Option, - /// Installation acknowledgement, retained if the following Ready observation fails. - pub installed: Option, - /// Exact original cause. - pub cause: SandboxCause, -} -impl fmt::Debug for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxFailure") - .field("request", &self.request) - .field("phase", &self.phase) - .field("sandbox", &self.sandbox) - .field("has_control", &self.control.is_some()) - .field("handshake", &self.handshake) - .field("observed", &self.observed) - .field("cause", &self.cause) - .finish() - } -} -impl fmt::Display for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) - } -} -impl std::error::Error for SandboxFailure {} + /// Actual local Engine and authenticated daemon lifecycle organization. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SandboxApi { - pub(crate) docker: Docker, -} -/// Exact acknowledged container and authenticated control owner. -pub struct ManagedSandbox { - /// Owned runtime container; explicit stop/delete only. - pub runtime: Sandbox, - /// Existing sequential control owner; correlation and failed state are never reset. - pub(crate) control: Control, - /// Original acknowledged installation, when this create installed the Project. - pub installed: Option, - pub(crate) request: SandboxCreate, - /// Original correlated startup status; this is not kernel Workspace Ready. - pub startup: DaemonStatus, - /// Actual original duplex handshake work. - pub handshake: ChannelWork, - pub(crate) command_identity: layerfs_sandbox::CommandIdentity, -} -impl SandboxApi { - /// Uses one explicit local Engine endpoint and metadata wait policy. - pub fn new(docker: Docker) -> Self { - Self { docker } - } - /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. - pub fn create( - &self, - request: SandboxCreate, - project: &crate::SealedProject, - ) -> Result> { - self.create_file(request, &project.store, &project.manifest) - } - /// Creates a new daemon using exact persisted sealed-file and manifest facts. - pub fn create_file( - &self, - request: SandboxCreate, - store: &layerfs_persistence::SealedStore, - manifest: &layerfs_bridge::provision::StoreManifest, - ) -> Result> { - let existing = request.setup.existing_store.is_some(); - if let Some(selected) = &request.setup.existing_store { - let mut expected = manifest.clone(); - expected.daemon_sqlite = selected.daemon_sqlite.clone(); - if *selected != expected { - return Err(Box::new(SandboxFailure { - request, - phase: SandboxPhase::Validate, - sandbox: None, - control: None, - handshake: ChannelWork::default(), - observed: None, - installed: None, - cause: SandboxCause::Setup(ControlError("selected existing Project")), - })); - } - } - let mut owner = create(&self.docker, request)?; - if existing { - // The original startup already selected and waited for this Store. - // Installing again would change the existing one-attempt lifecycle. - return Ok(owner); - } - let installed = owner - .control - .install_file(store, manifest) - .map_err(|cause| { - Box::new(SandboxFailure { - request: owner.request.clone(), - phase: SandboxPhase::Install, - sandbox: None, - control: None, - handshake: owner.handshake, - observed: Some(owner.startup.clone()), - installed: None, - cause: SandboxCause::Install(cause), - }) - }); - let installed = match installed { - Ok(installed) => installed, - Err(mut failure) => { - failure.sandbox = Some(owner.runtime); - failure.control = Some(owner.control); - return Err(failure); - } - }; - owner.installed = Some(installed); - match hello( - &mut owner.control, - HelloRequest { - expected_instance: Some(owner.startup.instance), - wait_for_store: true, - }, - ) { - Ok(status) if status.phase == DaemonPhase::ControlReady => { - owner.startup = status; - Ok(owner) - } - Ok(status) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(status), - installed: owner.installed, - cause: SandboxCause::Setup(ControlError("original installed startup phase")), - })), - Err(cause) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(owner.startup), - installed: owner.installed, - cause: SandboxCause::Control(cause), - })), - } - } -} -impl ManagedSandbox { - /// Explicit startup observation/wait on the same incarnation and Control owner. - /// A later observation never resolves an earlier unknown operation. - pub(crate) fn status( - &mut self, - wait_for_store: bool, - ) -> Result> { - hello( - &mut self.control, - HelloRequest { - expected_instance: Some(self.startup.instance), - wait_for_store, - }, - ) - } + pub(crate) runtime: Arc, } + pub(super) fn hello( control: &mut Control, request: HelloRequest, @@ -301,118 +93,3 @@ pub(super) fn hello( reply => Err(OperationFailure::unexpected(request, reply)), } } -pub(crate) fn create( - docker: &Docker, - request: SandboxCreate, -) -> Result> { - let mut phase = SandboxPhase::Validate; - let mut sandbox = None; - let mut control = None; - let mut handshake = ChannelWork::default(); - let mut observed = None; - let result = (|| { - let config = request.setup.encode().map_err(SandboxCause::Setup)?; - if request.setup.listen != format!("0.0.0.0:{}", request.deployment.port) - || request.setup.store != "/layerfs-store/global/store.sqlite" - || request.setup.overlay != "/layerfs-local/overlay/overlay.sqlite" - || request.setup.mounts != "/workspaces" - || request.startup_wait.is_zero() - || public_key(&request.controller_private).map_err(SandboxCause::Channel)? - != request.setup.control_peer - || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? - != request.daemon_peer - { - return Err(SandboxCause::Setup(ControlError( - "concrete deployment topology/identity", - ))); - } - phase = SandboxPhase::Executable; - let mut file = File::open(&request.executable).map_err(SandboxCause::Io)?; - phase = SandboxPhase::Create; - sandbox = Some( - docker - .create_sandbox(request.deployment.clone()) - .map_err(SandboxCause::Create)?, - ); - let owner = sandbox.as_mut().expect("acknowledged container"); - phase = SandboxPhase::Upload; - owner - .upload(&mut file, &config) - .map_err(SandboxCause::Upload)?; - phase = SandboxPhase::Start; - owner.start().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Listener; - owner - .wait_listener(request.startup_wait) - .map_err(SandboxCause::Listener)?; - phase = SandboxPhase::Endpoint; - let endpoint = owner.endpoint().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Connect; - let address = - connect_address(endpoint, request.connect_host.as_deref()).map_err(SandboxCause::Io)?; - let stream = - TcpStream::connect_timeout(&address, request.startup_wait).map_err(SandboxCause::Io)?; - stream - .set_read_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - stream - .set_write_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - phase = SandboxPhase::Authenticate; - let start = initiate_observed(stream, &request.controller_private, request.daemon_peer); - handshake = start.work; - let connection = start.result.map_err(SandboxCause::Channel)?; - // The startup handshake waits are cleared before control/product operations. - // Bridge owns cloned duplex directions; clear policy through that actual owner. - control = Some(Control::new(connection)); - phase = SandboxPhase::Policy; - control - .as_ref() - .expect("authenticated") - .clear_io_waits() - .map_err(SandboxCause::Channel)?; - phase = SandboxPhase::Hello; - let status = hello( - control.as_mut().expect("authenticated"), - HelloRequest { - expected_instance: None, - wait_for_store: request.setup.existing_store.is_some(), - }, - ) - .map_err(SandboxCause::Control)?; - let expected = if request.setup.existing_store.is_some() { - DaemonPhase::ControlReady - } else { - DaemonPhase::InstallPending - }; - observed = Some(status.clone()); - if status.phase != expected { - return Err(SandboxCause::Setup(ControlError("original startup phase"))); - } - Ok(status) - })(); - match result { - Ok(startup) => Ok(ManagedSandbox { - request: request.clone(), - installed: None, - runtime: sandbox.take().expect("owned"), - control: control.take().expect("authenticated"), - startup, - handshake, - command_identity: layerfs_sandbox::CommandIdentity { - uid: request.setup.command_uid, - gid: request.setup.command_gid, - }, - }), - Err(cause) => Err(Box::new(SandboxFailure { - request, - phase, - sandbox, - control, - handshake, - observed, - installed: None, - cause, - })), - } -} diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index c0db8c8e5b..40e0a5e449 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -1,44 +1,30 @@ //! Optional ordinary Sandbox launch at the acknowledged mounted directory. -use crate::{ManagedSandbox, MountedWorkspace, WorkspaceApi}; -use layerfs_sandbox::{ - backend::docker::{ExecCreateFailure, ExecCreated, ExecRequest, Sandbox}, - CommandIdentity, -}; +use super::{MountedWorkspace, ProviderSandbox, WorkspaceApi}; +use layerfs_sandbox::{provider::ExecExit, ExecRequest, WireFailure}; +use std::io::Write; impl WorkspaceApi<'_> { - /// Creates one ordinary command at this mount using the deployment's admitted identity. - /// The returned Sandbox runtime owner owns Start, streams, exit status and cancellation. + /// Runs one ordinary command at this mount using the deployment's admitted identity. + /// The runtime provider owns execution, streams, exit status and cancellation. /// This sends no daemon request, registers no Exec and performs no Commit or unmount. pub fn exec( - sandbox: &ManagedSandbox, + sandbox: &ProviderSandbox, workspace: &MountedWorkspace, arguments: Vec, environment: Vec, - stdin: bool, - ) -> Result> { - launch( - &sandbox.runtime, - sandbox.command_identity, - arguments, - environment, - workspace.ready.directory.clone(), - stdin, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + sandbox.runtime.run( + &ExecRequest { + identity: sandbox.command_identity, + arguments, + environment, + directory: workspace.ready.directory.clone(), + stdin: false, + }, + stdout, + stderr, ) } } -pub(crate) fn launch( - sandbox: &Sandbox, - identity: CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result> { - sandbox.create_exec(ExecRequest { - identity, - arguments, - environment, - directory, - stdin, - }) -} diff --git a/engine/crates/layerfs-api/src/sandbox/mod.rs b/engine/crates/layerfs-api/src/sandbox/mod.rs index 6983a84746..1bf10e271d 100644 --- a/engine/crates/layerfs-api/src/sandbox/mod.rs +++ b/engine/crates/layerfs-api/src/sandbox/mod.rs @@ -1,6 +1,12 @@ //! Actual ordinary Sandbox lifecycle and its Workspaces, with exact logical and kernel readiness. #[cfg(unix)] mod close; +#[cfg(unix)] +mod connection; +#[cfg(unix)] +mod provider; +#[cfg(unix)] +pub use provider::{ProviderCreate, ProviderFailure, ProviderSandbox}; mod commit; #[cfg(unix)] pub(crate) mod create; @@ -12,9 +18,9 @@ mod unmount; pub mod values; pub use commit::CommitReply; #[cfg(unix)] -pub use create::{connect_address, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase}; +pub use create::SandboxApi; #[cfg(unix)] -pub use create::{ManagedSandbox, SandboxApi}; +pub use create::{connect_address, SandboxCause, SandboxPhase}; pub use mount::BoundWorkspace; pub use mount::WorkspaceApi; pub use mount::{MountFailure, MountedWorkspace}; diff --git a/engine/crates/layerfs-api/src/sandbox/mount.rs b/engine/crates/layerfs-api/src/sandbox/mount.rs index 15d2e70aab..4d0414cebc 100644 --- a/engine/crates/layerfs-api/src/sandbox/mount.rs +++ b/engine/crates/layerfs-api/src/sandbox/mount.rs @@ -12,9 +12,14 @@ pub struct WorkspaceApi<'a> { impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. #[cfg(unix)] - pub fn new(sandbox: &'a mut crate::ManagedSandbox) -> Self { + pub fn new(sandbox: &'a mut super::ProviderSandbox) -> Self { + Self::from_control(&mut sandbox.control) + } + /// Borrows an authenticated control channel directly. + #[cfg(unix)] + pub fn from_control(control: &'a mut Control) -> Self { Self { - control: &mut sandbox.control, + control, observation_scope: None, } } diff --git a/engine/crates/layerfs-api/src/sandbox/provider.rs b/engine/crates/layerfs-api/src/sandbox/provider.rs new file mode 100644 index 0000000000..098f66dc8b --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/provider.rs @@ -0,0 +1,373 @@ +//! Runtime startup uses the same authenticated connection and Store installation as Docker. +use super::{connection::Startup, SandboxApi, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonStatus}, + daemon_setup::DaemonSetup, + duplex::{public_key, ChannelWork}, + provision::StoreManifest, +}; +use layerfs_sandbox::{ + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, Sandbox, +}; +use std::{fmt, sync::Arc}; + +/// Runtime launch and private daemon authentication inputs. +#[derive(Clone)] +pub struct ProviderCreate { + /// Runtime launch; configuration is encoded from setup after validation. + pub launch: LaunchRequest, + /// Explicit Disposable daemon topology and Store selection. + pub setup: DaemonSetup, + /// Controller secret; never printed or sent in argv/environment. + pub controller_private: [u8; 32], + /// Expected daemon public identity. + pub daemon_peer: [u8; 32], +} +impl fmt::Debug for ProviderCreate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderCreate") + .field("launch", &self.launch) + .field("setup", &self.setup) + .field("daemon_peer", &self.daemon_peer) + .finish_non_exhaustive() + } +} +/// Exact startup failure and all acknowledged ownership, without automatic cleanup. +pub struct ProviderFailure { + /// Original private request. + pub request: ProviderCreate, + /// Last attempted stage. + pub phase: SandboxPhase, + /// Acknowledged runtime, including a partially launched instance. + pub runtime: Option>, + /// Authenticated channel when established. + pub control: Option, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Last correlated startup observation. + pub observed: Option, + /// Installation acknowledgement even when the following Hello fails. + pub installed: Option, + /// Original deciding cause. + pub cause: SandboxCause, +} +impl fmt::Debug for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderFailure") + .field("phase", &self.phase) + .field("runtime", &self.runtime.as_ref().map(|r| r.identity())) + .field("cause", &self.cause) + .finish_non_exhaustive() + } +} +impl fmt::Display for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) + } +} +impl std::error::Error for ProviderFailure {} +/// Runtime owner and its original authenticated daemon channel. +pub struct ProviderSandbox { + /// Original private launch and daemon configuration inputs. + pub request: ProviderCreate, + /// Explicit runtime lifecycle owner. + pub runtime: Box, + /// Original correlated startup observation. + pub startup: DaemonStatus, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Store installation acknowledgement, if installed by this create. + pub installed: Option, + pub(crate) control: Control, + /// Command identity admitted for this Sandbox deployment. + pub command_identity: CommandIdentity, +} +impl ProviderSandbox { + /// Explicit startup observation/wait on the same incarnation and Control owner. + pub(crate) fn status( + &mut self, + wait_for_store: bool, + ) -> Result> { + super::create::hello( + &mut self.control, + layerfs_bridge::control::HelloRequest { + expected_instance: Some(self.startup.instance), + wait_for_store, + }, + ) + } +} +impl SandboxApi { + /// Uses the deployment's selected runtime. + pub fn with_runtime(runtime: Arc) -> Self { + Self { runtime } + } + /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. + pub fn create( + &self, + request: ProviderCreate, + project: &crate::SealedProject, + ) -> Result> { + self.create_provider_file(request, &project.store, &project.manifest) + } + /// Launches once and authenticates to Hello without Store installation (internal proof only). + pub(crate) fn start( + &self, + request: ProviderCreate, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + ) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed: None, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed: None, + cause, + })), + } + } + /// Launches once, authenticates, selects or installs the Store, then observes Ready. + pub fn create_provider_file( + &self, + request: ProviderCreate, + store: &layerfs_persistence::SealedStore, + manifest: &StoreManifest, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let mut installed = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if request + .setup + .existing_store + .as_ref() + .is_some_and(|m| m.locator != request.setup.store) + || launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + if let Some(selected) = &request.setup.existing_store { + let mut expected = manifest.clone(); + expected.daemon_sqlite = selected.daemon_sqlite.clone(); + if *selected != expected { + return Err(SandboxCause::Setup(ControlError( + "selected existing Project", + ))); + } + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + let mut status = progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + )?; + if request.setup.existing_store.is_none() { + let mut phase = SandboxPhase::Install; + let result = super::connection::install( + progress.control.as_mut().expect("authenticated"), + &mut status, + &mut installed, + &mut phase, + store, + manifest, + ); + progress.phase = phase; + progress.observed = Some(status.clone()); + result?; + } + Ok(status) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed, + cause, + })), + } + } +} diff --git a/engine/crates/layerfs-api/tests/local_install.rs b/engine/crates/layerfs-api/tests/local_install.rs index 2bb5ff0c76..15ff14753a 100644 --- a/engine/crates/layerfs-api/tests/local_install.rs +++ b/engine/crates/layerfs-api/tests/local_install.rs @@ -55,7 +55,7 @@ fn root() -> bool { std::fs::metadata("/proc/self").is_ok_and(|m| m.uid() == 0) } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn import_installs_the_store_in_process() { use std::os::unix::fs::PermissionsExt; @@ -64,6 +64,7 @@ fn import_installs_the_store_in_process() { let destination = directory.join("volume/global/store.sqlite"); let sealed_bytes = std::fs::read(&project.store.path).unwrap(); let outcome = install_local(&project.store.path, &project.manifest, &destination); + #[cfg(target_os = "linux")] if !root() { let failure = outcome.unwrap_err(); assert!(failure.cause.contains("must run as user 0"), "{failure}"); @@ -117,9 +118,9 @@ fn an_installed_manifest_is_refused_before_anything_is_written() { std::fs::remove_dir_all(directory).unwrap(); } -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] -fn local_install_is_unsupported_off_linux() { +fn local_install_is_unsupported_on_other_platforms() { let directory = scratch("host"); let project = sealed(&directory); let destination = directory.join("volume/global/store.sqlite"); diff --git a/engine/crates/layerfs-api/tests/provider_ownership.rs b/engine/crates/layerfs-api/tests/provider_ownership.rs new file mode 100644 index 0000000000..3d57397ab7 --- /dev/null +++ b/engine/crates/layerfs-api/tests/provider_ownership.rs @@ -0,0 +1,143 @@ +//! Invalid native topology must refuse before spawning or touching another instance. +#![cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; +static NEXT: AtomicUsize = AtomicUsize::new(0); +struct ProjectFixture { + directory: PathBuf, + project: layerfs_api::SealedProject, +} +impl ProjectFixture { + fn new() -> Self { + use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId}; + use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; + let directory = std::env::temp_dir().join(format!( + "sdk-existing-project-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&directory).unwrap(); + let source = directory.join("source"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("payload"), b"original complete project").unwrap(); + let request = layerfs_api::InitRequest { + source, + store: PersistenceConfig::sqlite(directory.join("store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + locator: "/layerfs-store/global/store.sqlite".into(), + policy: layerfs_storage::StoragePolicy::frozen_default(), + catalog: HistoryCatalogConfig { + binding_key: b"sdk-existing-project".to_vec(), + incarnation: 1, + cursor_key: [7; 32], + }, + stack: LayerStackId::from_authority([3; 16]), + stack_name: HistoryName::new("project").unwrap(), + branch: BranchId::from_authority([4; 16]), + branch_name: HistoryName::new("main").unwrap(), + scope_seed: [5; 32], + deadline: Instant::now() + Duration::from_secs(10), + }; + let project = layerfs_telemetry::timer::Timing::disabled("sdk.existing.project", |timer| { + layerfs_api::ProjectApi::new().init(request, timer) + }) + .0 + .unwrap(); + Self { directory, project } + } +} +impl Drop for ProjectFixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +#[test] +fn native_resource_paths_are_validated_before_launch() { + use layerfs_api::{sandbox::ProviderCreate, SandboxApi, SandboxPhase}; + use layerfs_bridge::{ + daemon_setup::{DaemonLimits, DaemonSetup}, + duplex, + }; + use layerfs_sandbox::{ + backend::native::NativeMacosProvider, + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, + }; + use std::{ + os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, + }; + let fixture = ProjectFixture::new(); + let root = fixture.directory.join("runtime"); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = NativeMacosProvider::new(&root).unwrap(); + let paths = provider.paths("one").unwrap(); + let metadata = fs::metadata(&root).unwrap(); + let identity = CommandIdentity { + uid: metadata.uid(), + gid: metadata.gid(), + }; + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [32; 32], + control_peer: duplex::public_key(&[31; 32]).unwrap(), + store: fixture.project.store.path.display().to_string(), + overlay: paths.overlay.display().to_string(), + mounts: paths.mounts.display().to_string(), + command_uid: identity.uid, + command_gid: identity.gid, + existing_store: None, + limits: DaemonLimits { + connections: 2, + handshake_ms: 1000, + cache_bytes: 65536, + namespaces: 2, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + }; + let request = ProviderCreate { + setup, + controller_private: [31; 32], + daemon_peer: duplex::public_key(&[32; 32]).unwrap(), + launch: LaunchRequest { + name: "one".into(), + daemon_executable: "/bin/false".into(), + daemon_config: vec![], + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: identity, + environment: vec![], + }, + }, + }; + let api = SandboxApi::with_runtime(Arc::new(Runtime::MacosNative(provider))); + for field in ["overlay", "mounts", "store", "listen"] { + let mut invalid = request.clone(); + match field { + "overlay" => { + invalid.setup.overlay = root.join("other/overlay.sqlite").display().to_string() + } + "mounts" => invalid.setup.mounts = root.join("other/workspaces").display().to_string(), + "store" => { + invalid.setup.store = paths.directory.join("store.sqlite").display().to_string() + } + _ => invalid.setup.listen = "0.0.0.0:30421".into(), + } + let failure = api + .create_provider_file(invalid, &fixture.project.store, &fixture.project.manifest) + .err() + .unwrap(); + assert_eq!(failure.phase, SandboxPhase::Validate); + assert!(failure.runtime.is_none() && failure.control.is_none()); + assert!(!paths.directory.exists()); + } +} diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 64859816e2..507f177de5 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -3,20 +3,26 @@ #[allow(dead_code)] #[path = "../../layerfs-sandbox/tests/support/engine.rs"] mod engine; -use engine::{fixed, frame, Peer}; +use engine::{fixed, frame, upgrade, Peer}; use layerfs_api::{ - ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxPhase, WorkspaceApi, + ProviderCreate, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, WorkspaceApi, }; use layerfs_bridge::{ control::{Answer, Call, DaemonPhase, DaemonStatus, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, duplex, }; +use layerfs_sandbox::backend::docker::DockerProvider; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::{ fs, net::TcpListener, path::PathBuf, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, thread::{self, JoinHandle}, time::{Duration, Instant}, }; @@ -52,7 +58,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { struct Fixture { api: SandboxApi, - owner: Option, + owner: Option, peer: Option, daemon: Option>, binary: PathBuf, @@ -124,7 +130,7 @@ impl Fixture { ]; responses.extend(tail); let (docker, peer) = Peer::new(responses, 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let binary = std::env::temp_dir().join(format!( "sdk-operations-{}-{}", std::process::id(), @@ -137,19 +143,23 @@ impl Fixture { manifest.daemon_sqlite = Some("3.51.0".into()); deployment_setup.existing_store = Some(manifest); } - let request = SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + let request = ProviderCreate { + launch: LaunchRequest { + name: "sdk-operations".into(), + daemon_executable: binary.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: binary.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }; // No Store is constructed or opened. This is the original InstallPending // startup used by component proofs, not a fabricated SDK owner. @@ -201,10 +211,8 @@ fn failed_stop_preserves_original_owner_channel_inputs_and_attempt_without_delet let handshake = owner.handshake; let failure = fixture.api.close(owner).unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Stop); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(!runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -227,10 +235,8 @@ fn failed_delete_retains_acknowledged_stop_and_exact_delete_failure() { .close(fixture.owner.take().unwrap()) .unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Delete); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -299,39 +305,44 @@ fn workspace_exec_returns_original_runtime_owner_at_the_mount_with_no_daemon_reg const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; let mut fixture = Fixture::new(vec![ fixed(201, format!("{{\"Id\":\"{EID}\"}}").as_bytes()), + upgrade(&frame(1, b"")), + fixed( + 200, + format!( + "{{\"ID\":\"{EID}\",\"ContainerID\":\"{CID}\",\"Running\":false,\"ExitCode\":0,\"Pid\":42}}" + ) + .as_bytes(), + ), fixed(204, b""), fixed(204, b""), ]); let before = layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(); - let created = WorkspaceApi::exec( + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec!["SDK_TEST=1".into()], - false, + &mut stdout, + &mut stderr, ) .unwrap(); - assert_eq!(created.identity().0.to_string(), CID); - assert_eq!(created.identity().1.to_string(), EID); - assert_eq!(created.request().directory, "/workspaces/selected-mount"); - assert_eq!( - created.request().identity, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 } - ); + assert_eq!(exit.code, 0); + assert_eq!(exit.pid, Some(42)); assert_eq!( layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(), before ); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); let calls = fixture.calls(); - assert_eq!(calls.len(), 9); + assert_eq!(calls.len(), 11); let request = std::str::from_utf8(&calls[6]).unwrap(); assert!(request.starts_with(&format!("POST /v1.54/containers/{CID}/exec "))); assert!(request.contains(r#""WorkingDir":"/workspaces/selected-mount""#)); assert!(request.contains(r#""User":"501:20""#)); - assert!(!calls - .iter() - .any(|call| call.windows(12).any(|bytes| bytes == b"/exec/start "))); + assert!(calls[7].starts_with(format!("POST /v1.54/exec/{EID}/start ").as_bytes())); + assert!(calls[8].starts_with(format!("GET /v1.54/exec/{EID}/json ").as_bytes())); } #[test] fn workspace_exec_failure_keeps_exact_original_request_and_container() { @@ -340,19 +351,22 @@ fn workspace_exec_failure_keeps_exact_original_request_and_container() { fixed(204, b""), fixed(204, b""), ]); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); let failure = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec![], - true, + &mut stdout, + &mut stderr, ) .unwrap_err(); - assert_eq!(failure.container.to_string(), CID); - assert_eq!(failure.request.directory, "/workspaces/selected-mount"); - assert_eq!(failure.request.arguments, ["/bin/pwd"]); - assert!(failure.request.stdin); - assert!(failure.transfer.attempted); + assert_eq!( + failure.requested_container, + Some(layerfs_sandbox::ContainerId::parse(CID).unwrap()) + ); + assert!(failure.attempted); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); assert_eq!(fixture.calls().len(), 9); } @@ -429,25 +443,29 @@ fn create_matching_existing_project_keeps_one_original_hello_and_does_not_reinst fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_effect() { let project = ProjectFixture::new(); let (docker, peer) = Peer::new(vec![], 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let mut deployment_setup = setup([32; 32], duplex::public_key(&[31; 32]).unwrap()); let mut manifest = project.project.manifest.clone(); manifest.root[0] ^= 1; deployment_setup.existing_store = Some(manifest); let result = api.create( - SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "mismatched-sandbox".into(), + daemon_executable: project.project.store.path.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: project.project.store.path.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }, &project.project, ); @@ -455,6 +473,6 @@ fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_e panic!("mismatched Project must refuse before effects") }; assert_eq!(failure.phase, SandboxPhase::Validate); - assert!(failure.sandbox.is_none() && failure.control.is_none() && failure.installed.is_none()); + assert!(failure.runtime.is_none() && failure.control.is_none() && failure.installed.is_none()); assert!(peer.finish().is_empty()); } diff --git a/engine/crates/layerfs-daemon/Cargo.toml b/engine/crates/layerfs-daemon/Cargo.toml index ea15738653..b53132243f 100644 --- a/engine/crates/layerfs-daemon/Cargo.toml +++ b/engine/crates/layerfs-daemon/Cargo.toml @@ -11,6 +11,8 @@ layerfs-overlay = { path = "../layerfs-overlay" } layerfs-store-service = { path = "../layerfs-store-service" } layerfs-workspace = { path = "../layerfs-workspace" } layerfs-fuse = { path = "../layerfs-fuse" } +# NFS kernel serving: the native route's mount entry beside FUSE. +layerfs-nfs = { path = "../layerfs-nfs" } layerfs-bridge = { path = "../layerfs-bridge" } layerfs-content = { path = "../../../core/crates/layerfs-content" } layerfs-storage = { path = "../../../core/crates/layerfs-storage" } @@ -19,7 +21,7 @@ layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } # Concrete Store opening stays in application bootstrap, outside store/. layerfs-persistence = { path = "../layerfs-persistence" } -nix = { version = "=0.31.3", features = ["fs", "user", "process"] } +nix = { version = "=0.31.3", features = ["fs", "user", "process", "resource"] } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/engine/crates/layerfs-daemon/src/application/config.rs b/engine/crates/layerfs-daemon/src/application/config.rs index e3c404c8a0..a2269fba99 100644 --- a/engine/crates/layerfs-daemon/src/application/config.rs +++ b/engine/crates/layerfs-daemon/src/application/config.rs @@ -29,7 +29,10 @@ pub(super) fn read(path: &Path) -> io::Result { return Err(invalid("private config record size")); } let config = DaemonSetup::decode(&bytes).map_err(|_| invalid("private config fields"))?; - if config.command_uid == uid { + // The NFS route opens no FUSE device and the native route runs its + // commands as the daemon's own user, unconfined (no isolation); every + // other route keeps the two identities apart. + if config.command_uid == uid && !super::filesystem::nfs_route() { return Err(invalid("command and daemon identities must differ")); } // One early reservation restores any value below the refill window; a @@ -108,11 +111,21 @@ fn secure_process() -> io::Result<()> { } Ok(()) } -#[cfg(not(target_os = "linux"))] +/// macOS (the native NFS route): files the daemon creates are its own +/// alone, and it never writes a core file holding its keys or Store pages. +/// There is no dumpable flag; a debugger of the same user is not refused. +#[cfg(target_os = "macos")] +fn secure_process() -> io::Result<()> { + use nix::sys::resource::{setrlimit, Resource}; + nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o077)); + setrlimit(Resource::RLIMIT_CORE, 0, 0).map_err(io::Error::from)?; + Ok(()) +} +#[cfg(not(any(target_os = "linux", target_os = "macos")))] fn secure_process() -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Unsupported, - "production daemon requires Linux process protection", + "production daemon requires Linux or macOS process protection", )) } fn invalid(message: &'static str) -> io::Error { diff --git a/engine/crates/layerfs-daemon/src/application/filesystem.rs b/engine/crates/layerfs-daemon/src/application/filesystem.rs index a71a5e82b9..21bfcec597 100644 --- a/engine/crates/layerfs-daemon/src/application/filesystem.rs +++ b/engine/crates/layerfs-daemon/src/application/filesystem.rs @@ -1,10 +1,72 @@ //! Kernel serving assembly over the daemon's one Store and one overlay owner. use crate::{ - control::{KernelServing, Service}, + control::{ExportServing, ExportServingConfig, Helper, KernelServing, Mounter, Service}, store::Store, Owner, }; -use std::sync::Arc; +use layerfs_bridge::daemon_setup::DaemonSetup; +use std::{path::Path, sync::Arc}; + +/// Selects the kernel route: `nfs` serves every mount over loopback NFSv3 +/// (the native route); anything else, or unset, is FUSE. +pub const KERNEL_ROUTE: &str = "LAYERFS_KERNEL"; +/// Optional helper command lines (`{port}`, `{dir}` replaced) overriding the +/// host client's defaults for the NFS route. +pub const NFS_MOUNT: &str = "LAYERFS_NFS_MOUNT"; +pub const NFS_UNMOUNT: &str = "LAYERFS_NFS_UNMOUNT"; +pub const NFS_FORCE: &str = "LAYERFS_NFS_FORCE"; +pub const NFS_FLUSH: &str = "LAYERFS_NFS_FLUSH"; + +/// Whether this daemon serves its mounts over NFS. +/// macOS has no other route: there it is always NFS. +pub(super) fn nfs_route() -> bool { + cfg!(target_os = "macos") || std::env::var(KERNEL_ROUTE).is_ok_and(|route| route == "nfs") +} +/// NFS serving for the native route, from the protected setup and the +/// declared helper overrides. Nothing is mounted until an explicit Attach. +pub(super) fn assemble_export(setup: &DaemonSetup) -> std::io::Result>> { + if !nfs_route() { + return Ok(None); + } + let mounts = Path::new(&setup.mounts); + if !mounts.is_absolute() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "mount home must be absolute", + )); + } + if !mounts.is_dir() { + std::fs::create_dir(mounts)?; + } + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper(NFS_MOUNT) { + mounter.mount = mount; + } + if let Some(unmount) = helper(NFS_UNMOUNT) { + mounter.unmount = unmount; + } + if let Some(force) = helper(NFS_FORCE) { + mounter.force = force; + } + mounter.flush = helper(NFS_FLUSH); + let serving = ExportServing::start(ExportServingConfig::new( + mounts.to_owned(), + setup.command_uid, + setup.command_gid, + mounter, + )); + // A daemon of this mount home that ended without unmounting left its + // client mounts; they are forced off before this one serves. + for line in serving.reclaim() { + eprintln!("LAYERFS_NFS_RECLAIM {line}"); + } + Ok(Some(Arc::new(serving))) +} /// Control and kernel serving share the same Store handle and overlay owner; /// the Service's registry stays the sole authority for every mount. The @@ -13,19 +75,19 @@ pub(super) fn service( store: Arc, owner: &Owner, kernel: Option<&Arc>, + export: Option<&Arc>, serial_low_water: u64, ) -> Arc { store.set_serial_low_water(serial_low_water); - Arc::new(match kernel { - Some(kernel) => Service::with_kernel(store, owner, kernel.clone()), - None => Service::new(store, owner), + Arc::new(match (export, kernel) { + (Some(export), _) => Service::with_export(store, owner, export.clone()), + (None, Some(kernel)) => Service::with_kernel(store, owner, kernel.clone()), + (None, None) => Service::new(store, owner), }) } /// Kernel serving is a Linux capability; elsewhere control alone is assembled. #[cfg(not(target_os = "linux"))] -pub(super) fn assemble( - _setup: &layerfs_bridge::daemon_setup::DaemonSetup, -) -> std::io::Result>> { +pub(super) fn assemble(_setup: &DaemonSetup) -> std::io::Result>> { Ok(None) } #[cfg(target_os = "linux")] @@ -58,6 +120,10 @@ mod linux { pub(in crate::application) fn assemble( setup: &DaemonSetup, ) -> io::Result>> { + // The NFS route opens no FUSE device. + if super::nfs_route() { + return Ok(None); + } let mounts = Path::new(&setup.mounts); device()?; home(mounts)?; diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index 9206dee1ac..c15d6833d8 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -44,6 +44,8 @@ pub struct Application { pub(super) owner: Owner, /// Fixed shared kernel workers, entered before control readiness. pub(super) kernel: Option>, + /// NFS serving of the native route, when selected. + pub(super) export: Option>, pub(super) state: Mutex, pub(super) changed: Condvar, pub(super) workers: Mutex>>>, @@ -120,11 +122,21 @@ impl Application { }) } }; + let export = match super::filesystem::assemble_export(&setup) { + Ok(export) => export, + Err(cause) => { + return Err(ApplicationError::AfterOverlay { + owner: Box::new(owner), + cause: Box::new(ApplicationError::Io(cause)), + }) + } + }; let service = existing.as_ref().map(|opened| { super::filesystem::service( opened.store.clone(), &owner, kernel.as_ref(), + export.as_ref(), l.serial_low_water, ) }); @@ -139,6 +151,7 @@ impl Application { instance, owner, kernel, + export, state: Mutex::new(State { startup: Startup { phase, diff --git a/engine/crates/layerfs-daemon/src/application/serve.rs b/engine/crates/layerfs-daemon/src/application/serve.rs index f88db6a0e6..645571dd32 100644 --- a/engine/crates/layerfs-daemon/src/application/serve.rs +++ b/engine/crates/layerfs-daemon/src/application/serve.rs @@ -96,6 +96,7 @@ impl Application { installed.opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.installed = Some(installed); @@ -119,6 +120,7 @@ impl Application { opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.phase = DaemonPhase::ControlReady; diff --git a/engine/crates/layerfs-daemon/src/control/attach.rs b/engine/crates/layerfs-daemon/src/control/attach.rs index 0478136630..5531daa8f2 100644 --- a/engine/crates/layerfs-daemon/src/control/attach.rs +++ b/engine/crates/layerfs-daemon/src/control/attach.rs @@ -35,6 +35,11 @@ impl Service { token: WorkspaceToken, timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return crate::telemetry::phase(timing, "kernel.attach", || { + self.attach_export(&export, token) + }); + } let Some(serving) = self.kernel.clone() else { return Err(Failure::Rejected( ControlCode::Invalid, diff --git a/engine/crates/layerfs-daemon/src/control/detach.rs b/engine/crates/layerfs-daemon/src/control/detach.rs index 0bd63f90a5..ce692cc176 100644 --- a/engine/crates/layerfs-daemon/src/control/detach.rs +++ b/engine/crates/layerfs-daemon/src/control/detach.rs @@ -51,7 +51,12 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + Kernel::Unattached + | Kernel::Attaching + | Kernel::Retained(_) + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => { unreachable!("handled by terminal admission") } } @@ -179,16 +184,6 @@ impl Service { } } } - /// Poison recovery stores this exact owner only; a poisoned registry never - /// authorizes another product operation. - pub(super) fn settle(&self, token: WorkspaceToken, kernel: Kernel, activity: Activity) { - let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); - if let Ok(binding) = bound_mut(&mut entries, token) { - binding.kernel = kernel; - binding.activity = activity; - binding.epoch = binding.epoch.saturating_add(1); - } - } /// Stops a terminal operation after effects, keeping its owner in the entry. /// A forced teardown's facts are stored with it, as they stood at the stop. pub(super) fn retain( diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs new file mode 100644 index 0000000000..00bbd91a8f --- /dev/null +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -0,0 +1,932 @@ +//! NFS kernel serving: the native route's mount entry, beside FUSE. +//! +//! Each Attach serves the Workspace's engine mount as one loopback NFSv3 +//! export (`layerfs-nfs`) on a fresh port, then has the host's own NFS client +//! mount it on the Workspace's mount directory with the configured helper +//! (`mount_nfs` on macOS). The registry keeps the export as the entry's +//! kernel half; the Workspace API, its control records and every engine and +//! Store step are the ones the FUSE route uses: +//! +//! - Ready carries the same receipt record; an NFS receipt is recognized by +//! `abi_major == 0`, and its `mount_id` is the export's TCP port. +//! - Commit first has the client write back what it buffered (the flush +//! helper, `sync(2)` by default) and closes the export's implicit file +//! descriptors, then runs the unchanged Commit. A Commit that installed +//! another root closes the export's descriptors again; the client +//! revalidates attributes on its own schedule (close-to-open, `actimeo`). +//! - Unmount asks the helper to unmount (a busy answer is reversible and the +//! export keeps serving), then stops the export (every connection joined, +//! every reference and descriptor given back), revokes the engine mount +//! and runs the one logical Close. A step that is not established keeps +//! the export and its engine mount in the entry, as FUSE custody is kept. +//! - ForceUnmount runs the forced helper once, then the same teardown. +use super::{ + export_helper::{Helpers, Ran}, + kernel::{bounded, Kernel}, + registry::{bound_mut, idle}, + Failure, Service, Success, +}; +use crate::store::BoundWorkspace; +use layerfs_bridge::control::{ + AbortDisposition, Activity, ControlCode, DetachDisposition, ForceUnmounted, ForcedCleanup, + ForcedFacts, ForcedOutcome, MountPhase, MountReceipt, MountStatus, MountWork, ReadyMount, + Reply, TeardownCustody, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{CleanupState, KernelMount}; +#[cfg(target_os = "macos")] +use std::process::{Command, Stdio}; +use std::{ + fmt, + net::{IpAddr, Ipv4Addr, TcpListener}, + path::PathBuf, + sync::Arc, +}; + +/// Largest write and read the export negotiates (FSINFO wtmax/rtmax). +const WINDOW: u32 = 128 * 1024; +pub use super::export_helper::{Helper, HELPER_WAIT}; + +/// Stale inodes one install refreshes through the mount point at most. +pub const REFRESHED: usize = 4096; +/// `0` turns the post-install refresh off (see [`Service::installed_export`]). +pub const REFRESH: &str = "LAYERFS_NFS_REFRESH"; + +/// One mode-preserving chmod of each path below `root`, through the host's +/// NFS client; a symbolic link (chmod would follow it) and a path that is +/// gone are skipped. The export answers each without a change. +fn refresh(root: &std::path::Path, paths: &[PathBuf]) { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + for path in paths { + let path = root.join(path); + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + if !metadata.file_type().is_symlink() { + let mode = std::fs::Permissions::from_mode(metadata.mode() & 0o7777); + let _ = std::fs::set_permissions(&path, mode); + } + } +} +/// How the host's NFS client mounts, flushes and unmounts an export. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Mounter { + pub mount: Helper, + pub unmount: Helper, + pub force: Helper, + /// Writes back what the client buffered before a Commit; `sync(2)` of + /// the daemon itself when absent. + pub flush: Option, +} +impl Mounter { + /// The host's own client: `mount_nfs` on macOS (no root needed), the + /// Linux NFS client elsewhere (root, as the daemon is in its container). + /// The macOS mount is soft (an export that ended fails calls after the + /// retries instead of hanging them) and hidden from Finder (`nobrowse`). + /// `dumbtimer,timeo=300` keeps a slow answer from failing as a timeout: + /// against a loopback server the round-trip estimator settles near its + /// floor, and a soft call then gave up within seconds. `mutejukebox` + /// keeps an `NFS3ERR_JUKEBOX` answer from raising the "server not + /// responding" dialog. A dead export still fails calls (connection + /// refused), within seconds. `noatime` stops the client from updating the + /// access time: this export keeps one timestamp and shows it as atime, + /// mtime and ctime alike, and `layerfs-nfs` decodes and drops a SETATTR's + /// access time, so the update would cost one SETATTR per file a read or a + /// scan touches and change nothing. + pub fn host() -> Self { + let helper = |words: &[&str]| Helper(words.iter().map(|w| (*w).to_owned()).collect()); + if cfg!(target_os = "macos") { + Self { + mount: helper(&[ + "/sbin/mount_nfs", + "-o", + "vers=3,tcp,port={port},mountport={port},locallocks,nfc,noatime,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["/sbin/umount", "{dir}"]), + force: helper(&["/sbin/umount", "-f", "{dir}"]), + flush: None, + } + } else { + Self { + mount: helper(&[ + "mount", + "-t", + "nfs", + "-o", + "vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["umount", "{dir}"]), + force: helper(&["umount", "-f", "-l", "{dir}"]), + flush: None, + } + } + } +} +/// Declared NFS serving inputs, assembled once by the application. +#[derive(Clone, Debug)] +pub struct ExportServingConfig { + /// Existing daemon-owned directory; each mount is a new child of it. + pub mounts: PathBuf, + /// Runtime command identity presented as every inode's owner. + pub command_uid: u32, + pub command_gid: u32, + /// Address each export listens on, port chosen by the system. + pub listen: IpAddr, + pub mounter: Mounter, + /// Refuse creating macOS AppleDouble `._*` names. + pub refuse_apple_double: bool, + /// Lookup references one export keeps before collecting. + pub lookups_held: usize, +} +impl ExportServingConfig { + pub fn new(mounts: PathBuf, command_uid: u32, command_gid: u32, mounter: Mounter) -> Self { + Self { + mounts, + command_uid, + command_gid, + listen: IpAddr::V4(Ipv4Addr::LOCALHOST), + mounter, + refuse_apple_double: true, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } +} +/// Process-wide NFS serving: configuration only; each Attach starts its own +/// export and each terminal operation stops it. +#[derive(Debug)] +pub struct ExportServing { + config: ExportServingConfig, + helpers: Helpers, +} +impl ExportServing { + pub fn start(config: ExportServingConfig) -> Self { + Self { + config, + helpers: Helpers::default(), + } + } + pub fn config(&self) -> &ExportServingConfig { + &self.config + } + fn directory(&self, token: WorkspaceToken) -> PathBuf { + self.config.mounts.join(token.namespace.to_string()) + } + /// At startup, before any Attach: every mount the host still has at or + /// under the mount home is one a daemon that ended without unmounting + /// left (its export is gone, so the client would hang on it). Each is + /// forced off once with the force helper; one line per mount says what + /// happened. Nothing is retried. + pub fn reclaim(&self) -> Vec { + let points = match mounted_under(&self.config.mounts) { + Ok(points) => points, + Err(error) => return vec![format!("mount table unreadable: {error}")], + }; + (points.into_iter()) + .map(|point| { + let text = point.display().to_string(); + match self.helpers.run(&self.config.mounter.force, 0, &text) { + Ran::Done => format!("forced off {text}"), + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + format!("left {text}: {detail}") + } + } + }) + .collect() + } +} +/// Mount points at or under `home`, deepest first, from the host's table: +/// `mount`'s ` on ()` lines on macOS, +/// `/proc/self/mountinfo` (field 5, octal escapes) on Linux. +fn mounted_under(home: &std::path::Path) -> std::io::Result> { + #[cfg(target_os = "macos")] + let points: Vec = { + let out = Command::new("/sbin/mount") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + (String::from_utf8_lossy(&out.stdout).lines()) + .filter_map(|line| { + Some(PathBuf::from( + line.split_once(" on ")?.1.rsplit_once(" (")?.0, + )) + }) + .collect() + }; + #[cfg(not(target_os = "macos"))] + let points: Vec = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) + .filter_map(|line| line.split(' ').nth(4)) + .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) + .collect(); + let mut found: Vec = (points.into_iter()) + .filter(|point| point.starts_with(home)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} +/// A serving export and the mount the host client holds of it. +pub(super) struct Exported { + pub server: Arc, + pub mount: KernelMount, + pub ready: ReadyMount, +} +/// A terminal operation owns the export; status still observes it. +pub(super) struct ExportLeaving { + pub ready: ReadyMount, + pub phase: MountPhase, + pub work: MountWork, +} +/// Export custody a terminal or attach step could not finish disposing: +/// kept exactly, never retried or settled by a later observation. +pub(super) struct ExportKept { + pub ready: Option, + pub stage: TeardownStage, + pub detail: String, + pub work: Option, + pub evidence: Box, + pub forced: Option, +} +impl ExportKept { + pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { + TeardownCustody { + token, + stage: self.stage, + detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), + work: self.work, + detail: bounded(&self.detail), + forced: self.forced.clone(), + } + } +} +impl fmt::Debug for ExportKept { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExportKept") + .field("ready", &self.ready) + .field("stage", &self.stage) + .field("detail", &self.detail) + .field("work", &self.work) + .field("evidence", &self.evidence) + .field("forced", &self.forced) + .finish() + } +} +impl fmt::Debug for Exported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Exported") + .field("address", &self.server.local_addr()) + .field("mount", &self.mount) + .field("held", &self.server.held()) + .finish() + } +} +/// The maintained counters of a serving export. +pub(super) fn work(server: &Server) -> MountWork { + let held = server.held(); + let served = server.counters().iter().map(|count| count.calls).sum(); + MountWork { + loops_configured: 1, + loops_entered: 1, + loops_exited: 0, + loops_joined: 0, + active: 0, + retained: u32::try_from(held.retained).unwrap_or(u32::MAX), + completed: served, + served, + inline: 0, + refused: 0, + terminal: 0, + forget_units: held.collected, + } +} +/// The status block of an export-owning entry. +pub(super) fn block(kernel: &Kernel) -> Option { + Some(match kernel { + Kernel::Exported(exported) => MountStatus { + phase: MountPhase::Ready, + ready: Some(exported.ready.clone()), + detached: false, + work: Some(work(&exported.server)), + }, + Kernel::ExportLeaving(leaving) => MountStatus { + phase: leaving.phase, + ready: Some(leaving.ready.clone()), + detached: leaving.phase == MountPhase::Draining, + work: Some(leaving.work), + }, + Kernel::ExportKept(kept) => MountStatus { + phase: MountPhase::Retained, + ready: kept.ready.clone(), + detached: matches!(kept.stage, TeardownStage::Revoke | TeardownStage::Close), + work: kept.work, + }, + _ => return None, + }) +} +fn receipt(token: WorkspaceToken, directory: String, mount: KernelMount, port: u16) -> ReadyMount { + ReadyMount { + token, + directory, + receipt: MountReceipt { + mount: mount.owner_id(), + root: mount.root_serial(), + mount_id: u64::from(port), + device_major: 0, + device_minor: 0, + abi_major: 0, + abi_minor: 0, + offered: 0, + selected: 0, + max_write: WINDOW, + max_readahead: WINDOW, + max_background: 0, + congestion_threshold: 0, + page_size: 4096, + loops: 1, + abort_bound: true, + }, + } +} +fn kernel_failure(code: ControlCode, phase: &'static str, detail: impl Into) -> Failure { + Failure::Kernel(Box::new(super::types::KernelFailure { + code, + phase, + detail: detail.into(), + evidence: None, + })) +} +/// Where a failed export Attach leaves the entry. +enum Stop { + Unattached(Failure), + Unknown(Failure), + Retained(Box), +} +impl Service { + /// Attach over NFS: Ready is acknowledged only after the export serves + /// and the host client's mount helper reported the mount. + pub(super) fn attach_export( + &self, + serving: &ExportServing, + token: WorkspaceToken, + ) -> Result { + let workspace = self.admit_export(token)?; + match self.export_admitted(serving, token, &workspace) { + Ok(exported) => { + let reply = Reply::Ready(Box::new(exported.ready.clone())); + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + Ok(Success::reply(reply)) + } + Err(Stop::Unattached(failure)) => { + self.settle(token, Kernel::Unattached, Activity::Idle); + Err(failure) + } + Err(Stop::Unknown(failure)) => { + self.settle(token, Kernel::Attaching, Activity::Uncertain); + Err(failure) + } + Err(Stop::Retained(kept)) => { + let custody = kept.custody(token); + self.settle(token, Kernel::ExportKept(kept), Activity::Uncertain); + Err(Failure::Retained(Box::new(custody))) + } + } + } + fn admit_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } + idle(value)?; + if !matches!(value.kernel, Kernel::Unattached) { + return Err(kernel_failure( + ControlCode::Invalid, + "attach:admission", + "Workspace already owns a kernel connection", + )); + } + value.kernel = Kernel::Attaching; + value.activity = Activity::Attaching; + value.epoch = value.epoch.saturating_add(1); + Ok(value.workspace.clone()) + } + fn export_admitted( + &self, + serving: &ExportServing, + token: WorkspaceToken, + workspace: &Arc, + ) -> Result { + let config = &serving.config; + let directory = serving.directory(token); + let Some(text) = directory + .to_str() + .filter(|text| text.len() <= MOUNT_DIRECTORY_LIMIT) + .map(str::to_owned) + else { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Invalid, + "attach:directory", + "mount directory exceeds the control record bound", + ))); + }; + let unknown = |failure: Failure| { + if failure.uncertain() { + Stop::Unknown(failure) + } else { + Stop::Unattached(failure) + } + }; + // The same preparation as the FUSE Attach: the root serial, then the + // initial authoritative serial range before any request can create. + let operation = workspace + .operation() + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + let root = operation + .workspace() + .base() + .map(|base| base.root().root_inode().serial()) + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + if let Err(error) = operation.workspace().prepare_serials(operation.ports()) { + let failure = match operation.ports().failure() { + Ok(Some(cause)) => match cause.as_ref() { + crate::store::PortError::History(error) => Failure::History(error.clone()), + _ => Failure::Workspace(error), + }, + Ok(None) | Err(_) => Failure::Workspace(error), + }; + return Err(unknown(failure)); + } + drop(operation); + if let Err(error) = std::fs::create_dir(&directory) { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Failed, + "attach:directory", + format!("mount directory {text}: {error}"), + ))); + } + let mount = match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_mount(route, root) + }) { + Ok(mount) => mount, + Err(failure) => { + drop(std::fs::remove_dir(&directory)); + return Err(unknown(failure)); + } + }; + let export = ExportConfig { + path: "/".into(), + uid: config.command_uid, + gid: config.command_gid, + read_only: false, + refuse_apple_double: config.refuse_apple_double, + lookups_held: config.lookups_held, + }; + let started = TcpListener::bind((config.listen, 0)).and_then(|listener| { + Server::start( + listener, + Arc::new(crate::service::Mounted(workspace.clone())), + mount, + export, + ) + }); + let server = match started { + Ok(server) => server, + Err(error) => { + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + None, + kernel_failure(ControlCode::Failed, "attach:export", error.to_string()), + )); + } + }; + let port = server.local_addr().port(); + match serving.helpers.run(&config.mounter.mount, port, &text) { + Ran::Done => {} + Ran::Interrupted(detail) => { + // The mount helper may have reached the kernel before its + // deadline. Keep the export alive and do not revoke/delete + // through a possibly mounted directory. + return Err(Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Detach, + detail, + work: Some(work(&server)), + evidence: Box::new(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }), + forced: None, + }))); + } + Ran::Busy(detail) | Ran::Failed(detail) => { + let released = server.stop(); + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + Some(released), + kernel_failure(ControlCode::Failed, "attach:mount", detail), + )); + } + } + Ok(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }) + } + /// No client mount and no export of this attempt remains: revoke its + /// engine mount so the entry returns to Unattached. + fn unwind_export( + &self, + workspace: &BoundWorkspace, + mount: KernelMount, + released: Option, + failure: Failure, + ) -> Stop { + match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + Ok(()) => Stop::Unattached(failure), + Err(revoke) => Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Revoke, + detail: revoke.wire().detail, + work: None, + evidence: Box::new((failure, mount, released, revoke)), + forced: None, + })), + } + } + /// Before a Commit: the client writes back what it buffered, and the + /// export closes its implicit file descriptors. Runs outside the + /// registry lock; a Workspace without an export has nothing to flush. + pub(super) fn flush_export(&self, token: WorkspaceToken) -> Result<(), Failure> { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(()), + } + }; + let Some(serving) = self.export.as_ref() else { + return Ok(()); + }; + match &serving.config.mounter.flush { + None => nix::unistd::sync(), + Some(helper) => { + match serving + .helpers + .run(helper, server.local_addr().port(), &directory) + { + Ran::Done => {} + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) + } + } + } + } + server.close_files(); + Ok(()) + } + /// After a Commit installed another root: the export's descriptors of the + /// replaced tree are closed, and the inodes the install made stale that + /// the client may cache are refreshed through the mount point (see + /// `layerfs_nfs::refresh`): a mode-preserving chmod of each, at most + /// [`REFRESHED`], on a thread of its own after this returns, so the + /// Commit reply never waits on the client. False without an export. + pub(super) fn installed_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + shown: layerfs_content::ObjectId, + installed: layerfs_content::ObjectId, + ) -> Result { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(false), + } + }; + server.installed(); + // `LAYERFS_NFS_REFRESH=0` leaves the client to its own revalidation + // (`actimeo`, close-to-open): the comparison without the refresh. + if std::env::var(REFRESH).is_ok_and(|value| value == "0") { + return Ok(true); + } + let mut stale = std::collections::BTreeSet::new(); + // A walk that stops (at the bound, or failing) refreshes what it found; + // the rest is revalidated on the client's own schedule. + let _ = workspace.stale_after_install(shown, installed, |found| { + stale.insert(match found { + crate::store::Stale::Inode(serial) => serial, + crate::store::Stale::Entry { parent, .. } => parent, + }); + if stale.len() >= REFRESHED { + return Err(std::io::Error::other("refresh bound")); + } + Ok(()) + }); + let paths = server.refresh(stale); + if !paths.is_empty() { + let root = PathBuf::from(directory); + let _ = std::thread::Builder::new() + .name("layerfs-nfs-refresh".into()) + .spawn(move || refresh(&root, &paths)); + } + Ok(true) + } + /// Normal terminal unmount of an export-owning entry, admitted by + /// [`take`]. A busy client mount is the reversible answer. + pub(super) fn unmount_export( + &self, + token: WorkspaceToken, + workspace: Arc, + exported: Exported, + ) -> Result { + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.unmount.clone()); + let port = exported.server.local_addr().port(); + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => {} + Ran::Busy(detail) => { + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + return Err(kernel_failure( + ControlCode::Busy, + "unmount:kernel", + format!("client reported the mount busy; no terminal effect: {detail}"), + )); + } + Ran::Failed(detail) | Ran::Interrupted(detail) => { + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("client unmount: {detail}"), + exported, + None, + )) + } + } + self.teardown_export(token, &workspace, exported, None)?; + self.close(token, &workspace, None) + } + /// After the client mount is gone: stop the export, revoke the engine + /// mount, remove the directory and leave the entry Unattached for Close. + fn teardown_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + exported: Exported, + forced: Option, + ) -> Result<(), Failure> { + self.export_phase(token, MountPhase::Draining, Some(work(&exported.server))); + let Exported { + server, + mount, + ready, + } = exported; + let released = match Arc::try_unwrap(server) { + Ok(server) => server.stop(), + Err(server) => { + return Err(self.keep_export( + token, + TeardownStage::Requests, + "export still shared by a running control step".into(), + Exported { + server, + mount, + ready, + }, + forced, + )) + } + }; + if let Err(failure) = self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + let custody = ExportKept { + ready: Some(ready), + stage: TeardownStage::Revoke, + detail: failure.wire().detail, + work: None, + evidence: Box::new((failure, mount, released)), + forced, + }; + let reply = custody.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(custody)), + Activity::Uncertain, + ); + return Err(Failure::Retained(Box::new(reply))); + } + drop(std::fs::remove_dir(&ready.directory)); + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = Kernel::Unattached; + } + } + self.released + .lock() + .map(|mut last| *last = Some((token, released))) + .ok(); + Ok(()) + } + fn keep_export( + &self, + token: WorkspaceToken, + stage: TeardownStage, + detail: String, + exported: Exported, + forced: Option, + ) -> Failure { + let kept = ExportKept { + ready: Some(exported.ready.clone()), + stage, + detail, + work: Some(work(&exported.server)), + evidence: Box::new(exported), + forced, + }; + let custody = kept.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(kept)), + Activity::Uncertain, + ); + Failure::Retained(Box::new(custody)) + } + fn export_phase(&self, token: WorkspaceToken, phase: MountPhase, work: Option) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + if let Kernel::ExportLeaving(leaving) = &mut binding.kernel { + leaving.phase = phase; + if let Some(work) = work { + leaving.work = work; + } + } + } + } + } + /// Describes retained export custody for an operator. Observation only. + pub fn retained_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + Ok(match &bound_mut(&mut entries, token)?.kernel { + Kernel::ExportKept(kept) => Some(format!("{kept:?}")), + _ => None, + }) + } + /// What the last stopped export of `token` gave back (diagnostics). + pub fn export_released(&self, token: WorkspaceToken) -> Option { + let last = self.released.lock().ok()?; + last.filter(|(seen, _)| *seen == token) + .map(|(_, released)| released) + } + /// The current holdings of a serving export (diagnostics). + pub fn export_held(&self, token: WorkspaceToken) -> Option { + let entries = self.entries.lock().ok()?; + match &super::registry::bound(&entries, token).ok()?.kernel { + Kernel::Exported(exported) => Some(exported.server.held()), + _ => None, + } + } + /// Forced terminal unmount of an export-owning entry: the same + /// before-effect guard as FUSE, one forced client unmount, then the + /// normal teardown and Close. + pub(super) fn force_export( + &self, + token: WorkspaceToken, + relinquish_unknown: bool, + ) -> Result { + let (workspace, exported, commit) = { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + let commit = super::force::guard(value, token, relinquish_unknown)?; + let exported = take(&mut value.kernel, MountPhase::Stopping)?; + value.activity = Activity::Closing; + value.epoch = value.epoch.saturating_add(1); + (value.workspace.clone(), exported, commit) + }; + let mut facts = ForcedFacts { + abort: AbortDisposition::Written, + detach: DetachDisposition::NotAttempted, + commit, + fenced: 0, + }; + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.force.clone()); + let port = exported.server.local_addr().port(); + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => facts.detach = DetachDisposition::Detached, + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + facts.detach = DetachDisposition::Failed; + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("forced client unmount: {detail}"), + exported, + Some(facts), + )); + } + } + let work = work(&exported.server); + self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; + let kernel = self.closed(token, &workspace, None, Some(facts.clone()))?; + let (cleanup, observation_failure) = match self + .owner + .observe_cleanup(None, token.namespace, token.workspace.to_bytes()) + .map_err(Failure::Owner) + { + Ok(CleanupState::Held) => (ForcedCleanup::Held, None), + Ok(CleanupState::Queued) => (ForcedCleanup::Queued, None), + Ok(CleanupState::Gone) => (ForcedCleanup::Gone, None), + Ok(CleanupState::Live) => ( + ForcedCleanup::Unobserved, + Some(Box::new(Failure::Rejected( + ControlCode::Unknown, + "closed Workspace remains live", + ))), + ), + Err(error) => (ForcedCleanup::Unobserved, Some(Box::new(error))), + }; + self.depart( + token, + Success { + reply: Reply::ForceUnmounted(Box::new(ForceUnmounted { + token, + outcome: ForcedOutcome { + facts, + work, + cleanup, + }, + })), + commit: None, + observation_failure, + kernel, + }, + ) + } +} +/// Terminal admission over an export-owning entry; the entry observes the +/// export as leaving until the operation settles it. +pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + match kernel { + Kernel::Exported(_) => {} + Kernel::ExportLeaving(_) => { + return Err(kernel_failure( + ControlCode::Busy, + "unmount:admission", + "terminal unmount already admitted", + )) + } + _ => { + return Err(kernel_failure( + ControlCode::Invalid, + "unmount:admission", + "Workspace owns no export", + )) + } + } + let Kernel::Exported(exported) = std::mem::replace(kernel, Kernel::Unattached) else { + unreachable!("checked export") + }; + *kernel = Kernel::ExportLeaving(Box::new(ExportLeaving { + ready: exported.ready.clone(), + phase, + work: work(&exported.server), + })); + Ok(*exported) +} diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs new file mode 100644 index 0000000000..a4796ab725 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -0,0 +1,155 @@ +//! One bounded NFS client helper attempt, including post-KILL observation. +use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use std::{ + io::{self, Read}, + process::{Child, Command, Stdio}, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// How long one helper run may take before it is killed and reported. +pub const HELPER_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const PAUSE: Duration = Duration::from_millis(5); +const OUTPUT: usize = 2048; + +/// One helper command; `{port}` and `{dir}` in its words are replaced. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Helper(pub Vec); +impl Helper { + /// Splits a configured command line on whitespace. + pub fn words(line: &str) -> Option { + let words: Vec = line.split_whitespace().map(str::to_owned).collect(); + (!words.is_empty()).then_some(Self(words)) + } + fn command(&self, port: u16, directory: &str) -> Option { + let mut words = self.0.iter().map(|word| { + word.replace("{port}", &port.to_string()) + .replace("{dir}", directory) + }); + let mut command = Command::new(words.next()?); + command.args(words); + Some(command) + } +} + +pub(super) enum Ran { + Done, + Busy(String), + Failed(String), + /// Timeout or observation failure after spawn: effects are not known. + Interrupted(String), +} + +/// Retained children are never waited on synchronously or signalled twice. +/// A later helper attempt may only reap their observed exits, not replay them. +#[derive(Debug, Default)] +pub(super) struct Helpers(Mutex>); +impl Helpers { + pub fn run(&self, helper: &Helper, port: u16, directory: &str) -> Ran { + { + let mut held = self.0.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let Some(mut command) = helper.command(port, directory) else { + return Ran::Failed("empty helper command".into()); + }; + let mut child = match command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(error) => return Ran::Failed(format!("{:?}: {error}", helper.0.first())), + }; + let mut pipes: Vec> = Vec::new(); + let mut pipe_error = None; + // Even an exited helper can leave a descendant holding its pipe open. + // Never wait for EOF, and drain while it runs so verbose helpers progress. + for pipe in [ + child.stderr.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + child.stdout.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + ] + .into_iter() + .flatten() + { + match pipe.1 { + Ok(_) => pipes.push(pipe.0), + Err(error) => pipe_error = Some(format!("helper output: {error}")), + } + } + let mut output = Vec::new(); + let deadline = Instant::now() + HELPER_WAIT; + let status = loop { + drain(&mut pipes, &mut output); + if let Some(error) = pipe_error.take() { + break Err(error); + } + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) => break Err(format!("helper did not finish in {HELPER_WAIT:?}")), + Err(error) => break Err(format!("helper wait: {error}")), + } + }; + let status = match status { + Ok(status) => Ok(status), + Err(detail) => { + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break false, + } + }; + let pid = child.id(); + let disposition = if reaped { + "exit reaped".to_owned() + } else { + self.0 + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; helper may remain in kernel wait; child custody retained".into() + }; + Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) + } + }; + drain(&mut pipes, &mut output); + let output = String::from_utf8_lossy(&output); + match status { + Ok(status) if status.success() => Ran::Done, + Ok(status) => { + let detail = format!("{status}: {}", output.trim()); + if output.to_ascii_lowercase().contains("busy") { + Ran::Busy(detail) + } else { + Ran::Failed(detail) + } + } + Err(detail) => Ran::Interrupted(format!("{detail}: {}", output.trim())), + } + } +} + +fn drain(pipes: &mut [Box], output: &mut Vec) { + for pipe in pipes { + let mut bytes = [0; OUTPUT]; + // One bounded read per pipe and observation: even continuous output + // cannot defer the helper's deadline. Excess bytes are discarded. + match pipe.read(&mut bytes) { + Ok(count) => output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(_) => {} + } + } +} diff --git a/engine/crates/layerfs-daemon/src/control/force.rs b/engine/crates/layerfs-daemon/src/control/force.rs index 6eb780c616..1699cdc211 100644 --- a/engine/crates/layerfs-daemon/src/control/force.rs +++ b/engine/crates/layerfs-daemon/src/control/force.rs @@ -45,7 +45,7 @@ fn refused(code: ControlCode, phase: &'static str, detail: &'static str) -> Fail /// The whole before-effect decision, made under the registry lock and in this /// order. A refusal changes nothing. Admitted, it returns the Commit knowledge /// the receipt will carry: copied here, never settled by a later read. -fn guard( +pub(super) fn guard( value: &Binding, token: WorkspaceToken, relinquish_unknown: bool, @@ -53,6 +53,9 @@ fn guard( // Stored custody answers first: no second abort or detach ever follows // it. The one exception is a normal unmount stalled at Join, admitted // idle with no Commit custody, whose abort control is still unused. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { if kept.forceable() { @@ -60,8 +63,6 @@ fn guard( } return Err(Failure::Retained(Box::new(kept.custody(token)))); } - #[cfg(not(target_os = "linux"))] - let _ = token; if matches!( value.activity, Activity::Committing | Activity::Attaching | Activity::Closing @@ -113,7 +114,6 @@ fn connection(kernel: &Kernel) -> Result { ADMISSION, "Workspace owns no kernel connection", )), - #[cfg(target_os = "linux")] Kernel::Attaching => Err(refused( ControlCode::Unknown, ADMISSION, @@ -129,6 +129,13 @@ fn connection(kernel: &Kernel) -> Result { )), #[cfg(target_os = "linux")] Kernel::Ready(attached) => Ok(attached.session.abort_bound()), + // The forced client unmount is the export's abort. + Kernel::Exported(_) => Ok(true), + Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => Err(refused( + ControlCode::Busy, + ADMISSION, + "terminal unmount already admitted", + )), } } #[cfg(not(target_os = "linux"))] @@ -139,6 +146,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; guard(bound_mut(&mut entries, token)?, token, relinquish_unknown)?; Err(refused( @@ -199,6 +209,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let admitted = self.admit_force(token, relinquish_unknown)?; self.force_admitted(token, admitted) } diff --git a/engine/crates/layerfs-daemon/src/control/invalidate.rs b/engine/crates/layerfs-daemon/src/control/invalidate.rs index 7a7b0dc8b5..19f668df8a 100644 --- a/engine/crates/layerfs-daemon/src/control/invalidate.rs +++ b/engine/crates/layerfs-daemon/src/control/invalidate.rs @@ -17,6 +17,12 @@ impl Service { shown: ObjectId, installed: ObjectId, ) -> Result<(), Failure> { + // An NFS client holds no invalidation channel: the export closes its + // descriptors of the replaced tree and refreshes the stale inodes the + // client may cache through the mount point. + if self.installed_export(token, workspace, shown, installed)? { + return Ok(()); + } #[cfg(target_os = "linux")] { use super::{kernel::Kernel, registry::bound}; diff --git a/engine/crates/layerfs-daemon/src/control/kernel.rs b/engine/crates/layerfs-daemon/src/control/kernel.rs index 92c07ef476..dc37ac187c 100644 --- a/engine/crates/layerfs-daemon/src/control/kernel.rs +++ b/engine/crates/layerfs-daemon/src/control/kernel.rs @@ -31,7 +31,6 @@ pub use super::serving::KernelServing; pub(super) enum Kernel { Unattached, /// One Attach is admitted, or its engine outcome is unknown. - #[cfg(target_os = "linux")] Attaching, #[cfg(target_os = "linux")] Ready(Box), @@ -40,6 +39,12 @@ pub(super) enum Kernel { Leaving(Box), #[cfg(target_os = "linux")] Retained(Box), + /// NFS route: a serving export the host client has mounted. + Exported(Box), + /// NFS route: a terminal operation owns the export. + ExportLeaving(Box), + /// NFS route: export custody a teardown could not finish disposing. + ExportKept(Box), } impl Kernel { /// Maintained observations only: no syscall, SQL job or kernel request. @@ -52,8 +57,10 @@ impl Kernel { }; match self { Self::Unattached => plain(MountPhase::Unattached), - #[cfg(target_os = "linux")] Self::Attaching => plain(MountPhase::Attaching), + Self::Exported(_) | Self::ExportLeaving(_) | Self::ExportKept(_) => { + super::export::block(self).unwrap_or_else(|| plain(MountPhase::Unattached)) + } #[cfg(target_os = "linux")] Self::Ready(attached) => { let facts = attached.session.facts(); diff --git a/engine/crates/layerfs-daemon/src/control/mod.rs b/engine/crates/layerfs-daemon/src/control/mod.rs index 9d42da651e..75abdf4e74 100644 --- a/engine/crates/layerfs-daemon/src/control/mod.rs +++ b/engine/crates/layerfs-daemon/src/control/mod.rs @@ -5,6 +5,8 @@ mod cleanup; #[cfg(target_os = "linux")] mod detach; mod dirty; +mod export; +mod export_helper; mod failure; mod force; mod inspect; @@ -19,6 +21,7 @@ mod serving; mod status; mod types; mod unmount; +pub use export::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; pub use kernel::{KernelConfig, KernelServing}; pub(crate) use observation::{reply_token, request_token}; pub use registry::Service; diff --git a/engine/crates/layerfs-daemon/src/control/observation.rs b/engine/crates/layerfs-daemon/src/control/observation.rs index b8f0edda08..e403c58c59 100644 --- a/engine/crates/layerfs-daemon/src/control/observation.rs +++ b/engine/crates/layerfs-daemon/src/control/observation.rs @@ -57,7 +57,11 @@ impl Service { Kernel::Ready(attached) => Some(attached.session.observer()), Kernel::Leaving(leaving) => Some(leaving.observer.clone()), Kernel::Retained(kept) => kept.observer.clone(), - Kernel::Unattached | Kernel::Attaching => None, + Kernel::Unattached + | Kernel::Attaching + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => None, }) } } diff --git a/engine/crates/layerfs-daemon/src/control/operations.rs b/engine/crates/layerfs-daemon/src/control/operations.rs index 3b44d81058..260b5da1a0 100644 --- a/engine/crates/layerfs-daemon/src/control/operations.rs +++ b/engine/crates/layerfs-daemon/src/control/operations.rs @@ -217,6 +217,12 @@ impl Service { run: impl FnOnce(&BoundWorkspace) -> Result>, ) -> Result { let workspace = self.admit(token, Activity::Committing)?; + // NFS route: what the client buffered is written back first, as a + // FUSE write has already reached the Workspace when it returned. + if let Err(failure) = self.flush_export(token) { + self.settle_activity(token, Activity::Idle); + return Err(failure); + } let result = run(&workspace); // An install that replaced what the Workspace showed is invalidated // in the kernel before the answer. @@ -313,6 +319,31 @@ impl Service { }, } } + /// Poison recovery stores this exact owner only; a poisoned registry never + /// authorizes another product operation. + pub(super) fn settle( + &self, + token: WorkspaceToken, + kernel: super::kernel::Kernel, + activity: Activity, + ) { + let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = kernel; + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + /// Returns an admitted entry to `activity` when its operation stopped + /// before any effect. + fn settle_activity(&self, token: WorkspaceToken, activity: Activity) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + } pub(super) fn job( &self, route: layerfs_workspace::Route, diff --git a/engine/crates/layerfs-daemon/src/control/registry.rs b/engine/crates/layerfs-daemon/src/control/registry.rs index f21131caa2..2bf5e2e910 100644 --- a/engine/crates/layerfs-daemon/src/control/registry.rs +++ b/engine/crates/layerfs-daemon/src/control/registry.rs @@ -33,6 +33,10 @@ pub struct Service { pub(super) entries: Mutex>, /// Absent for a control-only Service: Attach is then refused before effects. pub(super) kernel: Option>, + /// NFS serving (native route); when present, Attach serves NFS. + pub(super) export: Option>, + /// What the last stopped export gave back, for diagnostics. + pub(super) released: Mutex>, } impl Service { /// Binds control to the already initialized owner and its configured capacity. @@ -43,8 +47,26 @@ impl Service { capacity: owner.configuration().namespaces, entries: Mutex::new(BTreeMap::new()), kernel: None, + export: None, + released: Mutex::new(None), } } + /// Binds control and NFS serving (the native route's mount entry) to the + /// same Store and overlay owner. + pub fn with_export( + store: Arc, + owner: &Owner, + export: Arc, + ) -> Self { + Self { + export: Some(export), + ..Self::new(store, owner) + } + } + /// Whether Attach serves NFS rather than FUSE. + pub fn serves_nfs(&self) -> bool { + self.export.is_some() + } /// Binds control and kernel serving to the same Store and overlay owner. pub fn with_kernel(store: Arc, owner: &Owner, kernel: Arc) -> Self { Self { diff --git a/engine/crates/layerfs-daemon/src/control/status.rs b/engine/crates/layerfs-daemon/src/control/status.rs index 9b86d52495..43a4a1a30c 100644 --- a/engine/crates/layerfs-daemon/src/control/status.rs +++ b/engine/crates/layerfs-daemon/src/control/status.rs @@ -44,7 +44,7 @@ impl Service { value.epoch, value.published.clone(), // Copied with the control fields: maintained counters only. - self.kernel.as_ref().map(|_| value.kernel.block()), + (self.kernel.is_some() || self.export.is_some()).then(|| value.kernel.block()), ) }; let observed = self.job( diff --git a/engine/crates/layerfs-daemon/src/control/types.rs b/engine/crates/layerfs-daemon/src/control/types.rs index 320b574fd1..8d4df91f7e 100644 --- a/engine/crates/layerfs-daemon/src/control/types.rs +++ b/engine/crates/layerfs-daemon/src/control/types.rs @@ -65,7 +65,6 @@ impl Success { } } impl Failure { - #[cfg(target_os = "linux")] pub(super) fn kernel( code: ControlCode, phase: &'static str, diff --git a/engine/crates/layerfs-daemon/src/control/unmount.rs b/engine/crates/layerfs-daemon/src/control/unmount.rs index 062b26a19f..5d90daa83d 100644 --- a/engine/crates/layerfs-daemon/src/control/unmount.rs +++ b/engine/crates/layerfs-daemon/src/control/unmount.rs @@ -17,6 +17,8 @@ pub(super) enum Departure { Unattached, #[cfg(target_os = "linux")] Session(Box), + /// NFS route: the export and the client mount of it. + Export(Box), } impl Service { /// Refuses uncommitted changes with their paths unless `discard`. @@ -30,6 +32,7 @@ impl Service { let (workspace, departure) = self.admit_unmount(token)?; match departure { Departure::Unattached => {} + Departure::Export(exported) => return self.unmount_export(token, workspace, *exported), #[cfg(target_os = "linux")] Departure::Session(attached) => { return self.unmount_kernel(token, workspace, *attached) @@ -46,6 +49,9 @@ impl Service { let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; let value = bound_mut(&mut entries, token)?; // Retained custody answers before ordinary activity admission. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { return Err(Failure::Retained(Box::new(kept.custody(token)))); @@ -53,7 +59,12 @@ impl Service { idle(value)?; let departure = match value.kernel { Kernel::Unattached => Departure::Unattached, - #[cfg(target_os = "linux")] + Kernel::Exported(_) | Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => { + Departure::Export(Box::new(super::export::take( + &mut value.kernel, + layerfs_bridge::control::MountPhase::Probing, + )?)) + } Kernel::Attaching => { return Err(Failure::kernel( layerfs_bridge::control::ControlCode::Unknown, @@ -175,13 +186,16 @@ impl Service { } failure } - /// Attach is a Linux capability; elsewhere it is refused before effects. + /// FUSE Attach is a Linux capability; elsewhere only NFS serving attaches. #[cfg(not(target_os = "linux"))] pub(super) fn attach( &self, - _token: WorkspaceToken, + token: WorkspaceToken, _timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return self.attach_export(&export, token); + } Err(Failure::Rejected( layerfs_bridge::control::ControlCode::Invalid, "kernel serving requires Linux", diff --git a/engine/crates/layerfs-nfs/Cargo.toml b/engine/crates/layerfs-nfs/Cargo.toml new file mode 100644 index 0000000000..13828bf3ae --- /dev/null +++ b/engine/crates/layerfs-nfs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "layerfs-nfs" +description = "Loopback NFSv3 service of one mounted Workspace over the kernel request ports" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +layerfs-content = { path = "../../../core/crates/layerfs-content" } +layerfs-overlay = { path = "../layerfs-overlay" } +layerfs-workspace = { path = "../layerfs-workspace" } +layerfs-fuse = { path = "../layerfs-fuse" } + +[dev-dependencies] +layerfs-bridge = { path = "../layerfs-bridge" } +layerfs-daemon = { path = "../layerfs-daemon" } +layerfs-history = { path = "../../../core/crates/layerfs-history" } +layerfs-persistence = { path = "../layerfs-persistence" } +layerfs-project = { path = "../layerfs-project" } +layerfs-storage = { path = "../../../core/crates/layerfs-storage" } +layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } diff --git a/engine/crates/layerfs-nfs/examples/daemon_nfs.rs b/engine/crates/layerfs-nfs/examples/daemon_nfs.rs new file mode 100644 index 0000000000..2ce0ea9dd1 --- /dev/null +++ b/engine/crates/layerfs-nfs/examples/daemon_nfs.rs @@ -0,0 +1,172 @@ +//! The daemon's control Service with NFS serving, driven by control requests +//! (A3 kernel-client runs). +//! +//! usage: daemon_nfs +//! +//! Initializes a Store from `` (Branch `main`), then runs the +//! Workspace control requests the daemon's control connection answers: +//! Mount and Attach (`BOUND`, `READY `); the client mount, flush +//! and unmount are the helpers named by `LAYERFS_NFS_MOUNT`, +//! `LAYERFS_NFS_FLUSH`, `LAYERFS_NFS_UNMOUNT` and `LAYERFS_NFS_FORCE` +//! (`{port}`, `{dir}` replaced), as in the daemon. `LAYERFS_NFS_LOOKUPS` sets +//! the export's reference bound. Then, whenever a trigger file appears: +//! `.status` prints the Status kernel block and the export's +//! holdings (`STATUS …`); `.commit` runs a control Commit +//! (`COMMITTED …`) and serves an independent view of the committed Branch +//! read-only on `` (`VIEW …`); `.unmount` runs a +//! control Unmount without discard (`UNMOUNT …`), and on success stops. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use layerfs_bridge::control::{CommitControl, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter}; +use layerfs_history::WorkspaceId; +use std::{net::TcpListener, path::PathBuf, sync::Arc, time::Duration}; + +fn main() { + let arguments: Vec = std::env::args().collect(); + if arguments.len() != 7 { + eprintln!("usage: daemon_nfs "); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let mounts = PathBuf::from(&arguments[3]); + let view_port: u16 = arguments[4].parse().expect("view port"); + let uid: u32 = arguments[5].parse().expect("uid"); + let gid: u32 = arguments[6].parse().expect("gid"); + let trigger = |name: &str| PathBuf::from(format!("{}.{name}", work.display())); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!("INIT entries={}", rig.entries); + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper("LAYERFS_NFS_MOUNT") { + mounter.mount = mount; + } + if let Some(unmount) = helper("LAYERFS_NFS_UNMOUNT") { + mounter.unmount = unmount; + } + if let Some(force) = helper("LAYERFS_NFS_FORCE") { + mounter.force = force; + } + mounter.flush = helper("LAYERFS_NFS_FLUSH"); + let mut config = ExportServingConfig::new(mounts, uid, gid, mounter); + if let Some(bound) = std::env::var("LAYERFS_NFS_LOOKUPS") + .ok() + .and_then(|value| value.parse().ok()) + { + config.lookups_held = bound; + } + let service = rig.service(Arc::new(ExportServing::start(config))); + let workspace = WorkspaceId::from_authority([41; 32]).unwrap(); + let token = match service + .execute_control(&Request::Mount { + workspace, + branch: rig.branch, + }) + .expect("Mount") + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + println!("BOUND {token:?}"); + let ready = match service.execute_control(&Request::Attach(token)) { + Ok(success) => match success.reply { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }, + Err(failure) => { + println!("ATTACH_FAILED {failure:?}"); + std::process::exit(1); + } + }; + println!("READY {} {}", ready.receipt.mount_id, ready.directory); + let mut views = Vec::new(); + loop { + if trigger("status").exists() { + match service + .execute_control(&Request::Status(token)) + .map(|s| s.reply) + { + Ok(Reply::Status(status)) => println!( + "STATUS phase={:?} work={:?} dirty_inodes={:?} held={:?}", + status.kernel.as_ref().map(|k| k.phase), + status.kernel.as_ref().and_then(|k| k.work), + status.local.as_ref().map(|l| l.dirty_inodes), + service.export_held(token) + ), + other => println!("STATUS {other:?}"), + } + drop(std::fs::remove_file(trigger("status"))); + } + if trigger("commit").exists() { + let at = std::time::Instant::now(); + let done = service.execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("a3 over nfs".into()), + })); + let ms = at.elapsed().as_millis(); + match done { + Ok(success) => { + let kind = match &success.reply { + Reply::Committed(_) => "Committed", + Reply::MergeCommitted(_) => "MergeCommitted", + Reply::Conflict(_) => "Conflict", + _ => "Other", + }; + println!("COMMITTED {kind} ms={ms}"); + let mut authority = [90; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let port = view_port + views.len() as u16; + let viewer = rig.serve_view( + TcpListener::bind(("127.0.0.1", port)).expect("bind view"), + &view, + uid, + gid, + ); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + } + Err(failure) => println!("COMMIT_FAILED ms={ms} {failure:?}"), + } + drop(std::fs::remove_file(trigger("commit"))); + } + if trigger("unmount").exists() { + drop(std::fs::remove_file(trigger("unmount"))); + let held = service.export_held(token); + match service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: false, + })) { + Ok(success) => { + println!("UNMOUNT {:?} held_before={held:?}", success.reply); + if matches!(success.reply, Reply::Unmounted(_)) { + break; + } + } + Err(failure) => println!("UNMOUNT_FAILED {failure:?}"), + } + } + std::thread::sleep(Duration::from_millis(200)); + } + let released = service.export_released(token).unwrap_or_default(); + println!("RELEASED {released:?}"); + println!( + "DIRECTORY_GONE {}", + !std::path::Path::new(&ready.directory).exists() + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + drop(service); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); +} diff --git a/engine/crates/layerfs-nfs/examples/serve_tree.rs b/engine/crates/layerfs-nfs/examples/serve_tree.rs new file mode 100644 index 0000000000..676d063b45 --- /dev/null +++ b/engine/crates/layerfs-nfs/examples/serve_tree.rs @@ -0,0 +1,101 @@ +//! Serves a host tree, committed as a Branch, over loopback NFSv3. +//! +//! usage: serve_tree [uid gid] [write] +//! +//! Prints `READY
` once the export accepts calls. With `write` the +//! export is writable, and whenever `.commit` appears the export +//! closes its implicit file descriptors, the Workspace is committed with the +//! product Commit (`COMMITTED …`), and a second, independent Workspace bound +//! to the committed Branch is served read-only on `+1` (`VIEW …`); +//! the trigger file is then removed. It serves until `.stop` +//! exists; it then prints per-procedure call counts and time, gives back +//! every engine reference, revokes the engine mounts and removes +//! ``. Mount on Linux with, for example: +//! `mount -t nfs -o vers=3,proto=tcp,port=P,mountport=P,mountproto=tcp,nolock HOST:/ DIR`. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use std::{net::TcpListener, path::PathBuf, time::Duration}; + +fn main() { + let mut arguments: Vec = std::env::args().collect(); + let writable = arguments.last().is_some_and(|last| last == "write"); + if writable { + arguments.pop(); + } + if arguments.len() != 4 && arguments.len() != 6 { + eprintln!("usage: serve_tree [uid gid] [write]"); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let port: u16 = arguments[3].parse().expect("port"); + let (uid, gid) = if arguments.len() == 6 { + (arguments[4].parse().unwrap(), arguments[5].parse().unwrap()) + } else { + (501, 20) + }; + let stop = PathBuf::from(format!("{}.stop", work.display())); + let commit = PathBuf::from(format!("{}.commit", work.display())); + let started = std::time::Instant::now(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!( + "INIT entries={} seconds={:.2}", + rig.entries, + started.elapsed().as_secs_f64() + ); + let listener = TcpListener::bind(("127.0.0.1", port)).expect("bind"); + let server = rig.export(listener, uid, gid, !writable); + println!("READY {} writable={writable}", server.local_addr()); + let mut views = Vec::new(); + while !stop.exists() { + if writable && commit.exists() { + let closed = server.close_files(); + let at = std::time::Instant::now(); + let done = rig.commit(); + println!( + "COMMITTED closed_files={closed} ms={} history={:?} replaced={:?}", + at.elapsed().as_millis(), + done.history, + done.replaced + ); + drop(done); + let mut authority = [80; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let listener = + TcpListener::bind(("127.0.0.1", port + 1 + views.len() as u16)).expect("bind"); + let viewer = rig.serve_view(listener, &view, uid, gid); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + drop(std::fs::remove_file(&commit)); + } + std::thread::sleep(Duration::from_millis(200)); + } + let mut total = (0, 0); + for count in server.counters() { + total.0 += count.calls; + total.1 += count.nanoseconds; + println!( + "PROC {:<12} calls={:<8} total_ms={:<10.1} avg_us={:.1}", + count.procedure, + count.calls, + count.nanoseconds as f64 / 1e6, + count.nanoseconds as f64 / 1e3 / count.calls as f64 + ); + } + println!( + "TOTAL calls={} server_ms={:.1}", + total.0, + total.1 as f64 / 1e6 + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + let released = server.stop(); + println!("RELEASED {released:?}"); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); + drop(std::fs::remove_file(stop)); +} diff --git a/engine/crates/layerfs-nfs/src/apple_double.rs b/engine/crates/layerfs-nfs/src/apple_double.rs new file mode 100644 index 0000000000..b64bba7ab6 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/apple_double.rs @@ -0,0 +1,326 @@ +//! AppleDouble companions held by the export alone. +//! +//! On NFSv3 the macOS client keeps a file's extended attributes and resource +//! fork in a `._` companion file (see [`crate::names`]). When the export +//! keeps such names out of the Workspace, a CREATE of one makes a file that +//! lives only here, in memory, under its own serial range: LOOKUP, GETATTR, +//! ACCESS, READ, WRITE, SETATTR, COMMIT, REMOVE and a RENAME to another +//! AppleDouble name work on it, READDIR does not list it, and no mutation of +//! it ever reaches the engine, so the Workspace stays clean and no Commit can +//! capture it. Everything is dropped when the export is released. +//! +//! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] +//! each and [`TOTAL_BYTES`] in all. Beyond the count a creation is refused +//! (`NFS3ERR_ACCES`, as before this module), beyond the bytes a write +//! (`NFS3ERR_FBIG` per file, `NFS3ERR_NOSPC` in all). +use crate::{ + attributes::Status, + change::{How, Sattr, Tickets}, + kernel::Kernel, +}; +use layerfs_content::{filesystem::attributes::PortableMetadata, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::ViewStat; +use std::{ + collections::HashMap, + sync::atomic::Ordering, + time::{SystemTime, UNIX_EPOCH}, +}; + +/// The first serial of the export's own companions, far above any serial the +/// engine allocates. +pub const SERIALS: u64 = 1 << 62; +/// Companions held at once. +pub const HELD: usize = 4096; +/// Largest companion kept. +pub const FILE_BYTES: u64 = 1 << 20; +/// Bytes kept over all companions. +pub const TOTAL_BYTES: usize = 64 << 20; +/// Mode of a companion created without one. +const MODE: u32 = 0o644; + +struct Companion { + named: (u64, Vec), + data: Vec, + metadata: PortableMetadata, +} +/// The companions of one export. +#[derive(Default)] +pub(crate) struct Companions { + next: u64, + files: HashMap, + names: HashMap<(u64, Vec), u64>, + bytes: usize, +} +impl Companions { + pub(crate) fn len(&self) -> usize { + self.files.len() + } + fn stat(serial: u64, file: &Companion) -> ViewStat { + ViewStat { + serial, + kind: InodeKind::RegularFile, + metadata: file.metadata, + namespace_refs: 1, + logical_len: file.data.len() as u64, + subdirs: 0, + } + } + fn drop_serial(&mut self, serial: u64) { + if let Some(file) = self.files.remove(&serial) { + self.names.remove(&file.named); + self.bytes -= file.data.len(); + } + } + /// Resizes `serial` to `size`, within the bounds. + fn resize(&mut self, serial: u64, size: u64) -> Result<(), Status> { + if size > FILE_BYTES { + return Err(Status::FBig); + } + let file = self.files.get_mut(&serial).ok_or(Status::Stale)?; + let old = file.data.len(); + let new = size as usize; + if new > old && self.bytes + (new - old) > TOTAL_BYTES { + return Err(Status::NoSpc); + } + file.data.resize(new, 0); + self.bytes = self.bytes + new - old; + Ok(()) + } +} +fn now() -> PortableMetadata { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + PortableMetadata { + mode: MODE, + mtime_seconds: i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX), + mtime_nanoseconds: elapsed.subsec_nanos(), + } +} +fn touch(metadata: &mut PortableMetadata) { + let fresh = now(); + metadata.mtime_seconds = fresh.mtime_seconds; + metadata.mtime_nanoseconds = fresh.mtime_nanoseconds; +} +/// A serial of the export's own companions. +pub fn companion(serial: u64) -> bool { + serial >= SERIALS +} + +impl Kernel { + fn companions(&self) -> std::sync::MutexGuard<'_, Companions> { + self.companions + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// GETATTR of a companion serial. + pub(crate) fn companion_stat(&self, serial: u64) -> Result { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + Ok(Companions::stat(serial, file)) + } + /// LOOKUP of a companion name, if one is held. + pub(crate) fn companion_named(&self, parent: u64, name: &[u8]) -> Option { + let companions = self.companions(); + let serial = *companions.names.get(&(parent, name.to_vec()))?; + companions + .files + .get(&serial) + .map(|file| Companions::stat(serial, file)) + } + /// CREATE of an AppleDouble name. + pub(crate) fn companion_create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + if let Some(stat) = self.companion_named(parent, name) { + // Opened again: UNCHECKED applies its size, as an open with + // truncation would; GUARDED and EXCLUSIVE find it taken. + let How::Unchecked(sattr) = how else { + return Err(Status::Exist); + }; + return match sattr.size { + Some(size) => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.companion_setattr(stat.serial, sized, None, tickets) + } + None => Ok(stat), + }; + } + // A directory must exist and a companion the tree itself carries + // stays the tree's: neither is ours to shadow. + let directory = self.getattr(parent)?; + if directory.kind != InodeKind::Directory { + return Err(Status::NotDir); + } + let taken = match self.lookup(parent, name) { + Ok(_) => true, + Err(Status::NoEnt) => false, + Err(status) => return Err(status), + }; + let mut companions = self.companions(); + if taken || companions.len() >= HELD { + drop(companions); + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + let mut metadata = now(); + if let How::Unchecked(sattr) | How::Guarded(sattr) = how { + if let Some(mode) = sattr.mode { + metadata.mode = mode & 0o777; + } + } + let serial = SERIALS + companions.next; + companions.next += 1; + let named = (parent, name.to_vec()); + companions.names.insert(named.clone(), serial); + companions.files.insert( + serial, + Companion { + named, + data: Vec::new(), + metadata, + }, + ); + let file = &companions.files[&serial]; + let stat = Companions::stat(serial, file); + drop(companions); + self.names_detached.fetch_add(1, Ordering::Relaxed); + Ok(stat) + } + /// REMOVE of a companion name; `false` when none is held. + pub(crate) fn companion_remove(&self, parent: u64, name: &[u8]) -> bool { + let mut companions = self.companions(); + match companions.names.get(&(parent, name.to_vec())).copied() { + Some(serial) => { + companions.drop_serial(serial); + true + } + None => false, + } + } + /// RENAME of a companion name: only to another AppleDouble name, which + /// it replaces when held. `None` when no companion has that name. + pub(crate) fn companion_rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + ) -> Option> { + let mut companions = self.companions(); + let serial = *companions.names.get(&(from, name.to_vec()))?; + if !crate::names::apple_double(new_name) || new_name.len() > 255 { + return Some(Err(Status::Access)); + } + let target = (to, new_name.to_vec()); + if let Some(replaced) = companions.names.get(&target).copied() { + if replaced != serial { + companions.drop_serial(replaced); + } + } + companions.names.remove(&(from, name.to_vec())); + companions.names.insert(target.clone(), serial); + if let Some(file) = companions.files.get_mut(&serial) { + file.named = target; + } + Some(Ok(())) + } + /// READ of a companion. + pub(crate) fn companion_read( + &self, + serial: u64, + offset: u64, + count: u32, + ) -> Result<(Vec, bool), Status> { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let length = file.data.len() as u64; + let start = offset.min(length) as usize; + let end = offset.saturating_add(u64::from(count)).min(length) as usize; + Ok((file.data[start..end].to_vec(), end as u64 >= length)) + } + /// WRITE to a companion: all of it or nothing. + pub(crate) fn companion_write( + &self, + serial: u64, + offset: u64, + data: &[u8], + ) -> Result<(u32, ViewStat), Status> { + let mut companions = self.companions(); + let end = offset.checked_add(data.len() as u64).ok_or(Status::FBig)?; + let length = companions + .files + .get(&serial) + .ok_or(Status::Stale)? + .data + .len() as u64; + if end > length { + companions.resize(serial, end)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + file.data[offset as usize..end as usize].copy_from_slice(data); + touch(&mut file.metadata); + let stat = Companions::stat(serial, file); + Ok((data.len() as u32, stat)) + } + /// SETATTR of a companion: mode, size and modification time; the + /// owner is the export's identity, as for every inode. + pub(crate) fn companion_setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + _tickets: &mut Tickets, + ) -> Result { + let identity = self.identity(); + let foreign = sattr.uid.is_some_and(|uid| uid != identity.uid) + || sattr.gid.is_some_and(|gid| gid != identity.gid); + if foreign { + return Err(Status::Perm); + } + let mut companions = self.companions(); + if let Some(expected) = guard { + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let current = Companions::stat(serial, file); + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if let Some(size) = sattr.size { + companions.resize(serial, size)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + if let Some(mode) = sattr.mode { + file.metadata.mode = mode & 0o777; + } + match sattr.mtime { + Some(Stamp::At(time)) => { + file.metadata.mtime_seconds = time.seconds; + file.metadata.mtime_nanoseconds = time.nanoseconds; + } + Some(Stamp::Processing) => touch(&mut file.metadata), + None if sattr.size.is_some() => touch(&mut file.metadata), + None => {} + } + Ok(Companions::stat(serial, file)) + } + /// Drops every companion; their count. + pub(crate) fn companions_release(&self) -> usize { + let mut companions = self.companions(); + let held = companions.len(); + *companions = Companions::default(); + held + } + /// Companions held now. + pub(crate) fn companions_held(&self) -> usize { + self.companions().len() + } +} diff --git a/engine/crates/layerfs-nfs/src/attributes.rs b/engine/crates/layerfs-nfs/src/attributes.rs new file mode 100644 index 0000000000..70fd740c1d --- /dev/null +++ b/engine/crates/layerfs-nfs/src/attributes.rs @@ -0,0 +1,204 @@ +//! NFSv3 (RFC 1813) status codes and the projection of Workspace inode facts. +use crate::xdr::Writer; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_workspace::{Refusal, ViewStat}; + +/// `nfsstat3` values this server returns. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum Status { + Ok = 0, + Perm = 1, + NoEnt = 2, + Io = 5, + Access = 13, + Exist = 17, + NotDir = 20, + IsDir = 21, + Inval = 22, + FBig = 27, + NoSpc = 28, + RoFs = 30, + MLink = 31, + NameTooLong = 63, + NotEmpty = 66, + Stale = 70, + BadHandle = 10001, + NotSync = 10002, + BadCookie = 10003, + NotSupp = 10004, + TooSmall = 10005, + ServerFault = 10006, + /// The request may succeed later; the client retries after a delay. + Jukebox = 10008, +} +impl Status { + pub const fn code(self) -> u32 { + self as u32 + } +} +/// A Workspace refusal as the status the client sees; the same errnos the +/// FUSE adapter replies with. +pub fn refusal(value: Refusal) -> Status { + match value { + Refusal::Exists => Status::Exist, + Refusal::Missing => Status::NoEnt, + Refusal::NotDirectory => Status::NotDir, + Refusal::IsDirectory => Status::IsDir, + Refusal::NotEmpty => Status::NotEmpty, + Refusal::NotPermitted => Status::Perm, + Refusal::Unsupported => Status::NotSupp, + Refusal::Invalid | Refusal::AncestryRequired | Refusal::AncestryMismatch => Status::Inval, + Refusal::TooManyLinks => Status::MLink, + Refusal::TooLarge => Status::FBig, + } +} + +/// `ftype3` of a Workspace inode kind. +pub const fn file_type(kind: InodeKind) -> u32 { + match kind { + InodeKind::RegularFile => 1, + InodeKind::Directory => 2, + InodeKind::Symlink => 5, + } +} + +/// The identity every inode is shown as, and the export's fixed facts. +#[derive(Clone, Copy, Debug)] +pub struct Identity { + /// Serial of the mount's root directory. + pub root: u64, + pub uid: u32, + pub gid: u32, + /// `fsid` of every attribute; one export is one filesystem. + pub fsid: u64, +} +impl Identity { + /// POSIX link count, as the FUSE adapter projects it: a directory's is + /// derived (0 once removed, else two plus its child directories); any + /// other kind's is its namespace reference count. + pub fn nlink(self, value: &ViewStat) -> u32 { + if value.kind == InodeKind::Directory { + if value.namespace_refs == 0 && value.serial != self.root { + 0 + } else { + u32::try_from(value.subdirs.saturating_add(2)).unwrap_or(u32::MAX) + } + } else { + u32::try_from(value.namespace_refs).unwrap_or(u32::MAX) + } + } + /// Encodes one `fattr3`. The Workspace keeps one timestamp; atime, mtime + /// and ctime all show it. `fileid` is the serial itself. + pub fn fattr3(self, out: &mut Writer, value: &ViewStat) { + let (seconds, nanoseconds) = time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ); + let used = value.logical_len.div_ceil(512).saturating_mul(512); + out.u32(file_type(value.kind)); + out.u32(value.metadata.mode & 0o7777); + out.u32(self.nlink(value)); + out.u32(self.uid).u32(self.gid); + out.u64(value.logical_len).u64(used); + out.u32(0).u32(0); // rdev + out.u64(self.fsid).u64(value.serial); + for _ in 0..3 { + out.u32(seconds).u32(nanoseconds); + } + } + /// `post_op_attr` carrying attributes. + pub fn post_op(self, out: &mut Writer, value: &ViewStat) { + out.bool(true); + self.fattr3(out, value); + } +} +/// The `ctime` a client sees for `value` (the one stored timestamp), as the +/// SETATTR guard compares it. +pub fn ctime(value: &ViewStat) -> (u32, u32) { + time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ) +} +/// `wcc_attr` (size, mtime, ctime) of a `pre_op_attr` carrying attributes. +pub fn pre_op(out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => { + let (seconds, nanoseconds) = ctime(value); + out.bool(true).u64(value.logical_len); + out.u32(seconds).u32(nanoseconds); + out.u32(seconds).u32(nanoseconds); + } + None => { + out.bool(false); + } + } +} +/// `post_op_attr`, present or absent. +pub fn post_op_maybe(identity: Identity, out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => identity.post_op(out, value), + None => { + out.bool(false); + } + } +} +/// `nfstime3` holds unsigned seconds; times before the epoch show as the epoch. +fn time(seconds: i64, nanoseconds: u32) -> (u32, u32) { + match u32::try_from(seconds) { + Ok(seconds) => (seconds, nanoseconds.min(999_999_999)), + Err(_) if seconds < 0 => (0, 0), + Err(_) => (u32::MAX, 999_999_999), + } +} + +/// ACCESS3 bits. +pub const ACCESS_READ: u32 = 0x01; +pub const ACCESS_LOOKUP: u32 = 0x02; +pub const ACCESS_MODIFY: u32 = 0x04; +pub const ACCESS_EXTEND: u32 = 0x08; +pub const ACCESS_DELETE: u32 = 0x10; +pub const ACCESS_EXECUTE: u32 = 0x20; + +/// The ACCESS answer for a caller. Every inode belongs to the one command +/// identity; a caller with that uid gets the owner bits, a caller in that +/// group the group bits, anyone else the other bits. uid 0 gets no +/// superuser override: LayerFS has no root semantics, and root inside a +/// sandbox must not bypass the mode. A read-only export grants no change. +pub fn access( + identity: Identity, + value: &ViewStat, + caller: Option<(u32, u32, &[u32])>, + requested: u32, + read_only: bool, +) -> u32 { + let mode = value.metadata.mode; + let bits = match caller { + Some((uid, _, _)) if uid == identity.uid || uid == 0 => (mode >> 6) & 7, + Some((_, gid, groups)) if gid == identity.gid || groups.contains(&identity.gid) => { + (mode >> 3) & 7 + } + _ => mode & 7, + }; + let (read, write, execute) = (bits & 4 != 0, bits & 2 != 0, bits & 1 != 0); + let directory = value.kind == InodeKind::Directory; + let mut granted = 0; + if read { + granted |= ACCESS_READ; + } + if write && !read_only { + granted |= ACCESS_MODIFY | ACCESS_EXTEND; + if directory { + granted |= ACCESS_DELETE; + } + } + if execute { + granted |= if directory { + ACCESS_LOOKUP + } else { + ACCESS_EXECUTE + }; + } + granted & requested +} diff --git a/engine/crates/layerfs-nfs/src/block.rs b/engine/crates/layerfs-nfs/src/block.rs new file mode 100644 index 0000000000..7449bf9d34 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/block.rs @@ -0,0 +1,32 @@ +//! Runs one port future to completion on the connection thread. +//! +//! The service ports return boxed futures; the daemon's implementations run +//! their owner job on the calling thread and are ready on first poll. A +//! future that is not ready parks this thread until its waker unparks it. +use std::{ + future::Future, + sync::Arc, + task::{Context, Poll, Wake, Waker}, + thread::{self, Thread}, +}; + +struct Unpark(Thread); +impl Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + fn wake_by_ref(self: &Arc) { + self.0.unpark(); + } +} +pub fn wait(future: impl Future) -> T { + let mut future = std::pin::pin!(future); + let waker = Waker::from(Arc::new(Unpark(thread::current()))); + let mut context = Context::from_waker(&waker); + loop { + if let Poll::Ready(value) = future.as_mut().poll(&mut context) { + return value; + } + thread::park(); + } +} diff --git a/engine/crates/layerfs-nfs/src/change.rs b/engine/crates/layerfs-nfs/src/change.rs new file mode 100644 index 0000000000..a1c5008d16 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/change.rs @@ -0,0 +1,549 @@ +//! The changing procedures over the kernel mutation ports: one decided plan +//! per engine visit, and every publication ticket released only after the +//! call's single reply attempt. +//! +//! Each procedure runs [`KernelMutation::perform`], the same orchestration +//! the FUSE connection uses: the serial of a creating operation is reserved +//! there (an exact allocator contention is the declined `Contended`, served +//! as `NFS3ERR_JUKEBOX` so the client retries later with nothing created), +//! need rounds read base facts, and the one publishing visit hands back a +//! [`KernelMutation`] that still owns its ticket. The procedure puts it in +//! the call's [`Tickets`]; the connection releases them with +//! [`Tickets::replied`] after `write_record` returned, whether or not the +//! reply reached the client. A failure that keeps custody is retained, never +//! released or undone on a guess; a fenced step or a failed base demand of +//! this request alone is relinquished, as the FUSE connection does. +use crate::{ + attributes::{refusal, Status}, + block::wait, + kernel::Kernel, + names, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::{ + create, link, remove, rename, + write::{self, AttributeChange, Stamp}, + Declined, KernelMutation, MutationFailure, MutationInput, MutationRequest, +}; +use layerfs_overlay::WRITE_WINDOW; +use layerfs_workspace::{Time, ViewStat}; +use std::{io, sync::atomic::Ordering, time::SystemTime}; + +/// The page a client reads a symlink target into; a longer target could not +/// be read back whole and is refused at creation, as on the FUSE side. +pub const LINK_PAGE: u32 = 4096; +/// Modes when a CREATE or MKDIR names none (clients normally send one). +const FILE_MODE: u32 = 0o644; +const DIRECTORY_MODE: u32 = 0o755; +/// EXCLUSIVE CREATE carries no attributes; the client sets them right after. +const EXCLUSIVE_MODE: u32 = 0o600; + +/// Mutations decided while serving one call, each owed exactly one release +/// of its publication ticket after that call's reply attempt. A declined or +/// unchanged mutation holds no ticket; its release issues no job. +#[derive(Default)] +#[must_use = "every decided mutation is owed its post-reply release"] +pub struct Tickets(Vec); +impl Tickets { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + /// Called only after the reply's single send attempt returned. A release + /// that fails is retained with its remaining custody. + pub fn replied(self, kernel: &Kernel) { + for mutation in self.0 { + if let Err(failure) = wait(mutation.replied()) { + kernel.retain(failure); + } + } + } +} + +/// The settable attributes of `sattr3`. Access time is accepted and not +/// stored (the Workspace keeps one timestamp; the mount is `noatime`). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Sattr { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + pub mtime: Option, +} +impl Sattr { + fn change(self) -> AttributeChange { + AttributeChange { + mode: self.mode, + owner: self.uid, + group: self.gid, + size: self.size, + mtime: self.mtime, + } + } + fn is_empty(self) -> bool { + self.change() == AttributeChange::default() + } +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Sattr), + Guarded(Sattr), + Exclusive([u8; 8]), +} +/// What a published result must carry for its reply to be composed. +#[derive(Clone, Copy)] +enum Expect { + Nothing, + /// The published inode, with this serial when given. + Stat(Option), +} +/// A declined mutation as the status the client sees. +pub fn declined(reason: Declined) -> Status { + match reason { + Declined::Refused(value) => refusal(value), + Declined::NoSpace => Status::NoSpc, + Declined::Contended => Status::Jukebox, + Declined::TargetTooLong => Status::NameTooLong, + } +} + +impl Kernel { + /// One kernel mutation. Its decided [`KernelMutation`] goes to `tickets` + /// whatever the answer; a failure is relinquished or retained. + fn mutate( + &self, + protected: u64, + handle: Option, + expect: Expect, + tickets: &mut Tickets, + build: impl FnOnce(Time) -> Result, + ) -> Result, Status> { + let now = write::time(SystemTime::now()).ok_or(Status::Io)?; + let input = build(now).map_err(declined)?; + let (services, request) = self.request()?; + let request = MutationRequest { + mount: self.mount(), + request, + protected, + handle, + input, + open: None, + now, + }; + let mutation = match wait(KernelMutation::perform(services, request)) { + Ok(mutation) => mutation, + Err(failure) => { + let status = if crate::kernel::failure_is_stale(&failure.reason) { + Status::Stale + } else { + Status::Io + }; + self.failed(failure); + return Err(status); + } + }; + let stat = match mutation.value() { + Err(reason) => { + tickets.0.push(mutation); + return Err(declined(reason)); + } + Ok(published) => published.stat.clone(), + }; + if let Expect::Stat(serial) = expect { + let composable = stat + .as_ref() + .is_some_and(|stat| serial.is_none_or(|serial| serial == stat.serial)); + if !composable { + // The reply cannot be composed from the published result: + // an error for the client and a retained mutation. + let reason = io::Error::other("NFS reply composition").into(); + self.retain(mutation.retain(reason)); + return Err(Status::Io); + } + } + tickets.0.push(mutation); + Ok(stat) + } + fn failed(&self, failure: MutationFailure) { + if failure.fenced() || failure.base_demand().is_some() { + if let Err(failure) = wait(failure.relinquish()) { + self.retain(failure); + } + } else { + self.retain(failure); + } + } + /// A name a changing procedure may bind (`creating`) or remove. + fn checked(&self, name: &[u8], creating: bool) -> Result { + match name { + b"" => return Err(Status::Inval), + b"." | b".." => { + return Err(if creating { + Status::Exist + } else { + Status::Inval + }) + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ if creating && self.refuses_apple_double() && names::apple_double(name) => { + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + _ => {} + } + PathName::from_bytes(name).map_err(|_| Status::Inval) + } + /// The entry a name binds now, or none. + fn existing(&self, parent: u64, name: &[u8]) -> Result, Status> { + match self.lookup(parent, name) { + Ok(stat) => Ok(Some(stat)), + Err(Status::NoEnt) => Ok(None), + Err(status) => Err(status), + } + } + /// CREATE of a regular file. + pub fn create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + if self.refuses_apple_double() && names::apple_double(name) && name.len() <= 255 { + return self.companion_create(parent, name, how, tickets); + } + let checked = self.checked(name, true)?; + let key = (parent, name.to_vec()); + if let How::Exclusive(verifier) = how { + // A retransmitted EXCLUSIVE CREATE finds its own file. + let known = self.table().verifiers.get(&key).copied(); + if let Some((stored, serial, _)) = known { + if stored == verifier { + if let Some(stat) = self.existing(parent, name)? { + if stat.serial == serial { + return Ok(stat); + } + } + } + } + } + let (mode, rest) = match how { + How::Unchecked(sattr) | How::Guarded(sattr) => ( + sattr.mode.unwrap_or(FILE_MODE), + Sattr { + mode: None, + ..sattr + }, + ), + How::Exclusive(_) => (EXCLUSIVE_MODE, Sattr::default()), + }; + let created = self.mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::create(parent, checked, mode))) + }); + let stat = match (created, how) { + (Ok(stat), _) => stat.ok_or(Status::ServerFault)?, + (Err(Status::Exist), How::Unchecked(sattr)) => { + // UNCHECKED over an existing regular file: only its size is + // applied, as an open with truncation would. + let stat = self.existing(parent, name)?.ok_or(Status::Exist)?; + if stat.kind != InodeKind::RegularFile { + return Err(Status::Exist); + } + return match sattr.size { + Some(size) if size != stat.logical_len => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.setattr(stat.serial, sized, None, tickets) + } + _ => Ok(stat), + }; + } + (Err(status), _) => return Err(status), + }; + self.entry_taken(&stat, parent, name); + if let How::Exclusive(verifier) = how { + let mut table = self.table(); + let tick = table.tick(); + if table.verifiers.len() >= crate::kernel::VERIFIERS_HELD { + let oldest = table + .verifiers + .iter() + .min_by_key(|(_, entry)| entry.2) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + table.verifiers.remove(&oldest); + } + } + table.verifiers.insert(key, (verifier, stat.serial, tick)); + } + self.settle(stat, rest, tickets) + } + /// Applies what a creating call's attributes ask beyond the mode. + fn settle( + &self, + stat: ViewStat, + rest: Sattr, + tickets: &mut Tickets, + ) -> Result { + let rest = Sattr { + size: rest.size.filter(|size| *size != stat.logical_len), + ..rest + }; + if rest.is_empty() { + return Ok(stat); + } + self.setattr(stat.serial, rest, None, tickets) + } + /// MKDIR. + pub fn mkdir( + &self, + parent: u64, + name: &[u8], + sattr: Sattr, + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let mode = sattr.mode.unwrap_or(DIRECTORY_MODE); + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::mkdir(parent, checked, mode))) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + let rest = Sattr { + mode: None, + size: None, + ..sattr + }; + self.settle(stat, rest, tickets) + } + /// SYMLINK. A symlink's mode is fixed; its attributes are not applied. + pub fn symlink( + &self, + parent: u64, + name: &[u8], + target: &[u8], + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + create::symlink(parent, checked, target, LINK_PAGE).map(MutationInput::Named) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// The silly-named serial `name` denotes in `parent`, if any. + fn silly_entry(&self, parent: u64, name: &[u8]) -> Option { + self.table().silly.get(&(parent, name.to_vec())).copied() + } + /// Drops one silly name; the inode goes back once no silly name keeps it. + fn drop_silly(&self, parent: u64, name: &[u8]) { + let serial = { + let mut table = self.table(); + let serial = table.silly.remove(&(parent, name.to_vec())); + serial.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = serial { + self.unlinked(serial); + } + } + /// REMOVE of a non-directory name. + pub fn remove(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + if self.companion_remove(parent, name) { + return Ok(()); + } + if self.silly_entry(parent, name).is_some() { + self.drop_silly(parent, name); + return Ok(()); + } + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(parent, checked))) + })?; + if target.namespace_refs <= 1 && !self.table().pinned(target.serial) { + self.unlinked(target.serial); + } + Ok(()) + } + /// RMDIR. + pub fn rmdir(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::rmdir(parent, checked))) + })?; + self.unlinked(target.serial); + Ok(()) + } + /// RENAME, replacing the destination. A rename onto a client silly name + /// is the source's unlink with the inode kept alive by the export. + pub fn rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + tickets: &mut Tickets, + ) -> Result<(), Status> { + if let Some(renamed) = self.companion_rename(from, name, to, new_name) { + return renamed; + } + if let Some(serial) = self.silly_entry(from, name) { + // Moving a silly name to another silly name keeps it ours; + // giving it a real name again would need a link of an unlinked + // inode, which the Workspace refuses. + if !names::silly(new_name) { + return Err(Status::Access); + } + let mut table = self.table(); + table.silly.remove(&(from, name.to_vec())); + table.silly.insert((to, new_name.to_vec()), serial); + return Ok(()); + } + let source_name = self.checked(name, false)?; + let destination_name = self.checked(new_name, true)?; + let source = self.existing(from, name)?.ok_or(Status::NoEnt)?; + if names::silly(new_name) && source.kind == InodeKind::RegularFile { + // Keep a writable descriptor before the last name may go. + self.open_file(source.serial, true)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(from, source_name))) + })?; + self.silly_renamed.fetch_add(1, Ordering::Relaxed); + let replaced = { + let mut table = self.table(); + let replaced = table.silly.insert((to, new_name.to_vec()), source.serial); + replaced.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = replaced { + self.unlinked(serial); + } + return Ok(()); + } + let destination = self.existing(to, new_name)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + rename::rename(from, source_name, to, destination_name, 0).map(MutationInput::Named) + })?; + if source.kind == InodeKind::Directory { + self.saw_directory(source.serial, to); + } + self.renamed(source.serial, to, new_name); + if let Some(replaced) = destination.filter(|stat| stat.serial != source.serial) { + let last = replaced.kind == InodeKind::Directory || replaced.namespace_refs <= 1; + if last && !self.table().pinned(replaced.serial) { + self.unlinked(replaced.serial); + } + } + Ok(()) + } + /// LINK: one more name for `serial`. + pub fn link( + &self, + serial: u64, + parent: u64, + name: &[u8], + tickets: &mut Tickets, + ) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::Access); + } + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(Some(serial)), tickets, |_| { + Ok(MutationInput::Named(link::link(serial, parent, checked))) + })? + .ok_or(Status::ServerFault)?; + // The engine took one more lookup reference; the export holds one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// SETATTR. `guard` is the client's expected ctime (seconds, + /// nanoseconds), compared with a fresh GETATTR first: the check is not + /// atomic with the change. + pub fn setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + tickets: &mut Tickets, + ) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_setattr(serial, sattr, guard, tickets); + } + if let Some(expected) = guard { + let current = self.getattr(serial)?; + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if sattr.is_empty() || self.refreshed(serial, &sattr) { + return self.getattr(serial); + } + let identity = (self.identity().uid, self.identity().gid); + let change = sattr.change(); + // A silly-named inode has no name left: a size change goes through + // its descriptor, as an ftruncate would. + let pinned = self.table().pinned(serial); + let handle = if pinned && change.size.is_some() { + Some(self.open_file(serial, true)?.handle) + } else { + None + }; + let stat = self + .mutate(serial, handle, Expect::Stat(Some(serial)), tickets, |now| { + write::set_attributes(serial, handle.is_some(), identity, change, now) + })? + .ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok(stat) + } + /// WRITE through the implicit writable descriptor, in engine windows of + /// at most `WRITE_WINDOW` bytes, each its own publication. A failure after + /// the first window is a short write of the windows already published. + pub fn write( + &self, + serial: u64, + offset: u64, + data: &[u8], + tickets: &mut Tickets, + ) -> Result<(u32, ViewStat), Status> { + if crate::apple_double::companion(serial) { + return self.companion_write(serial, offset, data); + } + let open = self.open_file(serial, true)?; + if data.is_empty() { + return Ok((0, self.getattr(serial)?)); + } + let mut written = 0usize; + let mut last = None; + for window in data.chunks(WRITE_WINDOW) { + let at = offset.checked_add(written as u64).ok_or(Status::FBig)?; + let published = self.mutate( + serial, + Some(open.handle), + Expect::Stat(Some(serial)), + tickets, + |_| write::write(at, window, false), + ); + match published { + Ok(stat) => { + written += window.len(); + last = stat; + } + Err(status) if written == 0 => return Err(status), + Err(_) => break, + } + } + let stat = last.ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok((u32::try_from(written).unwrap_or(u32::MAX), stat)) + } +} diff --git a/engine/crates/layerfs-nfs/src/changing.rs b/engine/crates/layerfs-nfs/src/changing.rs new file mode 100644 index 0000000000..408c914639 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/changing.rs @@ -0,0 +1,268 @@ +//! Arguments and results of the changing NFSv3 procedures (RFC 1813): +//! SETATTR, WRITE, CREATE, MKDIR, SYMLINK, MKNOD, REMOVE, RMDIR, RENAME, +//! LINK and COMMIT. +//! +//! Every directory a procedure changes is answered with `wcc_data`: its +//! attributes read just before the change and just after. The two reads are +//! separate visits, not atomic with the change; the connection serves one +//! call at a time, so they bracket it exactly unless another connection or +//! a Commit installs a change in between, in which case the client sees an +//! unexplained difference and drops its cache, which is safe. +//! +//! WRITE answers `FILE_SYNC`: every window was published to the Workspace +//! before the reply, as every FUSE write is. That is publication, not +//! durability; the write verifier changes with every export, so a client +//! that ever sent UNSTABLE data resends it to a restarted export. +use crate::{ + attributes::{post_op_maybe, pre_op, Status}, + change::{How, Sattr, Tickets}, + kernel::Kernel, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::{Time, ViewStat}; + +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Longest symlink target a SYMLINK may carry. +const PATH_LIMIT: usize = 64 * 1024; +/// Longest WRITE payload accepted in one call (two engine windows of slack +/// over what FSINFO declares). +const WRITE_LIMIT: usize = 1 << 20; +/// `stable_how` FILE_SYNC. +const FILE_SYNC: u32 = 2; + +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +fn time_how(arguments: &mut Reader<'_>) -> Result, Garbage> { + Ok(match arguments.u32()? { + 0 => None, + 1 => Some(Stamp::Processing), + 2 => { + let seconds = arguments.u32()?; + let nanoseconds = arguments.u32()?; + Some(Stamp::At(Time { + seconds: i64::from(seconds), + nanoseconds: nanoseconds.min(999_999_999), + })) + } + _ => return Err(Garbage), + }) +} +/// `sattr3`. The access time is decoded and dropped. +pub fn sattr(arguments: &mut Reader<'_>) -> Result { + let mode = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let uid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let gid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let size = arguments.bool()?.then(|| arguments.u64()).transpose()?; + let _atime = time_how(arguments)?; + let mtime = time_how(arguments)?; + Ok(Sattr { + mode, + uid, + gid, + size, + mtime, + }) +} +/// `wcc_data` of one object around a change. +fn wcc(kernel: &Kernel, out: &mut Writer, before: Option<&ViewStat>, after: Option<&ViewStat>) { + pre_op(out, before); + post_op_maybe(kernel.identity(), out, after); +} +/// Attributes of `serial` when the handle was accepted. +fn attributes(kernel: &Kernel, serial: &Result) -> Option { + serial + .as_ref() + .ok() + .and_then(|serial| kernel.getattr(*serial).ok()) +} +/// The results of CREATE, MKDIR and SYMLINK. +fn created( + kernel: &Kernel, + out: &mut Writer, + result: Result, + directory: &Result, + before: Option<&ViewStat>, +) { + let after = attributes(kernel, directory); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.bool(true).opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + wcc(kernel, out, before, after.as_ref()); +} +/// Decodes the arguments of changing `procedure`, runs it and appends its +/// results. Mutations it decides go to `tickets`, released by the caller +/// after the reply attempt. `Err` is a garbage-arguments refusal. +pub fn call( + kernel: &Kernel, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Garbage> { + match procedure { + 2 => { + let serial = handle(kernel, arguments)?; + let change = sattr(arguments)?; + let guard = if arguments.bool()? { + Some((arguments.u32()?, arguments.u32()?)) + } else { + None + }; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.setattr(serial, change, guard, tickets)); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 7 => { + let serial = handle(kernel, arguments)?; + let offset = arguments.u64()?; + let count = arguments.u32()? as usize; + let _stable = arguments.u32()?; + let data = arguments.opaque(WRITE_LIMIT)?; + let data = &data[..count.min(data.len())]; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.write(serial, offset, data, tickets)); + match result { + Ok((written, stat)) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + out.u32(written).u32(FILE_SYNC).fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 8..=10 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match procedure { + 8 => { + let how = match arguments.u32()? { + 0 => How::Unchecked(sattr(arguments)?), + 1 => How::Guarded(sattr(arguments)?), + 2 => { + let mut verifier = [0; 8]; + verifier.copy_from_slice(arguments.fixed(8)?); + How::Exclusive(verifier) + } + _ => return Err(Garbage), + }; + directory.and_then(|parent| kernel.create(parent, &name, how, tickets)) + } + 9 => { + let change = sattr(arguments)?; + directory.and_then(|parent| kernel.mkdir(parent, &name, change, tickets)) + } + _ => { + let _attributes = sattr(arguments)?; + let target = arguments.opaque(PATH_LIMIT)?.to_vec(); + directory.and_then(|parent| kernel.symlink(parent, &name, &target, tickets)) + } + }; + created(kernel, out, result, &directory, before.as_ref()); + } + 11 => { + // Only regular files, directories and symlinks exist in the + // canonical format: no device, socket or FIFO node. + let directory = handle(kernel, arguments)?; + let _name = arguments.opaque(NAME_LIMIT)?; + let before = attributes(kernel, &directory); + let status = directory.err().unwrap_or(Status::NotSupp); + created(kernel, out, Err(status), &directory, before.as_ref()); + } + 12 | 13 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = directory.and_then(|parent| { + if procedure == 12 { + kernel.remove(parent, &name, tickets) + } else { + kernel.rmdir(parent, &name, tickets) + } + }); + let after = attributes(kernel, &directory); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 14 => { + let from = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let to = handle(kernel, arguments)?; + let new_name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before_from = attributes(kernel, &from); + let before_to = attributes(kernel, &to); + let result = match (&from, &to) { + (Ok(from), Ok(to)) => kernel.rename(*from, &name, *to, &new_name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after_from = attributes(kernel, &from); + let after_to = attributes(kernel, &to); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before_from.as_ref(), after_from.as_ref()); + wcc(kernel, out, before_to.as_ref(), after_to.as_ref()); + } + 15 => { + let file = handle(kernel, arguments)?; + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match (&file, &directory) { + (Ok(file), Ok(parent)) => kernel.link(*file, *parent, &name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after = attributes(kernel, &directory); + let (status, object) = match result { + Ok(stat) => (Status::Ok, Some(stat)), + Err(status) => (status, attributes(kernel, &file)), + }; + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, object.as_ref()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 21 => { + // Every WRITE was answered FILE_SYNC, so nothing is uncommitted. + let serial = handle(kernel, arguments)?; + let _offset = arguments.u64()?; + let _count = arguments.u32()?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, None, Some(&stat)); + out.fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + wcc(kernel, out, None, None); + } + } + } + _ => return Err(Garbage), + } + Ok(()) +} diff --git a/engine/crates/layerfs-nfs/src/directory.rs b/engine/crates/layerfs-nfs/src/directory.rs new file mode 100644 index 0000000000..92a984bbb0 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/directory.rs @@ -0,0 +1,138 @@ +//! READDIR and READDIRPLUS over the engine's published directory cookies. +//! +//! The NFS cookie is the engine's kernel offset (1 for `.`, 2 for `..`, then +//! the offsets the engine publishes) on one implicit directory descriptor, +//! named to the client by `cookieverf`. Cookie 0 opens a fresh descriptor and +//! a fresh verifier. A verifier whose descriptor was closed, or an offset a +//! rewind retired, is `NFS3ERR_BAD_COOKIE`; the client then lists again from 0. +//! +//! One reply carries at most one engine window: exactly the names that fit +//! the client's byte budget are published, and the reply is not at EOF. The +//! next call resumes after the last accepted cookie; a call that finds no +//! names left replies EOF. READDIRPLUS entries carry no attributes and no +//! handle (both optional in RFC 1813), so the client looks each name up. +use crate::{ + attributes::Status, + block::wait, + kernel::{failure, Kernel}, + xdr::{opaque_size, Writer}, +}; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_fuse::operations::{DirectoryFailure, DirectoryStep, DirectoryStream}; + +/// One enumeration request. +pub struct Listing { + pub serial: u64, + pub cookie: u64, + pub verifier: u64, + /// The client's byte budget for the whole reply (`count` / `maxcount`). + pub budget: usize, + /// READDIRPLUS: each entry also carries two absent optional fields. + pub plus: bool, +} +/// Bytes of the reply around the entries: status, absent post_op_attr, +/// cookieverf, the list terminator and the EOF flag, plus the RPC reply header. +const FIXED: usize = 4 + 4 + 8 + 4 + 4 + 24; + +struct Entries { + out: Writer, + used: usize, + budget: usize, + plus: bool, + count: usize, +} +impl Entries { + fn size(&self, name: &[u8]) -> usize { + 4 + 8 + opaque_size(name.len()) + 8 + if self.plus { 8 } else { 0 } + } + /// Appends one entry when it fits the budget. + fn push(&mut self, fileid: u64, name: &[u8], cookie: u64) -> bool { + let size = self.size(name); + if self.used + size > self.budget { + return false; + } + self.used += size; + self.count += 1; + self.out.bool(true).u64(fileid).opaque(name).u64(cookie); + if self.plus { + self.out.bool(false).bool(false); + } + true + } +} + +/// The encoded `READDIR3resok`/`READDIRPLUS3resok` body after the status, +/// or the failure status. +pub fn list(kernel: &Kernel, request: Listing) -> Result, Status> { + let handle = if request.cookie == 0 { + kernel.open_directory(request.serial)? + } else { + kernel + .directory(request.serial, request.verifier) + .ok_or(Status::BadCookie)? + }; + let (services, id) = kernel.request()?; + let stream = wait(DirectoryStream::prepare( + services, + kernel.mount(), + id, + request.serial, + handle, + request.cookie, + )) + .map_err(directory_failure)?; + let mut entries = Entries { + out: Writer::with_capacity(request.budget.min(64 * 1024)), + used: FIXED, + budget: request.budget, + plus: request.plus, + count: 0, + }; + let dots: Vec<(&str, u64, u64)> = stream.dots().collect(); + for (name, serial, cookie) in dots { + if name == ".." && serial != request.serial { + kernel.saw_directory(request.serial, serial); + } + if !entries.push(serial, name.as_bytes(), cookie) { + return finish(entries, handle, false); + } + } + let eof = match wait(stream.next()).map_err(directory_failure)? { + DirectoryStep::End(_) => true, + DirectoryStep::Batch(batch) => { + let mut accepted = 0; + for (entry, cookie) in batch.entries() { + if !entries.push(entry.serial, &entry.name, cookie) { + break; + } + if entry.kind == InodeKind::Directory { + kernel.saw_directory(entry.serial, request.serial); + } + accepted += 1; + } + // Publishing ends this reply's window even when names remain. + drop(wait(batch.accept(accepted)).map_err(directory_failure)?); + false + } + }; + finish(entries, handle, eof) +} +fn finish(entries: Entries, verifier: u64, eof: bool) -> Result, Status> { + if entries.count == 0 && !eof { + return Err(Status::TooSmall); + } + let mut out = Writer::with_capacity(entries.out.len() + 24); + out.bool(false); // dir_attributes + out.u64(verifier); + out.fixed(entries.out.as_bytes()); + out.bool(false); // no further entry + out.bool(eof); + Ok(out.into_bytes()) +} +fn directory_failure(failure: DirectoryFailure) -> Status { + if failure.rewound() { + Status::BadCookie + } else { + self::failure(failure.reason) + } +} diff --git a/engine/crates/layerfs-nfs/src/handle.rs b/engine/crates/layerfs-nfs/src/handle.rs new file mode 100644 index 0000000000..1599a69205 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/handle.rs @@ -0,0 +1,69 @@ +//! Stable NFS file handles for one kernel mount. +//! +//! A handle is `[version][mount owner][serial][tag]`: 25 bytes, well inside +//! the 64-byte NFSv3 limit. The same serial always encodes to the same bytes +//! for the life of one mount, so a client may compare handles. The mount owner +//! is fresh for every engine mount; a handle from an earlier mount decodes as +//! stale. The tag is a keyed hash under a key drawn once per export, so a +//! local process cannot forge a handle for a serial it was never given. It is +//! not a cryptographic MAC (std SipHash, no third-party dependency); it raises +//! the cost of guessing, it does not authenticate the caller. +use std::{collections::hash_map::RandomState, hash::BuildHasher}; + +const VERSION: u8 = 1; +/// Encoded length of every handle this export issues. +pub const HANDLE_BYTES: usize = 25; + +/// Why a client's handle was not accepted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HandleError { + /// Not a handle this export ever issued (length, version or tag). + Bad, + /// Issued for an earlier mount of this export. + Stale, +} + +/// Encodes and checks the handles of one kernel mount. +pub struct Handles { + owner: u64, + key: RandomState, +} +impl Handles { + pub fn new(owner: u64) -> Self { + Self { + owner, + key: RandomState::new(), + } + } + fn tag(&self, owner: u64, serial: u64) -> u64 { + self.key.hash_one((VERSION, owner, serial)) + } + pub fn encode(&self, serial: u64) -> [u8; HANDLE_BYTES] { + let mut handle = [0; HANDLE_BYTES]; + handle[0] = VERSION; + handle[1..9].copy_from_slice(&self.owner.to_be_bytes()); + handle[9..17].copy_from_slice(&serial.to_be_bytes()); + handle[17..25].copy_from_slice(&self.tag(self.owner, serial).to_be_bytes()); + handle + } + /// The serial a handle names. A handle of another mount is stale even + /// when its tag cannot be checked, since its key is gone. + pub fn decode(&self, handle: &[u8]) -> Result { + if handle.len() != HANDLE_BYTES || handle[0] != VERSION { + return Err(HandleError::Bad); + } + let word = |range: std::ops::Range| { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&handle[range]); + u64::from_be_bytes(bytes) + }; + let (owner, serial, tag) = (word(1..9), word(9..17), word(17..25)); + if owner != self.owner { + return Err(HandleError::Stale); + } + if serial == 0 || tag != self.tag(owner, serial) { + return Err(HandleError::Bad); + } + Ok(serial) + } +} diff --git a/engine/crates/layerfs-nfs/src/kernel.rs b/engine/crates/layerfs-nfs/src/kernel.rs new file mode 100644 index 0000000000..77f0ae2aba --- /dev/null +++ b/engine/crates/layerfs-nfs/src/kernel.rs @@ -0,0 +1,847 @@ +//! The synthetic kernel: the lookup references, open descriptors and parent +//! index a FUSE kernel would hold, kept by the export for a stateless client. +//! +//! The engine fences every request on a kernel reference: the root's implicit +//! one, a lookup reference, or an open descriptor. An NFS client presents a +//! handle at any time and never sends FORGET, OPEN or RELEASE, so: +//! +//! - every serial the export hands out in a handle holds exactly one lookup +//! reference, taken by the LOOKUP visit or the creating mutation that +//! decided it; a later visit that adds one more for the same serial is +//! answered by forgetting the extra one at once; +//! - READ and WRITE use one implicit descriptor per file (read-only until the +//! first WRITE, which reopens it writable), and READDIR one implicit +//! directory descriptor per enumeration (`cookieverf`); both are bounded +//! and the least recently used is closed when the bound is reached; +//! - `..` is answered from the parent recorded when a directory was looked +//! up, created, moved or listed, since the engine resolves names only +//! downward; +//! - a serial whose last name a REMOVE, RMDIR or replacing RENAME of this +//! export took away gives its reference and descriptor back at once, so the +//! engine can reclaim it; its handle is stale afterwards, as on any server; +//! - a client's silly rename (`.nfs…`) is not a namespace change: the source +//! name is unlinked and the inode is kept alive by the export alone, under +//! a name only this export knows (see [`crate::names`]). +//! +//! - the references are bounded (reference collection, "plan A" of the A2 +//! design, with no engine interface added): every handed-out serial keeps +//! the name it was last reached by. When more than the configured number +//! of serials hold a reference, the least recently used quarter that holds +//! no descriptor and no silly name is forgotten, its name memory kept. A +//! later handle of a forgotten serial is resolved again by the ordinary +//! LOOKUP of that remembered name (its parent first, recursively): the +//! same serial continues; another serial or none is a stale handle, as for +//! a file renamed or removed elsewhere. +//! +//! Everything still held is given back by [`Kernel::release`], which runs +//! before the engine mount is revoked. +use crate::{ + attributes::{refusal, Identity, Status}, + block::wait, + handle::{HandleError, Handles}, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::{ + operations::{KernelData, KernelRead, MutationFailure, ReadDataInput, ReadFailure}, + ports::{Fence, MountServices, RequestServices, ServiceError}, +}; +use layerfs_overlay::{KernelMount, OverlayError, READ_WINDOW}; +use layerfs_workspace::{KernelReadOperation, ViewStat}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, MutexGuard, + }, +}; + +/// Implicit file descriptors kept open at once. +pub const FILES_HELD: usize = 256; +/// Implicit directory descriptors (enumerations) kept open at once. +pub const DIRECTORIES_HELD: usize = 256; +/// EXCLUSIVE CREATE verifiers remembered for retransmissions. +pub const VERIFIERS_HELD: usize = 1024; +/// Lookup references an export keeps by default before collecting. +pub const LOOKUPS_HELD: usize = 65_536; +/// Remembered names beyond which a handle of a forgotten serial is stale. +const NAMES_HELD: usize = 4 * 1024 * 1024; +/// Parents resolved again for one handle at most (directory depth). +const DEPTH: usize = 4096; +/// Serials used within this many most recent uses are never collected: a +/// call another connection is serving may be about to use them. +pub const RECENT_KEPT: u64 = 64; + +/// A name in one directory: (directory serial, name bytes). +pub(crate) type Named = (u64, Vec); +/// An EXCLUSIVE CREATE's verifier, the serial it created and its last use. +pub(crate) type Verified = ([u8; 8], u64, u64); + +/// One implicit file descriptor. +#[derive(Clone, Copy)] +pub(crate) struct Held { + pub handle: u64, + /// The size last published or observed through this export. + pub size: u64, + pub tick: u64, + pub writable: bool, +} + +#[derive(Default)] +pub(crate) struct Table { + /// Serials holding the export's one lookup reference (the root's is implicit). + pub lookups: HashSet, + /// Directory serial -> the parent it was last seen under. + pub parents: HashMap, + /// File serial -> its implicit descriptor. + pub files: HashMap, + /// Directory descriptor -> (directory serial, last use). + pub directories: HashMap, + /// Silly-renamed inodes: (directory, client name) -> serial. Their + /// descriptors are never evicted; their source name is gone. + pub silly: HashMap, + /// EXCLUSIVE CREATE: (directory, name) -> (verifier, serial, last use). + pub verifiers: HashMap, + /// Serial -> the name it was last reached by, kept after its reference + /// is collected so a later handle can be resolved again. + pub names: HashMap, + /// Serial -> last use, for the collection order. + pub used: HashMap, + /// Serials marked for a cache refresh, and when (see [`crate::refresh`]). + pub refreshing: HashMap, + pub tick: u64, +} +impl Table { + pub fn tick(&mut self) -> u64 { + self.tick += 1; + self.tick + } + /// Serials kept alive by a silly name. + pub fn pinned(&self, serial: u64) -> bool { + self.silly.values().any(|held| *held == serial) + } +} +/// One opened file as READ and WRITE need it. +#[derive(Clone, Copy)] +pub(crate) struct OpenFile { + pub handle: u64, + pub size: u64, +} + +/// One kernel mount served over NFS: its ports and its synthetic kernel state. +pub struct Kernel { + services: Arc, + fence: Fence, + mount: KernelMount, + identity: Identity, + handles: Handles, + table: Mutex, + requests: AtomicU64, + /// Mutations whose failure keeps engine custody (possibly a publication + /// ticket): kept, never released or undone on a guess, as the FUSE + /// connection retains them. + retained: Mutex>, + /// Keep macOS AppleDouble (`._*`) names out of the Workspace: a regular + /// file lives in the export alone (see [`crate::apple_double`]), any + /// other kind is refused. + refuse_apple_double: bool, + /// The AppleDouble companions held in memory. + pub(crate) companions: Mutex, + /// Lookup references kept before the least recently used are collected. + lookups_held: usize, + /// References collected, handles resolved again, and resolutions that + /// found another serial or none (stale), over the export's life. + collected: AtomicU64, + resolved: AtomicU64, + unresolved: AtomicU64, + /// Silly renames served as unlinks, and creations refused by name. + pub(crate) silly_renamed: AtomicU64, + pub(crate) names_refused: AtomicU64, + /// AppleDouble companions created in memory. + pub(crate) names_detached: AtomicU64, +} +impl Kernel { + pub fn new(services: Arc, mount: KernelMount, uid: u32, gid: u32) -> Self { + let root = mount.root_serial(); + Self { + services, + fence: Fence::default(), + mount, + identity: Identity { + root, + uid, + gid, + fsid: mount.owner_id(), + }, + handles: Handles::new(mount.owner_id()), + table: Mutex::default(), + requests: AtomicU64::new(1), + retained: Mutex::default(), + refuse_apple_double: false, + companions: Mutex::default(), + lookups_held: LOOKUPS_HELD, + collected: AtomicU64::new(0), + resolved: AtomicU64::new(0), + unresolved: AtomicU64::new(0), + silly_renamed: AtomicU64::new(0), + names_refused: AtomicU64::new(0), + names_detached: AtomicU64::new(0), + } + } + /// Whether creating `._*` names is refused (see [`crate::names`]). + pub fn refusing_apple_double(mut self, refuse: bool) -> Self { + self.refuse_apple_double = refuse; + self + } + pub fn refuses_apple_double(&self) -> bool { + self.refuse_apple_double + } + /// How many lookup references are kept before collection (at least 4). + pub fn holding(mut self, lookups: usize) -> Self { + self.lookups_held = lookups.max(4); + self + } + pub fn identity(&self) -> Identity { + self.identity + } + pub fn root(&self) -> u64 { + self.identity.root + } + pub fn mount(&self) -> KernelMount { + self.mount + } + pub fn encode(&self, serial: u64) -> Vec { + self.handles.encode(serial).to_vec() + } + pub(crate) fn table(&self) -> MutexGuard<'_, Table> { + self.table + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// Keeps a failed mutation's remaining custody. + pub(crate) fn retain(&self, failure: MutationFailure) { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(failure); + } + /// The serial a client handle names, holding its reference: a collected + /// one is resolved again by its remembered name. + pub fn decode(&self, handle: &[u8]) -> Result { + let serial = self.handles.decode(handle).map_err(|error| match error { + HandleError::Bad => Status::BadHandle, + HandleError::Stale => Status::Stale, + })?; + self.held_reference(serial)?; + Ok(serial) + } + /// Ensures the export holds `serial`'s reference, resolving a collected + /// serial again; marks it used. + pub(crate) fn held_reference(&self, serial: u64) -> Result<(), Status> { + if serial == self.root() { + return Ok(()); + } + if crate::apple_double::companion(serial) { + // The export's own companion holds no engine reference. + return self.companion_stat(serial).map(|_| ()); + } + { + let mut table = self.table(); + if table.lookups.contains(&serial) { + let tick = table.tick(); + table.used.insert(serial, tick); + return Ok(()); + } + } + self.resolve(serial, 0) + } + /// One collected serial resolved by the LOOKUP of its remembered name, + /// after its parent. Another serial, or none, there is a stale handle. + fn resolve(&self, serial: u64, depth: usize) -> Result<(), Status> { + let named = self.table().names.get(&serial).cloned(); + let Some((parent, name)) = named else { + return Err(Status::Stale); + }; + if depth >= DEPTH { + return Err(Status::Stale); + } + let parent_held = parent == self.root() || self.table().lookups.contains(&parent); + if !parent_held { + self.resolve(parent, depth + 1)?; + } + let found = PathName::from_bytes(&name) + .map_err(|_| Status::Stale) + .and_then(|path| { + self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + ) + }); + match found { + Ok((stat, _, _)) => { + self.entry_taken(&stat, parent, &name); + if stat.serial == serial { + self.resolved.fetch_add(1, Ordering::Relaxed); + return Ok(()); + } + } + Err(Status::NoEnt | Status::Stale | Status::NotDir) => {} + Err(status) => return Err(status), + } + let mut table = self.table(); + if table + .names + .get(&serial) + .is_some_and(|(p, n)| *p == parent && *n == name) + { + table.names.remove(&serial); + } + self.unresolved.fetch_add(1, Ordering::Relaxed); + Err(Status::Stale) + } + pub fn request(&self) -> Result<(Arc, u64), Status> { + let services = self.services.request(&self.fence).map_err(failure)?; + Ok((services, self.requests.fetch_add(1, Ordering::Relaxed))) + } + fn observe( + &self, + protected: u64, + handle: Option, + operation: KernelReadOperation, + ) -> Result<(ViewStat, Option, Option), Status> { + let (services, request) = self.request()?; + let found = wait(KernelRead::prepare( + services, self.mount, request, protected, handle, operation, + )) + .map_err(read_failure)?; + let answer = found.value().map(|value| { + ( + value.stat.clone(), + value.file.map(|file| file.owner_id()), + value.directory.map(|directory| directory.owner_id()), + ) + }); + wait(found.dispose()).map_err(read_failure)?; + answer.map_err(refusal) + } + /// GETATTR. An inode that lost its last name while a silly name keeps it + /// is observed through its descriptor; any other inode no name reaches + /// any more is stale. + pub fn getattr(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_stat(serial); + } + let handle = { + let table = self.table(); + if table.pinned(serial) { + table.files.get(&serial).map(|held| held.handle) + } else { + None + } + }; + let stat = self + .observe(serial, handle, KernelReadOperation::Getattr { serial }) + .map(|(stat, _, _)| stat) + .map_err(|status| { + if status == Status::NoEnt { + Status::Stale + } else { + status + } + })?; + self.observed_size(&stat); + Ok(stat) + } + /// LOOKUP of `name` in directory `parent`, including `.`, `..` and the + /// export's own silly names. + pub fn lookup(&self, parent: u64, name: &[u8]) -> Result { + match name { + b"" => return Err(Status::NoEnt), + b"." => return self.getattr(parent), + b".." => { + let up = if parent == self.root() { + parent + } else { + let known = self.table().parents.get(&parent).copied(); + known.ok_or(Status::NoEnt)? + }; + self.held_reference(up)?; + return self.getattr(up); + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ => {} + } + if let Some(stat) = self.companion_named(parent, name) { + return Ok(stat); + } + let silly = self.table().silly.get(&(parent, name.to_vec())).copied(); + if let Some(serial) = silly { + return self.getattr(serial); + } + let path = PathName::from_bytes(name).map_err(|_| Status::Inval)?; + let (stat, _, _) = self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + )?; + // The deciding visit took one engine reference; keep exactly one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// A visit or mutation that decided `stat` as the entry `name` of + /// `parent` took one engine lookup reference on it: the export keeps + /// exactly one per serial, remembers the name and records a directory's + /// parent. Beyond the bound, the least recently used are collected. + pub(crate) fn entry_taken(&self, stat: &ViewStat, parent: u64, name: &[u8]) { + let (extra, collect) = { + let mut table = self.table(); + if stat.kind == InodeKind::Directory { + table.parents.insert(stat.serial, parent); + } + let root = stat.serial == self.root(); + if !root { + let tick = table.tick(); + table.used.insert(stat.serial, tick); + if table.names.len() < NAMES_HELD || table.names.contains_key(&stat.serial) { + table.names.insert(stat.serial, (parent, name.to_vec())); + } + } + let extra = root || !table.lookups.insert(stat.serial); + let collect = if table.lookups.len() > self.lookups_held { + collectable(&mut table, self.lookups_held - self.lookups_held / 4) + } else { + Vec::new() + }; + (extra, collect) + }; + if extra { + self.forget(stat.serial, 1); + } + self.collected + .fetch_add(collect.len() as u64, Ordering::Relaxed); + for serial in collect { + self.forget(serial, 1); + } + self.observed_size(stat); + } + /// Records that `serial` is now reached as `name` in `parent` (RENAME). + pub(crate) fn renamed(&self, serial: u64, parent: u64, name: &[u8]) { + let mut table = self.table(); + if table.names.contains_key(&serial) { + table.names.insert(serial, (parent, name.to_vec())); + } + } + pub(crate) fn forget(&self, serial: u64, count: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.forget(self.mount, serial, count))); + } + } + /// Gives back everything the export holds for `serial`, whose last name + /// is gone: its descriptor, its reference and its parent record. Later + /// handles of it are stale. + pub(crate) fn unlinked(&self, serial: u64) { + if serial == self.root() { + return; + } + let (file, directories, held) = { + let mut table = self.table(); + table.parents.remove(&serial); + table.names.remove(&serial); + table.used.remove(&serial); + table.verifiers.retain(|_, entry| entry.1 != serial); + let directories: Vec = table + .directories + .iter() + .filter(|(_, entry)| entry.0 == serial) + .map(|(handle, _)| *handle) + .collect(); + for handle in &directories { + table.directories.remove(handle); + } + ( + table.files.remove(&serial), + directories, + table.lookups.remove(&serial), + ) + }; + if let Some(file) = file { + self.close_file(serial, file.handle); + } + for handle in directories { + self.close_directory(serial, handle); + } + if held { + self.forget(serial, 1); + } + } + /// Records the parent of a directory seen in a listing or moved by RENAME. + pub fn saw_directory(&self, serial: u64, parent: u64) { + self.table().parents.insert(serial, parent); + } + /// The size a published or observed stat reports, for READ's EOF. + pub(crate) fn observed_size(&self, stat: &ViewStat) { + if let Some(held) = self.table().files.get_mut(&stat.serial) { + held.size = stat.logical_len; + } + } + /// READLINK: the raw target bytes. + pub fn readlink(&self, serial: u64) -> Result, Status> { + if crate::apple_double::companion(serial) { + return Err(Status::Inval); + } + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + None, + ReadDataInput::Link, + )) + .map_err(read_failure)? + .map_err(refusal)?; + Ok(data.bytes().to_vec()) + } + /// The implicit descriptor of `serial`, opened (or reopened writable) + /// when needed. + pub(crate) fn open_file(&self, serial: u64, writable: bool) -> Result { + { + let mut table = self.table(); + let tick = table.tick(); + if let Some(held) = table.files.get_mut(&serial) { + if held.writable || !writable { + held.tick = tick; + return Ok(OpenFile { + handle: held.handle, + size: held.size, + }); + } + } + } + let operation = KernelReadOperation::Open { serial, writable }; + let (stat, file, _) = self.observe(serial, None, operation)?; + let handle = file.ok_or(Status::ServerFault)?; + let (evicted, closed, chosen) = { + let mut table = self.table(); + let tick = table.tick(); + let current = table.files.get(&serial).copied(); + match current { + // Another connection opened a good enough one meanwhile. + Some(theirs) if theirs.writable || !writable => { + (None, Some((serial, handle)), theirs) + } + _ => { + let evicted = if current.is_none() && table.files.len() >= FILES_HELD { + let pinned: HashSet = table.silly.values().copied().collect(); + oldest( + table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.tick)), + ) + .and_then(|serial| { + table + .files + .remove(&serial) + .map(|held| (serial, held.handle)) + }) + } else { + None + }; + let held = Held { + handle, + size: stat.logical_len, + tick, + writable, + }; + table.files.insert(serial, held); + (evicted, current.map(|old| (serial, old.handle)), held) + } + } + }; + if let Some((old, descriptor)) = evicted { + self.close_file(old, descriptor); + } + if let Some((old, descriptor)) = closed { + self.close_file(old, descriptor); + } + Ok(OpenFile { + handle: chosen.handle, + size: chosen.size, + }) + } + pub(crate) fn close_file(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_file(self.mount, serial, handle))); + } + } + /// READ of at most one engine window. `eof` is set when the reply reaches + /// the size last seen for the file, or comes back short. + pub fn read(&self, serial: u64, offset: u64, count: u32) -> Result<(Vec, bool), Status> { + let length = count.min(READ_WINDOW as u32); + if crate::apple_double::companion(serial) { + return self.companion_read(serial, offset, length); + } + let open = self.open_file(serial, false)?; + if offset >= open.size { + return Ok((Vec::new(), true)); + } + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + Some(open.handle), + ReadDataInput::File { offset, length }, + )) + .map_err(read_failure)? + .map_err(refusal)?; + let bytes = data.bytes().to_vec(); + let end = offset.saturating_add(bytes.len() as u64); + let eof = bytes.len() < length as usize || end >= open.size; + Ok((bytes, eof)) + } + /// A fresh implicit directory descriptor for an enumeration from cookie 0. + pub fn open_directory(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::NotDir); + } + let (_, _, directory) = + self.observe(serial, None, KernelReadOperation::Opendir { serial })?; + let handle = directory.ok_or(Status::ServerFault)?; + let evicted = { + let mut table = self.table(); + let tick = table.tick(); + let evicted = if table.directories.len() >= DIRECTORIES_HELD { + oldest( + table + .directories + .iter() + .map(|(handle, entry)| (*handle, entry.1)), + ) + .and_then(|handle| { + table + .directories + .remove(&handle) + .map(|entry| (entry.0, handle)) + }) + } else { + None + }; + table.directories.insert(handle, (serial, tick)); + evicted + }; + if let Some((old, descriptor)) = evicted { + self.close_directory(old, descriptor); + } + Ok(handle) + } + /// The descriptor of a continuing enumeration, if it is still open. + pub fn directory(&self, serial: u64, handle: u64) -> Option { + let mut table = self.table(); + let tick = table.tick(); + let entry = table.directories.get_mut(&handle)?; + if entry.0 != serial { + return None; + } + entry.1 = tick; + Some(handle) + } + fn close_directory(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_directory(self.mount, serial, handle))); + } + } + /// Gives back everything the synthetic kernel holds: descriptors first, + /// then lookup references. Run once, before the engine mount is revoked. + /// Retained mutation failures keep their custody and are only counted. + pub fn release(&self) -> Released { + let table = std::mem::take(&mut *self.table()); + let companions = self.companions_release(); + let released = Released { + companions, + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + }; + for (serial, held) in table.files { + self.close_file(serial, held.handle); + } + for (handle, (serial, _)) in table.directories { + self.close_directory(serial, handle); + } + for serial in table.lookups { + self.forget(serial, 1); + } + released + } + fn retained_count(&self) -> usize { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .len() + } + /// Current holdings, for diagnostics. + pub fn held(&self) -> Released { + let table = self.table(); + Released { + companions: self.companions_held(), + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + } + } + /// Closes every implicit file descriptor that no silly name needs, so + /// nothing the export opened is held across a Commit. Returns how many. + pub fn close_files(&self) -> usize { + let closing: Vec<(u64, u64)> = { + let mut table = self.table(); + let pinned: HashSet = table.silly.values().copied().collect(); + let closing: Vec<(u64, u64)> = table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.handle)) + .collect(); + for (serial, _) in &closing { + table.files.remove(serial); + } + closing + }; + for (serial, handle) in &closing { + self.close_file(*serial, *handle); + } + closing.len() + } + /// Closes every enumeration descriptor; a continuing READDIR then + /// answers `NFS3ERR_BAD_COOKIE` and the client lists again from the + /// start. Used after a Commit installed another root. Returns how many. + pub fn close_directories(&self) -> usize { + let closing: Vec<(u64, u64)> = std::mem::take(&mut self.table().directories) + .into_iter() + .map(|(handle, (serial, _))| (serial, handle)) + .collect(); + for (serial, handle) in &closing { + self.close_directory(*serial, *handle); + } + closing.len() + } +} +/// Takes the least recently used references out of the table until `keep` +/// remain, skipping recently used serials, serials that hold a descriptor +/// or a silly name, and +/// directories a held serial was reached through: the engine's own parent +/// index of a held inode never loses its link, so collection proceeds from +/// the leaves. Names and parent records stay for a later resolution. +fn collectable(table: &mut Table, keep: usize) -> Vec { + let held_parents = table.lookups.iter().filter_map(|serial| { + table + .names + .get(serial) + .map(|named| named.0) + .or_else(|| table.parents.get(serial).copied()) + }); + let busy: HashSet = table + .silly + .values() + .copied() + .chain(table.files.keys().copied()) + .chain(table.directories.values().map(|entry| entry.0)) + .chain(held_parents) + .collect(); + let recent = table.tick.saturating_sub(RECENT_KEPT); + let mut candidates: Vec<(u64, u64)> = table + .lookups + .iter() + .filter(|serial| !busy.contains(serial)) + .map(|serial| (table.used.get(serial).copied().unwrap_or(0), *serial)) + .filter(|(used, _)| *used < recent) + .collect(); + let excess = table.lookups.len().saturating_sub(keep); + candidates.sort_unstable(); + candidates.truncate(excess); + for (_, serial) in &candidates { + table.lookups.remove(serial); + table.used.remove(serial); + } + candidates.into_iter().map(|(_, serial)| serial).collect() +} +/// What [`Kernel::release`] gave back. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Released { + pub files: usize, + pub directories: usize, + pub lookups: usize, + /// Silly names whose inode was kept alive until release. + pub silly: usize, + /// Failed mutations still holding engine custody. + pub retained: usize, + /// Client silly renames served as unlinks, over the export's life. + pub silly_renamed: u64, + /// Creations refused for their AppleDouble name, over the export's life. + pub names_refused: u64, + /// AppleDouble companions held in memory (see [`crate::apple_double`]). + pub companions: usize, + /// AppleDouble companions created, over the export's life. + pub names_detached: u64, + /// References collected beyond the bound, over the export's life. + pub collected: u64, + /// Handles of collected serials resolved again by their name. + pub resolved: u64, + /// Handles of collected serials whose name no longer reaches them. + pub unresolved: u64, +} +fn oldest(entries: impl Iterator) -> Option { + entries.min_by_key(|(_, tick)| *tick).map(|(key, _)| key) +} +fn stale(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + while let Some(error) = current { + if matches!( + error.downcast_ref::(), + Some(OverlayError::Stale) + ) { + return true; + } + current = error.source(); + } + false +} +/// Whether a failed step's cause chain holds an engine `Stale`. +pub(crate) fn failure_is_stale(error: &ServiceError) -> bool { + stale(error.as_ref()) +} +/// A failed port step: an engine `Stale` is the client's stale handle; any +/// other failure is an I/O error of this request alone. +pub fn failure(error: ServiceError) -> Status { + if stale(error.as_ref()) { + Status::Stale + } else { + Status::Io + } +} +fn read_failure(failure: ReadFailure) -> Status { + if stale(failure.reason.as_ref()) { + Status::Stale + } else { + Status::Io + } +} diff --git a/engine/crates/layerfs-nfs/src/lib.rs b/engine/crates/layerfs-nfs/src/lib.rs new file mode 100644 index 0000000000..32b1b450f0 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/lib.rs @@ -0,0 +1,36 @@ +//! A mounted Workspace served as a loopback NFSv3 export. +//! +//! The platform-independent half of the native macOS route: the same +//! [`MountServices`](layerfs_fuse::ports::MountServices) ports and kernel +//! operations the FUSE connection uses, answered over ONC RPC instead of +//! `/dev/fuse`. A stateless NFS client holds no lookup references or open +//! descriptors, so the export keeps them itself (see [`kernel`]). +//! +//! Every NFSv3 procedure is served (MKNOD answers `NFS3ERR_NOTSUPP`: the +//! canonical format has no special files) and the MOUNT protocol; an export +//! configured read-only answers every changing procedure `NFS3ERR_ROFS`. +//! A changing procedure's publication tickets are released only after its +//! reply was sent (see [`change`]); names a client invents for itself are +//! kept out of the Workspace (see [`names`] and [`apple_double`]). +//! There is no third-party dependency: XDR, RPC record marking and both +//! programs are implemented here over `std::net`. +#![forbid(unsafe_code)] + +pub mod apple_double; +pub mod attributes; +mod block; +pub mod change; +mod changing; +mod directory; +pub mod handle; +pub mod kernel; +mod mount; +pub mod names; +mod nfs; +pub mod refresh; +pub mod rpc; +mod server; +pub mod xdr; + +pub use kernel::Released; +pub use server::{Count, ExportConfig, Server}; diff --git a/engine/crates/layerfs-nfs/src/mount.rs b/engine/crates/layerfs-nfs/src/mount.rs new file mode 100644 index 0000000000..4b6656de04 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/mount.rs @@ -0,0 +1,66 @@ +//! MOUNT program 100005 version 3 (RFC 1813 appendix I): one export. +use crate::{ + kernel::Kernel, + rpc::Accept, + xdr::{Reader, Writer}, +}; + +pub const PROGRAM: u32 = 100_005; +pub const VERSION: u32 = 3; +const MNTPATHLEN: usize = 1024; +const MNT3_OK: u32 = 0; +const MNT3ERR_NOENT: u32 = 2; +const AUTH_UNIX: u32 = 1; + +/// Decodes and answers one MOUNT call for the export named `path`. +pub fn call( + kernel: &Kernel, + path: &str, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, +) -> Result<(), Accept> { + match procedure { + 0 | 4 => Ok(()), + 1 => { + let asked = arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + if exported(asked, path) { + out.u32(MNT3_OK); + out.opaque(&kernel.encode(kernel.root())); + out.u32(1).u32(AUTH_UNIX); + } else { + out.u32(MNT3ERR_NOENT); + } + Ok(()) + } + 2 => { + out.bool(false); + Ok(()) + } + 3 => { + arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + Ok(()) + } + 5 => { + out.bool(true).opaque(path.as_bytes()).bool(false); + out.bool(false); + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The export path, compared without trailing slashes ("/" names the root). +fn exported(asked: &[u8], path: &str) -> bool { + let trim = |bytes: &[u8]| -> Vec { + let mut bytes = bytes.to_vec(); + while bytes.len() > 1 && bytes.last() == Some(&b'/') { + bytes.pop(); + } + bytes + }; + trim(asked) == trim(path.as_bytes()) +} diff --git a/engine/crates/layerfs-nfs/src/names.rs b/engine/crates/layerfs-nfs/src/names.rs new file mode 100644 index 0000000000..9237471c15 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/names.rs @@ -0,0 +1,41 @@ +//! Names an NFS client invents for its own bookkeeping, kept out of the +//! Workspace by the export itself so they never reach a Commit. +//! +//! - **Silly renames.** A client that unlinks a file it still has open +//! renames it instead, and removes that name after the last close. Linux +//! names it `.nfs` + 16 hex digits (fileid) + 8 hex digits (counter); +//! macOS `.nfs.` + 8 hex + `.` + 4 hex. Such a RENAME is served as an +//! UNLINK of the source name while the export keeps the inode alive by its +//! own reference and an open descriptor; the silly name exists only in the +//! export's table (LOOKUP, GETATTR, READ, WRITE, SETATTR and REMOVE of it +//! work; READDIR does not list it). The final REMOVE gives the inode back. +//! - **AppleDouble.** On NFSv3 the macOS client stores extended attributes +//! and resource forks in `._` companion files. When the export is +//! configured to keep them out, such a regular file is created in the +//! export's memory alone (see [`crate::apple_double`]): the client's +//! attribute write (`copyfile`, `cp`, `std::fs::copy`, `ditto`, `tar`) +//! succeeds and reads back for the export's life, and nothing reaches the +//! Workspace. Any other kind under such a name (directory, symlink, link, +//! rename target) is refused with `NFS3ERR_ACCES`. +//! +//! The patterns are exact so that a user's own `.nfsrc` or `._config` is an +//! ordinary name (the AppleDouble refusal is the one deliberate exception +//! and is configurable). + +/// A client-generated silly-rename name. +pub fn silly(name: &[u8]) -> bool { + let hex = |bytes: &[u8]| bytes.iter().all(u8::is_ascii_hexdigit); + if let Some(rest) = name.strip_prefix(b".nfs.") { + // macOS: .nfs.XXXXXXXX.XXXX + return rest.len() == 13 && rest[8] == b'.' && hex(&rest[..8]) && hex(&rest[9..]); + } + if let Some(rest) = name.strip_prefix(b".nfs") { + // Linux: .nfs + 16 + 8 hex digits + return rest.len() == 24 && hex(rest); + } + false +} +/// A macOS AppleDouble companion name (`._` followed by at least one byte). +pub fn apple_double(name: &[u8]) -> bool { + name.len() > 2 && name.starts_with(b"._") +} diff --git a/engine/crates/layerfs-nfs/src/nfs.rs b/engine/crates/layerfs-nfs/src/nfs.rs new file mode 100644 index 0000000000..b257125300 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/nfs.rs @@ -0,0 +1,269 @@ +//! NFS program 100003 version 3: dispatch of every procedure. +//! +//! The reading procedures are answered here; the changing ones in +//! [`crate::changing`]. A read-only export answers every changing procedure +//! `NFS3ERR_ROFS` with its empty weak cache consistency data. +use crate::{ + attributes::{access, post_op_maybe, Status}, + change::Tickets, + changing, + directory::{list, Listing}, + kernel::Kernel, + rpc::{Accept, Credentials}, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_overlay::{READ_WINDOW, WRITE_WINDOW}; + +pub const PROGRAM: u32 = 100_003; +pub const VERSION: u32 = 3; +/// Procedure names by number, for diagnostics. +pub const PROCEDURES: [&str; 22] = [ + "NULL", + "GETATTR", + "SETATTR", + "LOOKUP", + "ACCESS", + "READLINK", + "READ", + "WRITE", + "CREATE", + "MKDIR", + "SYMLINK", + "MKNOD", + "REMOVE", + "RMDIR", + "RENAME", + "LINK", + "READDIR", + "READDIRPLUS", + "FSSTAT", + "FSINFO", + "PATHCONF", + "COMMIT", +]; +/// Longest handle (`NFS3_FHSIZE`) and the longest name a client may send. +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Declared capacity: fixed values, never an admission signal (as for FUSE statfs). +const DECLARED_BLOCKS: u64 = 1 << 30; +const DECLARED_FILES: u64 = 1 << 30; +const BLOCK_BYTES: u64 = 4096; +/// FSINFO properties: FSF3_LINK | FSF3_SYMLINK | FSF3_HOMOGENEOUS | FSF3_CANSETTIME. +const PROPERTIES: u32 = 0x01 | 0x02 | 0x08 | 0x10; +/// Preferred READDIR reply size the client is told to ask for. +const DIRECTORY_PREFERENCE: u32 = 32 * 1024; + +/// The export's fixed answers. +#[derive(Clone, Copy, Debug)] +pub struct Policy { + pub read_only: bool, +} + +/// Decodes the arguments of `procedure`, runs it and appends its results. +/// `Err` is an RPC-level refusal of the call. +#[allow(clippy::too_many_arguments)] +pub fn call( + kernel: &Kernel, + policy: Policy, + procedure: u32, + credentials: &Credentials, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Accept> { + let garbage = |_: Garbage| Accept::GarbageArguments; + match procedure { + 0 => Ok(()), + 1 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().fattr3(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + Ok(()) + } + 3 => { + let parent = handle(kernel, arguments).map_err(garbage)?; + let name = arguments.opaque(NAME_LIMIT).map_err(garbage)?; + // Both answers carry the directory's attributes (RFC 1813 + // LOOKUP3resok and LOOKUP3resfail), so the client keeps its + // cached view of the directory without a GETATTR of its own. + let directory = parent + .as_ref() + .ok() + .and_then(|parent| kernel.getattr(*parent).ok()); + match parent.and_then(|parent| kernel.lookup(parent, name)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + post_op_maybe(kernel.identity(), out, directory.as_ref()); + } + Err(status) => { + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, directory.as_ref()); + } + } + Ok(()) + } + 4 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let requested = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + let caller = credentials + .unix + .as_ref() + .map(|unix| (unix.uid, unix.gid, unix.groups.as_slice())); + let granted = access( + kernel.identity(), + &stat, + caller, + requested, + policy.read_only, + ); + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + out.u32(granted); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 5 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.readlink(serial)) { + Ok(target) => { + out.u32(Status::Ok.code()).bool(false).opaque(&target); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 6 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let offset = arguments.u64().map_err(garbage)?; + let count = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.read(serial, offset, count)) { + Ok((data, eof)) => { + out.u32(Status::Ok.code()).bool(false); + out.u32(data.len() as u32).bool(eof).opaque(&data); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 16 | 17 => { + let plus = procedure == 17; + let serial = handle(kernel, arguments).map_err(garbage)?; + let cookie = arguments.u64().map_err(garbage)?; + let verifier = arguments.fixed(8).map_err(garbage)?; + let mut budget = arguments.u32().map_err(garbage)?; + if plus { + // dircount bounds the names alone; maxcount the whole reply. + let _dircount = budget; + budget = arguments.u32().map_err(garbage)?; + } + let mut word = [0; 8]; + word.copy_from_slice(verifier); + let listing = |serial| Listing { + serial, + cookie, + verifier: u64::from_be_bytes(word), + budget: budget as usize, + plus, + }; + match serial.and_then(|serial| list(kernel, listing(serial))) { + Ok(body) => { + out.u32(Status::Ok.code()).fixed(&body); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 18..=20 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + filesystem(procedure, out); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 2 | 7..=15 | 21 if !policy.read_only => { + changing::call(kernel, procedure, arguments, out, tickets, verifier).map_err(garbage) + } + 2 | 7..=15 | 21 => { + // Each changing procedure's failure carries wcc_data (two absent + // optional fields) per directory it names; LINK also its object's + // post_op_attr. + let absent = match procedure { + 14 => 4, + 15 => 3, + _ => 2, + }; + out.u32(Status::RoFs.code()); + for _ in 0..absent { + out.bool(false); + } + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The serial of the call's handle argument; a handle that is not accepted is +/// the procedure's own failure status, not an RPC refusal. +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +/// FSSTAT, FSINFO and PATHCONF results after the object's attributes. +fn filesystem(procedure: u32, out: &mut Writer) { + match procedure { + 18 => { + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_FILES) + .u64(DECLARED_FILES) + .u64(DECLARED_FILES); + out.u32(0); // invarsec + } + 19 => { + let (read, write) = (READ_WINDOW as u32, WRITE_WINDOW as u32); + out.u32(read).u32(read).u32(BLOCK_BYTES as u32); + out.u32(write).u32(write).u32(BLOCK_BYTES as u32); + out.u32(DIRECTORY_PREFERENCE); + out.u64(i64::MAX as u64); + out.u32(0).u32(1); // time_delta: 1 ns + out.u32(PROPERTIES); + } + _ => { + out.u32(u32::MAX); // linkmax + out.u32(255); // name_max + out.bool(true); // no_trunc + out.bool(true); // chown_restricted + out.bool(false); // case_insensitive + out.bool(true); // case_preserving + } + } +} diff --git a/engine/crates/layerfs-nfs/src/refresh.rs b/engine/crates/layerfs-nfs/src/refresh.rs new file mode 100644 index 0000000000..32a7f1e071 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/refresh.rs @@ -0,0 +1,75 @@ +//! Cache refresh after a Commit installed another root (the EdenFS method). +//! +//! An NFS client has no invalidation channel: it keeps attributes and +//! listings until `actimeo` or its close-to-open check. A SETATTR it sends +//! itself is answered with fresh post-operation attributes, and a client +//! that sees them changed drops what it cached for that inode. So the daemon +//! names the inodes an install made stale ([`Kernel::refresh`]) and changes +//! their mode to the value it already has through the mount point. The +//! export answers each such mode-only SETATTR on a marked inode with its +//! current attributes and changes nothing (the mark is taken once and ends +//! after [`MARK_LIFE`]), so the refresh never writes into the Workspace and +//! never reverts a mode the install changed. +use crate::{change::Sattr, kernel::Kernel}; +use std::time::{Duration, Instant}; + +/// How long one refresh mark waits for its SETATTR. +pub const MARK_LIFE: Duration = Duration::from_secs(30); +/// Directory depth one path is built to at most. +const DEPTH: usize = 4096; + +impl Kernel { + /// Marks each of `serials` this export handed out and still knows by + /// name, and returns their paths below the export root (empty for the + /// root itself), in the order given. A serial the client never saw has + /// nothing cached and is left out. + pub fn refresh(&self, serials: impl IntoIterator) -> Vec>> { + let root = self.root(); + let mut table = self.table(); + let now = Instant::now(); + table + .refreshing + .retain(|_, marked| now.duration_since(*marked) < MARK_LIFE); + let mut paths = Vec::new(); + for serial in serials { + let mut path = Vec::new(); + let mut at = serial; + let mut known = true; + while at != root { + let Some((parent, name)) = table.names.get(&at) else { + known = false; + break; + }; + if path.len() >= DEPTH { + known = false; + break; + } + path.push(name.clone()); + at = *parent; + } + if known { + path.reverse(); + table.refreshing.insert(serial, now); + paths.push(path); + } + } + paths + } + /// Whether a SETATTR of `serial` is a pending refresh: mode only, on a + /// marked inode whose mark has not ended. Takes the mark. + pub(crate) fn refreshed(&self, serial: u64, sattr: &Sattr) -> bool { + let mode_only = sattr.mode.is_some() + && sattr.uid.is_none() + && sattr.gid.is_none() + && sattr.size.is_none() + && sattr.mtime.is_none(); + if !mode_only { + return false; + } + let mut table = self.table(); + match table.refreshing.remove(&serial) { + Some(marked) => marked.elapsed() < MARK_LIFE, + None => false, + } + } +} diff --git a/engine/crates/layerfs-nfs/src/rpc.rs b/engine/crates/layerfs-nfs/src/rpc.rs new file mode 100644 index 0000000000..43c2ded345 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/rpc.rs @@ -0,0 +1,181 @@ +//! ONC RPC v2 (RFC 5531) over TCP record marking: calls in, replies out. +use crate::xdr::{Garbage, Reader, Writer}; +use std::io::{self, Read, Write}; + +const CALL: u32 = 0; +const REPLY: u32 = 1; +const MSG_ACCEPTED: u32 = 0; +const MSG_DENIED: u32 = 1; +const RPC_MISMATCH: u32 = 0; +const AUTH_ERROR: u32 = 1; +const AUTH_NONE: u32 = 0; +const AUTH_UNIX: u32 = 1; +/// AUTH_TOOWEAK: the credential flavor is not accepted. +const AUTH_TOOWEAK: u32 = 5; +const LAST_FRAGMENT: u32 = 1 << 31; +/// Largest credential or verifier body RFC 5531 allows. +const AUTH_BODY: usize = 400; +/// Largest whole record accepted. A1 serves no WRITE, so every call is small; +/// the bound still admits a full-window WRITE for the later write path. +pub const RECORD_LIMIT: usize = 2 * 1024 * 1024; + +/// The accepted-reply status of one call. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Accept { + Success, + ProgramUnavailable, + /// The program exists in versions `low..=high` only. + ProgramMismatch { + low: u32, + high: u32, + }, + ProcedureUnavailable, + GarbageArguments, +} +/// The caller's identity as AUTH_UNIX states it. AUTH_NONE has none. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Credentials { + pub unix: Option, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UnixCredentials { + pub uid: u32, + pub gid: u32, + pub groups: Vec, +} +/// One decoded call header; its arguments follow in the same record. +#[derive(Debug)] +pub struct Call { + pub xid: u32, + pub program: u32, + pub version: u32, + pub procedure: u32, + pub credentials: Credentials, +} +/// A record that is not an answerable call. +#[derive(Debug)] +pub enum Refused { + /// Not a call message, or not decodable: no reply is possible. + Undecodable, + /// A call whose header can be answered only with a rejection. + Reply(Vec), +} + +/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`]. +/// `Ok(None)` is an orderly end of stream before a record started. +pub fn read_record(stream: &mut impl Read) -> io::Result>> { + let mut record = Vec::new(); + loop { + let mut header = [0; 4]; + if let Err(error) = stream.read_exact(&mut header) { + if error.kind() == io::ErrorKind::UnexpectedEof && record.is_empty() { + return Ok(None); + } + return Err(error); + } + let header = u32::from_be_bytes(header); + let length = (header & !LAST_FRAGMENT) as usize; + if record.len() + length > RECORD_LIMIT { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "RPC record exceeds its bound", + )); + } + let start = record.len(); + record.resize(start + length, 0); + stream.read_exact(&mut record[start..])?; + if header & LAST_FRAGMENT != 0 { + return Ok(Some(record)); + } + } +} +/// Writes one reply as a single last fragment. +pub fn write_record(stream: &mut impl Write, reply: &[u8]) -> io::Result<()> { + let mut framed = Vec::with_capacity(reply.len() + 4); + framed.extend_from_slice(&(reply.len() as u32 | LAST_FRAGMENT).to_be_bytes()); + framed.extend_from_slice(reply); + stream.write_all(&framed) +} + +/// Decodes the call header; the returned reader is positioned at the arguments. +pub fn parse_call(record: &[u8]) -> Result<(Call, Reader<'_>), Refused> { + let mut reader = Reader::new(record); + let header = (|| -> Result<(u32, u32, u32), Garbage> { + Ok((reader.u32()?, reader.u32()?, reader.u32()?)) + })(); + let (xid, kind, rpcvers) = header.map_err(|_| Refused::Undecodable)?; + if kind != CALL { + return Err(Refused::Undecodable); + } + if rpcvers != 2 { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(RPC_MISMATCH); + reply.u32(2).u32(2); + return Err(Refused::Reply(reply.into_bytes())); + } + let body = (|| -> Result<(u32, u32, u32, u32, &[u8]), Garbage> { + let program = reader.u32()?; + let version = reader.u32()?; + let procedure = reader.u32()?; + let flavor = reader.u32()?; + let credential = reader.opaque(AUTH_BODY)?; + let _verifier_flavor = reader.u32()?; + let _verifier = reader.opaque(AUTH_BODY)?; + Ok((program, version, procedure, flavor, credential)) + })(); + let (program, version, procedure, flavor, credential) = + body.map_err(|_| Refused::Undecodable)?; + let credentials = match flavor { + AUTH_NONE => Credentials::default(), + AUTH_UNIX => match unix_credentials(credential) { + Ok(unix) => Credentials { unix: Some(unix) }, + Err(Garbage) => return Err(Refused::Reply(denied_auth(xid))), + }, + _ => return Err(Refused::Reply(denied_auth(xid))), + }; + Ok(( + Call { + xid, + program, + version, + procedure, + credentials, + }, + reader, + )) +} +fn unix_credentials(body: &[u8]) -> Result { + let mut reader = Reader::new(body); + let _stamp = reader.u32()?; + let _machine = reader.opaque(255)?; + let uid = reader.u32()?; + let gid = reader.u32()?; + let count = reader.u32()? as usize; + if count > 16 { + return Err(Garbage); + } + let groups = (0..count) + .map(|_| reader.u32()) + .collect::, _>>()?; + Ok(UnixCredentials { uid, gid, groups }) +} +fn denied_auth(xid: u32) -> Vec { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(AUTH_ERROR); + reply.u32(AUTH_TOOWEAK); + reply.into_bytes() +} +/// The accepted-reply header for `xid`; procedure results follow a success. +pub fn accepted(xid: u32, status: Accept) -> Writer { + let mut reply = Writer::with_capacity(256); + reply.u32(xid).u32(REPLY).u32(MSG_ACCEPTED); + reply.u32(AUTH_NONE).u32(0); + match status { + Accept::Success => reply.u32(0), + Accept::ProgramUnavailable => reply.u32(1), + Accept::ProgramMismatch { low, high } => reply.u32(2).u32(low).u32(high), + Accept::ProcedureUnavailable => reply.u32(3), + Accept::GarbageArguments => reply.u32(4), + }; + reply +} diff --git a/engine/crates/layerfs-nfs/src/server.rs b/engine/crates/layerfs-nfs/src/server.rs new file mode 100644 index 0000000000..19b208866d --- /dev/null +++ b/engine/crates/layerfs-nfs/src/server.rs @@ -0,0 +1,335 @@ +//! One export on one loopback TCP listener: MOUNT and NFS on the same port. +//! +//! Each accepted connection is served by its own thread, one call at a time, +//! in arrival order. A call's reply is written first; only after that single +//! send attempt returned are the publication tickets of the mutations it +//! decided released (see [`crate::change::Tickets`]). Stop closes the listener and every connection, joins +//! their threads and only then gives back the synthetic kernel's holdings, +//! so no call is running when references are released. +use crate::{ + change::Tickets, + kernel::{Kernel, Released}, + mount, + nfs::{self, Policy}, + rpc::{self, Accept, Refused}, +}; +use layerfs_fuse::ports::MountServices; +use layerfs_overlay::KernelMount; +use std::{ + io, + net::{Shutdown, SocketAddr, TcpListener, TcpStream}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +/// What the export shows and allows. +#[derive(Clone, Debug)] +pub struct ExportConfig { + /// The path MNT accepts, normally "/". + pub path: String, + /// The identity every inode is shown as. + pub uid: u32, + pub gid: u32, + /// Changing procedures answer `NFS3ERR_ROFS`. + pub read_only: bool, + /// Keep macOS AppleDouble `._*` names out of the Workspace: a regular + /// file lives in the export's memory, any other kind is refused (see + /// [`crate::apple_double`]). + pub refuse_apple_double: bool, + /// Lookup references kept before the least recently used are collected + /// (see [`crate::kernel`]); [`crate::kernel::LOOKUPS_HELD`] by default. + pub lookups_held: usize, +} +impl ExportConfig { + /// A writable export of "/" shown as `uid`:`gid`, refusing AppleDouble + /// names, with the default reference bound. + pub fn writable(uid: u32, gid: u32) -> Self { + Self { + path: "/".into(), + uid, + gid, + read_only: false, + refuse_apple_double: true, + lookups_held: crate::kernel::LOOKUPS_HELD, + } + } +} + +/// Calls served and time spent, per procedure. +pub struct Counters { + nfs: [(AtomicU64, AtomicU64); 22], + mount: [(AtomicU64, AtomicU64); 6], +} +impl Default for Counters { + fn default() -> Self { + Self { + nfs: std::array::from_fn(|_| Default::default()), + mount: std::array::from_fn(|_| Default::default()), + } + } +} +/// One procedure's totals. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Count { + pub procedure: &'static str, + pub calls: u64, + pub nanoseconds: u64, +} +const MOUNT_PROCEDURES: [&str; 6] = ["MNT_NULL", "MNT", "DUMP", "UMNT", "UMNTALL", "EXPORT"]; +impl Counters { + fn charge(&self, program: u32, procedure: u32, started: Instant) { + let slot = match program { + nfs::PROGRAM => self.nfs.get(procedure as usize), + mount::PROGRAM => self.mount.get(procedure as usize), + _ => None, + }; + if let Some((calls, time)) = slot { + calls.fetch_add(1, Ordering::Relaxed); + let spent = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + time.fetch_add(spent, Ordering::Relaxed); + } + } + /// Procedures called at least once. + pub fn snapshot(&self) -> Vec { + let rows = |names: &[&'static str], slots: &[(AtomicU64, AtomicU64)]| { + names + .iter() + .zip(slots) + .map(|(name, (calls, time))| Count { + procedure: name, + calls: calls.load(Ordering::Relaxed), + nanoseconds: time.load(Ordering::Relaxed), + }) + .filter(|count| count.calls > 0) + .collect::>() + }; + let mut all = rows(&MOUNT_PROCEDURES, &self.mount); + all.extend(rows(&nfs::PROCEDURES, &self.nfs)); + all + } +} + +struct Shared { + kernel: Kernel, + /// The WRITE/COMMIT verifier: fresh for every export. + verifier: [u8; 8], + config: ExportConfig, + counters: Counters, + stopping: AtomicBool, + connections: Mutex)>>, +} +/// A running export. +pub struct Server { + address: SocketAddr, + shared: Arc, + accept: Option>, +} +impl Server { + /// Serves the engine mount `mount` over `services` on `listener`. + pub fn start( + listener: TcpListener, + services: Arc, + mount: KernelMount, + config: ExportConfig, + ) -> io::Result { + let address = listener.local_addr()?; + let shared = Arc::new(Shared { + kernel: Kernel::new(services, mount, config.uid, config.gid) + .refusing_apple_double(config.refuse_apple_double) + .holding(config.lookups_held), + verifier: fresh_verifier(), + config, + counters: Counters::default(), + stopping: AtomicBool::new(false), + connections: Mutex::default(), + }); + let accepting = shared.clone(); + let accept = thread::Builder::new() + .name("layerfs-nfs-accept".into()) + .spawn(move || accept_loop(&listener, &accepting))?; + Ok(Self { + address, + shared, + accept: Some(accept), + }) + } + pub fn local_addr(&self) -> SocketAddr { + self.address + } + pub fn counters(&self) -> Vec { + self.shared.counters.snapshot() + } + /// The synthetic kernel's current holdings. + pub fn held(&self) -> Released { + self.shared.kernel.held() + } + /// Closes the implicit file descriptors no silly name needs, for example + /// before a Commit; the next READ or WRITE reopens one. Returns how many. + pub fn close_files(&self) -> usize { + self.shared.kernel.close_files() + } + /// After a Commit installed a root other than the one the client saw: + /// closes the implicit file and enumeration descriptors, so the next + /// READ reopens the installed file and a continuing READDIR restarts. + /// Attributes the client cached are revalidated on its own schedule + /// (`actimeo`, close-to-open). Returns (files, directories) closed. + pub fn installed(&self) -> (usize, usize) { + ( + self.shared.kernel.close_files(), + self.shared.kernel.close_directories(), + ) + } + /// After an install: marks the stale `serials` the client may cache and + /// returns their paths below the mount point, for the mode-preserving + /// SETATTR that makes the client drop its cache (see [`crate::refresh`]). + pub fn refresh(&self, serials: impl IntoIterator) -> Vec { + use std::os::unix::ffi::OsStrExt; + (self.shared.kernel.refresh(serials).into_iter()) + .map(|names| { + (names.iter()) + .map(|name| std::ffi::OsStr::from_bytes(name)) + .collect() + }) + .collect() + } + /// The write verifier this export answers WRITE and COMMIT with. + pub fn verifier(&self) -> [u8; 8] { + self.shared.verifier + } + /// Stops accepting, closes every connection, joins every thread, then + /// releases the synthetic kernel's descriptors and references. The engine + /// mount itself stays with the caller, who revokes it afterwards. + pub fn stop(mut self) -> Released { + self.shutdown() + } + fn shutdown(&mut self) -> Released { + let Some(accept) = self.accept.take() else { + return Released::default(); + }; + self.shared.stopping.store(true, Ordering::Release); + // Wake the blocking accept; it sees the flag and returns. + drop(TcpStream::connect(self.address)); + drop(accept.join()); + let connections = std::mem::take( + &mut *self + .shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ); + for (stream, _) in &connections { + drop(stream.shutdown(Shutdown::Both)); + } + for (_, thread) in connections { + drop(thread.join()); + } + self.shared.kernel.release() + } +} +impl Drop for Server { + fn drop(&mut self) { + self.shutdown(); + } +} +fn accept_loop(listener: &TcpListener, shared: &Arc) { + for stream in listener.incoming() { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let Ok(stream) = stream else { continue }; + drop(stream.set_nodelay(true)); + let Ok(control) = stream.try_clone() else { + continue; + }; + let serving = shared.clone(); + let spawned = thread::Builder::new() + .name("layerfs-nfs-connection".into()) + .spawn(move || connection(stream, &serving)); + if let Ok(thread) = spawned { + let mut connections = shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + // A closed connection's thread has ended; forget it. + connections.retain(|(_, thread)| !thread.is_finished()); + connections.push((control, thread)); + } + } +} +fn connection(mut stream: TcpStream, shared: &Shared) { + while let Ok(Some(record)) = rpc::read_record(&mut stream) { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let mut tickets = Tickets::default(); + let Some(reply) = answer(shared, &record, &mut tickets) else { + tickets.replied(&shared.kernel); + continue; + }; + // The single reply attempt, then the release of every ticket it + // owed, whether or not the reply reached the client. + let sent = rpc::write_record(&mut stream, &reply); + tickets.replied(&shared.kernel); + if sent.is_err() { + return; + } + } +} +/// A verifier no earlier export of this process or machine is likely to +/// have used: the clock and the process, under a per-process random key. +fn fresh_verifier() -> [u8; 8] { + use std::hash::BuildHasher; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_nanos()); + std::collections::hash_map::RandomState::new() + .hash_one((now, std::process::id())) + .to_be_bytes() +} +/// The reply to one record, or none when the record is not a call. +/// Mutations the call decided go to `tickets`. +fn answer(shared: &Shared, record: &[u8], tickets: &mut Tickets) -> Option> { + let started = Instant::now(); + let (call, mut arguments) = match rpc::parse_call(record) { + Ok(parsed) => parsed, + Err(Refused::Reply(reply)) => return Some(reply), + Err(Refused::Undecodable) => return None, + }; + let mut out = rpc::accepted(call.xid, Accept::Success); + let result = match (call.program, call.version) { + (nfs::PROGRAM, nfs::VERSION) => nfs::call( + &shared.kernel, + Policy { + read_only: shared.config.read_only, + }, + call.procedure, + &call.credentials, + &mut arguments, + &mut out, + tickets, + shared.verifier, + ), + (mount::PROGRAM, mount::VERSION) => mount::call( + &shared.kernel, + &shared.config.path, + call.procedure, + &mut arguments, + &mut out, + ), + (nfs::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + (mount::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + _ => Err(Accept::ProgramUnavailable), + }; + let reply = match result { + Ok(()) => out.into_bytes(), + Err(status) => rpc::accepted(call.xid, status).into_bytes(), + }; + shared + .counters + .charge(call.program, call.procedure, started); + Some(reply) +} diff --git a/engine/crates/layerfs-nfs/src/xdr.rs b/engine/crates/layerfs-nfs/src/xdr.rs new file mode 100644 index 0000000000..18f572ed27 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/xdr.rs @@ -0,0 +1,120 @@ +//! XDR (RFC 4506) primitives over one received record and one reply buffer. +use std::fmt; + +/// The record ended before a field, or a length exceeded its declared bound. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Garbage; +impl fmt::Display for Garbage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("malformed XDR arguments") + } +} +impl std::error::Error for Garbage {} + +/// Sequential decoding of one record. Every read is bounds-checked. +pub struct Reader<'a> { + data: &'a [u8], + at: usize, +} +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data, at: 0 } + } + fn take(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let end = self.at.checked_add(length).ok_or(Garbage)?; + let slice = self.data.get(self.at..end).ok_or(Garbage)?; + self.at = end; + Ok(slice) + } + pub fn u32(&mut self) -> Result { + let bytes = self.take(4)?; + Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + pub fn u64(&mut self) -> Result { + Ok((u64::from(self.u32()?) << 32) | u64::from(self.u32()?)) + } + pub fn bool(&mut self) -> Result { + match self.u32()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(Garbage), + } + } + /// Fixed-length opaque data, followed by its padding. + pub fn fixed(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let value = self.take(length)?; + self.take(pad(length))?; + Ok(value) + } + /// Variable-length opaque data of at most `limit` bytes. + pub fn opaque(&mut self, limit: usize) -> Result<&'a [u8], Garbage> { + let length = self.u32()? as usize; + if length > limit { + return Err(Garbage); + } + self.fixed(length) + } + /// The bytes not yet decoded. + pub fn remaining(&self) -> usize { + self.data.len() - self.at + } +} + +/// Bytes of padding after `length` bytes of opaque data. +pub const fn pad(length: usize) -> usize { + (4 - length % 4) % 4 +} +/// Encoded size of variable-length opaque data of `length` bytes. +pub const fn opaque_size(length: usize) -> usize { + 4 + length + pad(length) +} + +/// One reply under construction. +#[derive(Default)] +pub struct Writer { + data: Vec, +} +impl Writer { + pub fn new() -> Self { + Self::default() + } + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: Vec::with_capacity(capacity), + } + } + pub fn u32(&mut self, value: u32) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn u64(&mut self, value: u64) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn bool(&mut self, value: bool) -> &mut Self { + self.u32(u32::from(value)) + } + /// Fixed-length opaque data and its padding. + pub fn fixed(&mut self, value: &[u8]) -> &mut Self { + self.data.extend_from_slice(value); + self.data.extend_from_slice(&[0; 3][..pad(value.len())]); + self + } + /// Variable-length opaque data; callers bound `value` by its protocol limit. + pub fn opaque(&mut self, value: &[u8]) -> &mut Self { + self.u32(value.len() as u32); + self.fixed(value) + } + pub fn len(&self) -> usize { + self.data.len() + } + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + pub fn into_bytes(self) -> Vec { + self.data + } + pub fn as_bytes(&self) -> &[u8] { + &self.data + } +} diff --git a/engine/crates/layerfs-nfs/tests/collect.rs b/engine/crates/layerfs-nfs/tests/collect.rs new file mode 100644 index 0000000000..b4dfaddbe9 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/collect.rs @@ -0,0 +1,170 @@ +//! Reference collection (A3, plan A): an export configured with a small +//! reference bound walks a tree far larger than the bound, repeatedly, and +//! keeps both its own holdings and the engine's kernel entries bounded. +//! Handles of collected serials keep working (resolved again by their name); +//! a handle whose name now reaches another inode is stale; the product +//! Commit afterwards holds exactly what the client made. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, OK, STALE}; +use std::{fs, net::TcpListener, path::PathBuf, time::Duration}; + +const UID: u32 = 501; +const GID: u32 = 20; +const BOUND: usize = 64; +const DIRECTORIES: usize = 8; +const FILES: usize = 120; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-c{}-{label}", std::process::id())) +} +fn content(directory: usize, file: usize) -> Vec { + format!("d{directory}/f{file}\n") + .repeat(file % 7 + 1) + .into_bytes() +} +fn kernel_entries(rig: &rig::Rig) -> u64 { + let operation = rig.bound.operation().unwrap(); + operation.overlay().resources().unwrap().open.kernel_entries +} + +#[test] +fn references_stay_bounded_and_collected_handles_resolve() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + for directory in 0..DIRECTORIES { + let path = source.join(format!("d{directory}/inner")); + fs::create_dir_all(&path).unwrap(); + for file in 0..FILES { + fs::write( + source.join(format!("d{directory}/f{file}")), + content(directory, file), + ) + .unwrap(); + } + fs::write(path.join("leaf"), b"leaf\n").unwrap(); + } + let mut rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + rig.lookups_held = BOUND; + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let total = DIRECTORIES * (FILES + 3); + // The bound is soft: the most recently used serials, the directories a + // held serial was reached through and serials holding a descriptor are + // never collected. + let ceiling = BOUND + layerfs_nfs::kernel::RECENT_KEPT as usize + DIRECTORIES * 2 + 8; + let mut first: Vec<(Vec, u64, usize, usize)> = Vec::new(); + let mut peak_held = 0; + let mut peak_engine = 0; + for round in 0..6 { + for directory in 0..DIRECTORIES { + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + for file in 0..FILES { + let (handle, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + if round == 0 && file % 15 == 0 { + first.push((handle, attr.fileid, directory, file)); + } + } + let (inner, _) = client.lookup(&dir, b"inner").unwrap(); + client.lookup(&inner, b"leaf").unwrap(); + // Churn: a file made and removed in every directory, every round. + let made = format!("tmp{round}"); + client.touch(&dir, made.as_bytes(), 0o644).unwrap(); + client.remove(&dir, made.as_bytes()).unwrap(); + peak_held = peak_held.max(server.held().lookups); + peak_engine = peak_engine.max(kernel_entries(&rig)); + } + println!( + "ROUND {round} held={:?} engine_kernel_entries={}", + server.held(), + kernel_entries(&rig) + ); + } + assert!(peak_held <= ceiling, "export held {peak_held} > {ceiling}"); + // The engine counts the export's references (plus the root's). + assert!( + peak_engine <= ceiling as u64 + 2, + "engine kernel entries {peak_engine} > {ceiling}" + ); + assert!( + server.held().collected as usize >= total * 5, + "{:?}", + server.held() + ); + // Every early handle was collected long ago; each resolves to its file. + for (handle, fileid, directory, file) in &first { + let attr = client.getattr(handle).unwrap(); + assert_eq!(attr.fileid, *fileid); + assert_eq!(client.read_all(handle, 4096), content(*directory, *file)); + } + let resolved = server.held().resolved; + assert!(resolved >= first.len() as u64, "{:?}", server.held()); + // A collected handle whose name was replaced by another file is stale. + let (victim, victim_fileid, directory, file) = first[1].clone(); + for _ in 0..3 { + // collect it again + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d7").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + } + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + let replacement = client.touch(&dir, b"replacement", 0o644).unwrap(); + client + .rename(&dir, b"replacement", &dir, format!("f{file}").as_bytes()) + .unwrap(); + assert_eq!(client.getattr(&victim).map(|attr| attr.fileid), Err(STALE)); + let (now, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + assert_ne!(attr.fileid, victim_fileid); + assert_eq!(now, replacement.handle); + // `..` of a directory whose parent was collected still answers. + let (d3, _) = client.lookup(&root, b"d3").unwrap(); + let (inner, _) = client.lookup(&d3, b"inner").unwrap(); + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d6").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + let (up, up_attr) = client.lookup(&inner, b"..").unwrap(); + assert_eq!(up_attr.fileid, client.getattr(&d3).unwrap().fileid); + assert_eq!(client.getattr(&up).map(|_| OK), Ok(OK)); + // The Commit holds what the client made. + let closed = server.close_files(); + let done = rig.commit(); + assert!(done.history.is_some()); + println!("COMMITTED closed={closed} held={:?}", server.held()); + let view = rig.view([93; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (replaced, _) = reader + .walk_to(&view_root, &format!("d{directory}/f{file}")) + .unwrap(); + assert!(reader.read_all(&replaced, 4096).is_empty()); + let (kept, _) = reader.walk_to(&view_root, "d5/f119").unwrap(); + assert_eq!(reader.read_all(&kept, 4096), content(5, 119)); + assert_eq!( + reader.lookup(&view_root, b"tmp0").map(|_| ()), + Err(client::NOENT) + ); + drop(reader); + viewer.stop(); + rig.close_view(view); + drop(client); + let released = server.stop(); + println!("RELEASED {released:?}"); + assert!(released.lookups <= ceiling); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(&source).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/daemon.rs b/engine/crates/layerfs-nfs/tests/daemon.rs new file mode 100644 index 0000000000..eacaf9e35b --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/daemon.rs @@ -0,0 +1,292 @@ +//! A3: the daemon's control Service with NFS serving (the native route's +//! mount entry). Every step is a control request through `execute_control`, +//! the function the daemon's control connection answers with: Mount (bind), +//! Attach (export + client mount helper), Status, Commit (client flush, then +//! the unchanged product Commit), Unmount (dirty refusal, reversible busy, +//! terminal) and ForceUnmount. The client mount itself is a recording helper +//! here (this host's kernel has no NFSv3 client); the protocol client of the +//! tests plays the kernel. The kernel-client run is in the A3 VM scripts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, NOENT}; +use layerfs_bridge::control::{ + CommitControl, MountPhase, ReadyMount, Reply, Request, UnmountControl, WorkspaceToken, +}; +use layerfs_daemon::control::{ + ExportServing, ExportServingConfig, Failure, Helper, Mounter, Service, +}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + net::{Ipv4Addr, SocketAddr}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-d{}-{label}", std::process::id())) +} +/// A helper that appends ` ` to `log`; it answers "busy" +/// (exit 32) while `busy` exists. +fn recording(what: &str, log: &Path, busy: Option<&Path>) -> Helper { + let refuse = busy + .map(|flag| { + format!( + "if [ -e {} ]; then echo 'umount: target is busy' >&2; exit 32; fi; ", + flag.display() + ) + }) + .unwrap_or_default(); + Helper(vec![ + "sh".into(), + "-c".into(), + format!("{refuse}echo {what} {{port}} {{dir}} >> {}", log.display()), + ]) +} +fn bind(service: &Service, tag: u8, branch: layerfs_history::BranchId) -> WorkspaceToken { + let workspace = WorkspaceId::from_authority([tag; 32]).unwrap(); + match service + .execute_control(&Request::Mount { workspace, branch }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + } +} +fn attach(service: &Service, token: WorkspaceToken) -> ReadyMount { + match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + } +} +fn phase(service: &Service, token: WorkspaceToken) -> MountPhase { + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => status.kernel.expect("kernel block").phase, + other => panic!("{other:?}"), + } +} +fn commit(service: &Service, token: WorkspaceToken) -> Reply { + service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("over nfs".into()), + })) + .unwrap() + .reply +} +fn unmount(service: &Service, token: WorkspaceToken, discard: bool) -> Result { + service + .execute_control(&Request::Unmount(UnmountControl { token, discard })) + .map(|success| success.reply) +} +fn connect(ready: &ReadyMount) -> Client { + let port = u16::try_from(ready.receipt.mount_id).unwrap(); + Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), UID, GID) +} + +#[test] +fn workspace_control_over_the_nfs_route() { + let source = temp("src"); + let work = temp("work"); + let mounts = temp("mounts"); + let log = temp("helpers.log"); + let busy = temp("busy"); + for path in [&source, &work, &mounts] { + drop(fs::remove_dir_all(path)); + } + drop(fs::remove_file(&log)); + fs::create_dir_all(source.join("src")).unwrap(); + fs::write(source.join("README.md"), b"base readme\n").unwrap(); + fs::write(source.join("src/lib.rs"), b"pub fn base() {}\n").unwrap(); + fs::write(source.join("gone.txt"), b"to be removed\n").unwrap(); + fs::create_dir_all(&mounts).unwrap(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let mounter = Mounter { + mount: recording("mount", &log, None), + unmount: recording("unmount", &log, Some(&busy)), + force: recording("force", &log, None), + flush: Some(recording("flush", &log, None)), + }; + let export = Arc::new(ExportServing::start(ExportServingConfig::new( + mounts.clone(), + UID, + GID, + mounter, + ))); + let service = rig.service(export); + assert!(service.serves_nfs()); + + // Mount + Attach: Ready with an NFS receipt; the helper mounted it. + let token = bind(&service, 31, rig.branch); + assert_eq!(phase(&service, token), MountPhase::Unattached); + let ready = attach(&service, token); + assert_eq!(ready.receipt.abi_major, 0); + assert_eq!( + ready.directory, + mounts.join(token.namespace.to_string()).to_str().unwrap() + ); + assert!(Path::new(&ready.directory).is_dir()); + assert_eq!(phase(&service, token), MountPhase::Ready); + let port = ready.receipt.mount_id; + let logged = fs::read_to_string(&log).unwrap(); + assert_eq!(logged, format!("mount {port} {}\n", ready.directory)); + // A second Attach of the same Workspace is refused before effects. + assert!(service.execute_control(&Request::Attach(token)).is_err()); + + // Writes through the export. + let mut nfs = connect(&ready); + let root = nfs.mnt("/"); + let made = nfs.touch(&root, b"made.txt", 0o644).unwrap(); + nfs.write_all(&made.handle, 0, b"made over nfs\n", 8); + let dir = nfs.mkdir(&root, b"newdir", 0o755).unwrap(); + let inner = nfs.touch(&dir.handle, b"inner.bin", 0o600).unwrap(); + nfs.write_all(&inner.handle, 0, &vec![7u8; 300_000], 65536); + nfs.remove(&root, b"gone.txt").unwrap(); + let (src, _) = nfs.lookup(&root, b"src").unwrap(); + nfs.rename(&src, b"lib.rs", &dir.handle, b"lib.rs").unwrap(); + nfs.symlink(&root, b"link", b"newdir/lib.rs").unwrap(); + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => { + let block = status.kernel.unwrap(); + assert!(block.work.unwrap().served > 10, "{block:?}"); + assert!(status.local.unwrap().dirty_inodes > 0); + } + other => panic!("{other:?}"), + } + // Unmount refuses uncommitted changes, naming them; nothing closes. + match unmount(&service, token, false).unwrap() { + Reply::Dirty(report) => { + assert!(report.paths.iter().any(|p| p == "/made.txt"), "{report:?}") + } + other => panic!("{other:?}"), + } + assert_eq!(phase(&service, token), MountPhase::Ready); + + // Commit: the flush helper ran first, then the product Commit. + match commit(&service, token) { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + assert!(fs::read_to_string(&log) + .unwrap() + .contains(&format!("flush {port} "))); + // The export keeps serving after the Commit. + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + + // An independent view of the committed Branch. + let view = rig.view([92; 32]); + let viewer = rig.serve_view( + std::net::TcpListener::bind("127.0.0.1:0").unwrap(), + &view, + UID, + GID, + ); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (handle, _) = reader.walk_to(&view_root, "made.txt").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"made over nfs\n"); + let (handle, attr) = reader.walk_to(&view_root, "newdir/inner.bin").unwrap(); + assert_eq!((attr.size, attr.mode & 0o777), (300_000, 0o600)); + assert_eq!(reader.read_all(&handle, 65536), vec![7u8; 300_000]); + let (handle, _) = reader.walk_to(&view_root, "newdir/lib.rs").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"pub fn base() {}\n"); + assert_eq!( + reader.walk_to(&view_root, "gone.txt").map(|_| ()), + Err(NOENT) + ); + assert_eq!( + reader.walk_to(&view_root, "src/lib.rs").map(|_| ()), + Err(NOENT) + ); + let (handle, _) = reader.walk_to(&view_root, "link").unwrap(); + assert_eq!(reader.readlink(&handle).unwrap(), b"newdir/lib.rs"); + drop(reader); + viewer.stop(); + rig.close_view(view); + + // A busy client mount is the reversible answer: still Ready, serving. + fs::write(&busy, b"").unwrap(); + let refused = unmount(&service, token, true).unwrap_err(); + assert!(format!("{refused:?}").contains("busy"), "{refused:?}"); + assert_eq!(phase(&service, token), MountPhase::Ready); + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + fs::remove_file(&busy).unwrap(); + + // Terminal unmount: client unmount, export stopped, engine mount + // revoked, directory removed, Workspace closed. + let held = service.export_held(token).unwrap(); + assert_eq!( + unmount(&service, token, true).unwrap(), + Reply::Unmounted(token) + ); + assert!(!Path::new(&ready.directory).exists()); + let released = service.export_released(token).unwrap(); + assert_eq!(released.lookups, held.lookups); + assert_eq!(released.retained, 0); + drop(nfs); + assert!(service.execute_control(&Request::Status(token)).is_err()); + + // ForceUnmount of a second Workspace with uncommitted changes. + let second = bind(&service, 32, rig.branch); + let ready2 = attach(&service, second); + let mut nfs = connect(&ready2); + let root = nfs.mnt("/"); + nfs.touch(&root, b"lost.txt", 0o644).unwrap(); + match service + .execute_control(&Request::ForceUnmount { + token: second, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => assert_eq!(done.token, second), + other => panic!("{other:?}"), + } + assert!(!Path::new(&ready2.directory).exists()); + drop(nfs); + let logged = fs::read_to_string(&log).unwrap(); + println!("HELPERS\n{logged}"); + assert_eq!( + logged.lines().filter(|l| l.starts_with("mount ")).count(), + 2 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("unmount ")).count(), + 1 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("force ")).count(), + 1 + ); + + drop(service); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + for path in [&source, &mounts] { + fs::remove_dir_all(path).unwrap(); + } + fs::remove_file(&log).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/export.rs b/engine/crates/layerfs-nfs/tests/export.rs new file mode 100644 index 0000000000..96c25acc10 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/export.rs @@ -0,0 +1,196 @@ +//! A committed Branch served read-only over loopback NFSv3: every path, +//! byte, link target, mode and identity of the source tree through the +//! protocol alone, the synthetic kernel's reference discipline, cookies, +//! refusals and stale handles across engine mounts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{Attr, Client, BADHANDLE, BAD_COOKIE, NOENT, ROFS, STALE}; +use layerfs_nfs::xdr::Writer; +use std::{ + collections::BTreeSet, + fs, + net::TcpListener, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + time::Duration, +}; +use tree::{oracle, walk, Node}; + +const UID: u32 = 501; +const GID: u32 = 20; +const MANY: usize = 700; +const BIG: usize = 300_001; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested/deep")).unwrap(); + fs::create_dir_all(root.join("many")).unwrap(); + fs::create_dir_all(root.join("emptydir")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write( + root.join("big.bin"), + (0..BIG).map(|n| (n % 251) as u8).collect::>(), + ) + .unwrap(); + fs::write(root.join("empty"), b"").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep/file.txt"), b"deep\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/lib-alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + std::os::unix::fs::symlink("../nowhere", root.join("dangling")).unwrap(); + fs::write(root.join("unicod\u{e9}.txt"), "caf\u{e9}\n").unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\necho run\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); + for n in 0..MANY { + fs::write(root.join(format!("many/f-{n:04}")), format!("{n}\n")).unwrap(); + } +} +#[test] +fn a_committed_branch_reads_completely_over_nfs() { + let base = temp("complete"); + let tree = base.join("tree"); + source(&tree); + let expected = oracle(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + assert_eq!(rig.entries as usize, expected.len()); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + + let (paths, bytes, identities) = walk(&mut client, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + assert!(bytes > BIG); + // One inode behind both names of the hard link, two names for it, and + // distinct inodes everywhere else. + let alias = expected.iter().find_map(|(path, node)| match node { + Node::File { ino, .. } if path == Path::new("src/lib.rs") => Some(*ino), + _ => None, + }); + let fileids = &identities[&alias.unwrap()]; + assert_eq!(fileids.len(), 1, "both names show one fileid"); + let lib = client.mnt("/"); + let (src, _) = client.lookup(&lib, b"src").unwrap(); + let (_, lib_attr) = client.lookup(&src, b"lib.rs").unwrap(); + assert_eq!(lib_attr.nlink, 2); + let all: BTreeSet = identities.values().flatten().copied().collect(); + assert_eq!( + all.len(), + identities.len(), + "distinct inodes, distinct fileids" + ); + + // Re-walking takes no further engine references: one per handed-out serial. + // Every path but the root holds one reference; the hard link's two names + // share one serial. + let serials = expected.len() - 2; + let held = server.held(); + assert_eq!(held.lookups, serials, "one reference per non-root serial"); + walk(&mut client, &expected, 64 * 1024, (UID, GID)); + assert_eq!(server.held().lookups, held.lookups); + + let released = server.stop(); + assert_eq!(released.lookups, serials); + assert!(released.files > 0 && released.directories > 0); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0, "every Store reader was given back"); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn cookies_refusals_and_stale_handles() { + let base = temp("protocol"); + let tree = base.join("tree"); + source(&tree); + let mut rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (many, _) = client.lookup(&root, b"many").unwrap(); + + // A small budget spreads one enumeration over many replies; every name + // appears exactly once, in both procedures. + for plus in [false, true] { + let (entries, calls) = client.list(&many, 1024, plus); + let names: BTreeSet> = entries.iter().map(|entry| entry.name.clone()).collect(); + assert_eq!(names.len(), entries.len(), "no duplicate (plus={plus})"); + assert_eq!(entries.len(), MANY + 2, "plus={plus}"); + assert!(calls > 10, "{calls} calls (plus={plus})"); + } + // A verifier the export never issued, and a budget too small for one entry. + assert_eq!( + client.readdir(&many, 5, 0xdead_beef, 4096, false), + Err(BAD_COOKIE) + ); + // Missing names, a foreign handle and every changing procedure. + assert_eq!(client.lookup(&root, b"absent").map(|_| ()), Err(NOENT)); + // Both LOOKUP answers carry the directory's attributes. + let root_attr = client.getattr(&root).unwrap(); + let (status, found, directory) = client.lookup_with_directory(&root, b"many"); + assert_eq!( + (status, found.map(|(handle, _)| handle)), + (0, Some(many.clone())) + ); + assert_eq!(directory, Some(root_attr.clone())); + let (status, found, directory) = client.lookup_with_directory(&root, b"absent"); + assert_eq!((status, found), (NOENT, None)); + assert_eq!(directory, Some(root_attr)); + // Not a handle this export issues: wrong version or length. A version-1 + // handle of another engine mount is stale (its key is gone); a forged + // tag under this mount is refused (see protocol.rs). + assert_eq!(client.getattr(&[2; 25]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 10]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 25]), Err(STALE)); + for procedure in [2, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21] { + let mut arguments = Writer::new(); + arguments.opaque(&root); + assert_eq!(client.nfs(procedure, &arguments).0, ROFS, "{procedure}"); + } + // A read-only export grants reads only, whatever the mode allows. + let (readme, readme_attr): (Vec, Attr) = client.lookup(&root, b"README.md").unwrap(); + let host = fs::metadata(tree.join("README.md")).unwrap().mode() & 0o7777; + assert_eq!(readme_attr.mode, host); + assert_eq!(host & 0o444, 0o444, "the fixture is readable by everyone"); + assert_eq!(client.access(&readme, 0x3f), Ok(0x01)); + let mut other = Client::connect(server.local_addr(), 1234, 1234); + assert_eq!(other.access(&readme, 0x3f), Ok(0x01), "other: r--"); + let (run, _) = client.lookup(&root, b"run.sh").unwrap(); + assert_eq!(client.access(&run, 0x3f), Ok(0x01 | 0x20)); + // Reads beyond the end, and windows that are not page aligned. + let (big, big_attr) = client.lookup(&root, b"big.bin").unwrap(); + assert_eq!( + client.read(&big, big_attr.size + 10, 4096), + Ok((vec![], true)) + ); + let (window, eof) = client.read(&big, 1000, 70_000).unwrap(); + assert!(!eof); + assert_eq!( + window, + (1000..71_000).map(|n| (n % 251) as u8).collect::>() + ); + + // A restarted export is a new engine mount: earlier handles are stale. + drop(client); + drop(other); + let released = server.stop(); + rig.remount(); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + assert_eq!(client.getattr(&readme), Err(STALE)); + assert_eq!(client.getattr(&root), Err(STALE)); + let fresh = client.mnt("/"); + assert!(client.getattr(&fresh).is_ok()); + drop(client); + let released_again = server.stop(); + assert!(released.lookups > 0); + let (_, outstanding) = rig.stop(released_again); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs new file mode 100644 index 0000000000..781e82710a --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -0,0 +1,142 @@ +//! Real control Service with process helpers, no kernel NFS mount required. +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use layerfs_bridge::control::{MountPhase, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + path::PathBuf, + process::Command, + sync::{mpsc, Arc}, + time::Duration, +}; + +fn base(label: &str) -> PathBuf { + let root = std::env::var_os("H07_TEST_ROOT") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + root.join(format!("helper-{}-{label}", std::process::id())) +} +fn run_case(label: &str, helper: Helper, before: impl FnOnce()) -> (String, MountPhase) { + let base = base(label); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let service = rig.service(Arc::new(ExportServing::start(ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper::words("true").unwrap(), + unmount: helper, + force: Helper::words("true").unwrap(), + flush: None, + }, + )))); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([33; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + service.execute_control(&Request::Attach(token)).unwrap(); + before(); + let reply = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + let phase = match service.execute_control(&Request::Status(token)) { + Ok(done) => match done.reply { + Reply::Status(status) => status.kernel.unwrap().phase, + other => panic!("{other:?}"), + }, + Err(_) => MountPhase::Unattached, + }; + if phase == MountPhase::Retained { + let repeated = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + assert_eq!( + reply, repeated, + "retained request must not launch another helper" + ); + } + // No actual kernel mount exists in this fixture. Service owns retained + // export custody until drop. + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + (reply, phase) +} +#[test] +fn exited_helper_does_not_wait_for_a_descendants_pipe_eof() { + let root = base("pipe-holder"); + fs::create_dir_all(&root).unwrap(); + let pidfile = root.join("descendant.pid"); + let script = format!("sleep 20 & echo $! > {}; exit 0", pidfile.display()); + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = run_case( + "pipes", + Helper(vec!["sh".into(), "-c".into(), script]), + || {}, + ); + let _ = send.send(result); + }); + let result = receive.recv_timeout(Duration::from_secs(3)); + // Always release the descendant, including on failure, before asserting. + if let Ok(pid) = fs::read_to_string(&pidfile) { + let _ = Command::new("kill").args(["-KILL", pid.trim()]).status(); + } + worker.join().unwrap(); + fs::remove_dir_all(root).unwrap(); + let (reply, _) = result.expect("exited helper blocked on descendant's open pipe"); + assert!(reply.contains("Unmounted"), "{reply}"); +} +#[test] +fn timed_out_helper_returns_retained_and_does_not_replay() { + // STOP makes the helper ignore TERM; KILL must reap it without wait(). + // A normal process cannot ignore SIGKILL: this tests the actual deadline, + // not the kernel's uninterruptible U state. + let (reply, phase) = run_case( + "timeout", + Helper(vec!["sh".into(), "-c".into(), "kill -STOP $$".into()]), + || {}, + ); + assert!( + reply.contains("Retained") && reply.contains("60s"), + "{reply}" + ); + assert_eq!(phase, MountPhase::Retained); + assert_eq!(HELPER_WAIT, Duration::from_secs(60)); +} + +#[test] +fn verbose_helper_preserves_busy_refusal_without_pipe_deadlock() { + let (reply, phase) = run_case( + "verbose", + Helper(vec!["sh".into(), "-c".into(), "echo busy >&2; i=0; while [ $i -lt 5000 ]; do echo verbose-helper-output; i=$((i+1)); done; exit 32".into()]), + || {}, + ); + assert!(reply.to_ascii_lowercase().contains("busy"), "{reply}"); + assert_eq!(phase, MountPhase::Ready); +} diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs new file mode 100644 index 0000000000..968ae2e0e0 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -0,0 +1,17 @@ +//! The native macOS client must not send atime updates the export ignores. +#![cfg(target_os = "macos")] +use layerfs_daemon::control::Mounter; + +#[test] +fn host_mount_disables_access_time_updates() { + let mounter = Mounter::host(); + let options = mounter + .mount + .0 + .windows(2) + .find(|pair| pair[0] == "-o") + .expect("mount options"); + assert!(options[1].split(',').any(|option| option == "noatime")); + // Keep the existing attribute-cache contract; no global noac tradeoff. + assert!(options[1].split(',').any(|option| option == "actimeo=60")); +} diff --git a/engine/crates/layerfs-nfs/tests/protocol.rs b/engine/crates/layerfs-nfs/tests/protocol.rs new file mode 100644 index 0000000000..c513df60aa --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/protocol.rs @@ -0,0 +1,123 @@ +//! XDR, RPC record marking, call headers and file handles through the public API. +use layerfs_nfs::{ + handle::{HandleError, Handles, HANDLE_BYTES}, + rpc::{parse_call, read_record, write_record, Refused, RECORD_LIMIT}, + xdr::{opaque_size, Reader, Writer}, +}; + +#[test] +fn xdr_round_trips_with_padding() { + let mut out = Writer::new(); + out.u32(7) + .u64(u64::MAX - 1) + .bool(true) + .opaque(b"abcde") + .fixed(b"xy"); + assert_eq!(out.len(), 4 + 8 + 4 + opaque_size(5) + 4); + let bytes = out.into_bytes(); + let mut reader = Reader::new(&bytes); + assert_eq!(reader.u32(), Ok(7)); + assert_eq!(reader.u64(), Ok(u64::MAX - 1)); + assert_eq!(reader.bool(), Ok(true)); + assert_eq!(reader.opaque(5), Ok(&b"abcde"[..])); + assert_eq!(reader.fixed(2), Ok(&b"xy"[..])); + assert_eq!(reader.remaining(), 0); + assert!(reader.u32().is_err(), "reads past the record fail"); + let mut long = Writer::new(); + long.opaque(&[0; 9]); + let long = long.into_bytes(); + assert!( + Reader::new(&long).opaque(8).is_err(), + "a length beyond its bound" + ); + assert!( + Reader::new(&[0, 0, 0, 2]).bool().is_err(), + "a bool is 0 or 1" + ); +} + +#[test] +fn records_join_fragments_and_respect_their_bound() { + let mut wire = Vec::new(); + wire.extend_from_slice(&3u32.to_be_bytes()); + wire.extend_from_slice(b"abc"); + wire.extend_from_slice(&(2u32 | (1 << 31)).to_be_bytes()); + wire.extend_from_slice(b"de"); + write_record(&mut wire, b"next").unwrap(); + let mut stream = &wire[..]; + assert_eq!(read_record(&mut stream).unwrap(), Some(b"abcde".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), Some(b"next".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), None, "orderly end"); + let huge = (RECORD_LIMIT as u32 + 1) | (1 << 31); + assert!(read_record(&mut &huge.to_be_bytes()[..]).is_err()); +} + +fn call(rpcvers: u32, flavor: u32, credential: &[u8]) -> Vec { + let mut out = Writer::new(); + out.u32(9).u32(0).u32(rpcvers).u32(100_003).u32(3).u32(1); + out.u32(flavor).opaque(credential).u32(0).u32(0); + out.u32(0xabcd); + out.into_bytes() +} + +#[test] +fn call_headers_carry_unix_credentials() { + let mut unix = Writer::new(); + unix.u32(0) + .opaque(b"host") + .u32(501) + .u32(20) + .u32(2) + .u32(12) + .u32(61); + let record = call(2, 1, unix.as_bytes()); + let (header, mut arguments) = parse_call(&record).unwrap(); + assert_eq!( + (header.xid, header.program, header.version, header.procedure), + (9, 100_003, 3, 1) + ); + let credentials = header.credentials.unix.unwrap(); + assert_eq!( + (credentials.uid, credentials.gid, credentials.groups), + (501, 20, vec![12, 61]) + ); + assert_eq!(arguments.u32(), Ok(0xabcd)); + assert!(parse_call(&call(2, 0, b"")) + .unwrap() + .0 + .credentials + .unix + .is_none()); + // RPC version 3 and an unknown flavor are answered with a denial. + for record in [call(3, 0, b""), call(2, 6, b"")] { + match parse_call(&record) { + Err(Refused::Reply(reply)) => { + assert_eq!(&reply[..12], &[0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 1]) + } + Err(Refused::Undecodable) => panic!("undecodable"), + Ok((call, _)) => panic!("{call:?}"), + } + } + assert!(matches!( + parse_call(&[0, 0, 0, 9, 0, 0, 0, 1]), + Err(Refused::Undecodable) + )); +} + +#[test] +fn handles_are_stable_per_mount_and_stale_after_it() { + let handles = Handles::new(41); + let first = handles.encode(1234); + assert_eq!(first.len(), HANDLE_BYTES); + assert_eq!( + handles.encode(1234), + first, + "the same serial, the same bytes" + ); + assert_eq!(handles.decode(&first), Ok(1234)); + let mut forged = first; + forged[16] ^= 1; + assert_eq!(handles.decode(&forged), Err(HandleError::Bad)); + assert_eq!(handles.decode(&first[..24]), Err(HandleError::Bad)); + assert_eq!(Handles::new(42).decode(&first), Err(HandleError::Stale)); +} diff --git a/engine/crates/layerfs-nfs/tests/refresh.rs b/engine/crates/layerfs-nfs/tests/refresh.rs new file mode 100644 index 0000000000..7ff99c4d96 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/refresh.rs @@ -0,0 +1,117 @@ +//! Cache refresh after an install (B): the inodes the daemon names are +//! marked and their paths returned; a mode-only SETATTR on a marked inode is +//! answered with its current attributes and changes nothing, once; every +//! other SETATTR still changes the inode; the Commit holds only real changes. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, Set}; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-r{}-{label}", std::process::id())) +} + +#[test] +fn a_refresh_chmod_changes_nothing_and_other_setattrs_still_do() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + fs::create_dir_all(source.join("a/b")).unwrap(); + fs::write(source.join("a/b/file"), b"inner\n").unwrap(); + fs::write(source.join("top.txt"), b"top\n").unwrap(); + for (path, mode) in [("a/b/file", 0o644), ("top.txt", 0o644), ("a", 0o755)] { + fs::set_permissions(source.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (file, file_attr) = client.walk_to(&root, "a/b/file").unwrap(); + let (a, a_attr) = client.lookup(&root, b"a").unwrap(); + let (top, top_attr) = client.lookup(&root, b"top.txt").unwrap(); + let root_id = client.getattr(&root).unwrap().fileid; + // Paths of the serials the client knows, the root's empty; an unknown serial is left out. + let paths = server.refresh([ + file_attr.fileid, + a_attr.fileid, + root_id, + u64::MAX - 7, + top_attr.fileid, + ]); + assert_eq!( + paths, + vec![ + PathBuf::from("a/b/file"), + PathBuf::from("a"), + PathBuf::new(), + PathBuf::from("top.txt"), + ] + ); + let mode = |set: Result| { + set.map(|wcc| wcc.after.map(|after| after.mode & 0o7777)) + .map_err(|(status, _)| status) + }; + let chmod = |mode: u32| Set { + mode: Some(mode), + ..Set::default() + }; + // A marked inode: the mode-only SETATTR is answered, nothing changes. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o644)) + ); + assert_eq!( + mode(client.setattr(&a, chmod(0o700), None)), + Ok(Some(0o755)) + ); + assert_eq!(client.getattr(&file).unwrap().mode & 0o7777, 0o644); + // The mark is taken once: the next chmod is real. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o600)) + ); + // A SETATTR that is not mode-only is never taken for a refresh. + let truncate = Set { + mode: Some(0o640), + size: Some(0), + ..Set::default() + }; + let after = client.setattr(&top, truncate, None).unwrap().after.unwrap(); + assert_eq!((after.mode & 0o7777, after.size), (0o640, 0)); + // The Commit holds the real changes only. + server.close_files(); + assert!(rig.commit().history.is_some()); + let view = rig.view([71; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let at = |reader: &mut Client, path: &str| reader.walk_to(&view_root, path).unwrap().1; + assert_eq!(at(&mut reader, "a/b/file").mode & 0o7777, 0o600); + assert_eq!(at(&mut reader, "a").mode & 0o7777, 0o755); + let top_view = at(&mut reader, "top.txt"); + assert_eq!((top_view.mode & 0o7777, top_view.size), (0o640, 0)); + drop(reader); + let _released = viewer.stop(); + rig.close_view(view); + drop(client); + let served = server.stop(); + let (_, outstanding) = rig.stop(served); + assert_eq!(outstanding, 0); + for path in [&source, &work] { + drop(fs::remove_dir_all(Path::new(path))); + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/client.rs b/engine/crates/layerfs-nfs/tests/support/client.rs new file mode 100644 index 0000000000..d98b8f6ff0 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/client.rs @@ -0,0 +1,608 @@ +//! A minimal NFSv3/MOUNT client over the export's public XDR and record +//! framing: what a kernel client sends, one call at a time. +use layerfs_nfs::{ + rpc::{read_record, write_record}, + xdr::{Reader, Writer}, +}; +use std::{ + net::TcpStream, + sync::{Arc, Condvar, Mutex}, + time::Duration, +}; + +/// Calls sent and replies fully received by clients sharing it, so a test's +/// port wrapper can tell whether a reply was already on the wire. +#[derive(Default)] +pub struct Wire { + pub counts: Mutex<(u64, u64)>, + pub changed: Condvar, +} + +pub const NFS: u32 = 100_003; +pub const MOUNT: u32 = 100_005; +pub const OK: u32 = 0; +pub const NOENT: u32 = 2; +pub const ROFS: u32 = 30; +pub const STALE: u32 = 70; +pub const BADHANDLE: u32 = 10001; +pub const BAD_COOKIE: u32 = 10003; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Attr { + pub kind: u32, + pub mode: u32, + pub nlink: u32, + pub uid: u32, + pub gid: u32, + pub size: u64, + pub fileid: u64, + pub mtime: (u32, u32), +} +fn attr(reader: &mut Reader<'_>) -> Attr { + let kind = reader.u32().unwrap(); + let mode = reader.u32().unwrap(); + let nlink = reader.u32().unwrap(); + let uid = reader.u32().unwrap(); + let gid = reader.u32().unwrap(); + let size = reader.u64().unwrap(); + let _used = reader.u64().unwrap(); + let _rdev = (reader.u32().unwrap(), reader.u32().unwrap()); + let _fsid = reader.u64().unwrap(); + let fileid = reader.u64().unwrap(); + let _atime = (reader.u32().unwrap(), reader.u32().unwrap()); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let _ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + Attr { + kind, + mode, + nlink, + uid, + gid, + size, + fileid, + mtime, + } +} +fn post_op(reader: &mut Reader<'_>) -> Option { + reader.bool().unwrap().then(|| attr(reader)) +} +/// A LOOKUP answer: its status, the object's handle and attributes when +/// found, and the directory's attributes. +pub type Lookup = (u32, Option<(Vec, Attr)>, Option); +/// One READDIR/READDIRPLUS entry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Entry { + pub name: Vec, + pub fileid: u64, + pub cookie: u64, +} + +pub struct Client { + stream: TcpStream, + xid: u32, + uid: u32, + gid: u32, + pub calls: u64, + pub wire: Option>, +} +impl Client { + pub fn connect(address: std::net::SocketAddr, uid: u32, gid: u32) -> Self { + let stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + Self { + stream, + xid: 1, + uid, + gid, + calls: 0, + wire: None, + } + } + /// One call; the accept status and the results after it. + pub fn call(&mut self, program: u32, procedure: u32, arguments: &[u8]) -> (u32, Vec) { + self.xid += 1; + self.calls += 1; + let mut credential = Writer::new(); + credential + .u32(0) + .opaque(b"test") + .u32(self.uid) + .u32(self.gid) + .u32(0); + let mut call = Writer::new(); + call.u32(self.xid) + .u32(0) + .u32(2) + .u32(program) + .u32(3) + .u32(procedure); + call.u32(1).opaque(credential.as_bytes()); + call.u32(0).u32(0); + call.fixed(arguments); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().0 += 1; + } + write_record(&mut self.stream, call.as_bytes()).unwrap(); + let record = read_record(&mut self.stream).unwrap().unwrap(); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().1 += 1; + wire.changed.notify_all(); + } + let mut reader = Reader::new(&record); + assert_eq!(reader.u32().unwrap(), self.xid); + assert_eq!(reader.u32().unwrap(), 1, "a reply"); + assert_eq!(reader.u32().unwrap(), 0, "accepted"); + let _flavor = reader.u32().unwrap(); + let _verifier = reader.opaque(400).unwrap(); + let status = reader.u32().unwrap(); + let rest = record[record.len() - reader.remaining()..].to_vec(); + (status, rest) + } + /// One NFS procedure that the RPC layer accepted: its nfsstat3 and results. + pub fn nfs(&mut self, procedure: u32, arguments: &Writer) -> (u32, Vec) { + let (accept, rest) = self.call(NFS, procedure, arguments.as_bytes()); + assert_eq!(accept, 0, "procedure {procedure} accepted"); + let mut reader = Reader::new(&rest); + let status = reader.u32().unwrap(); + (status, rest[4..].to_vec()) + } + pub fn mnt(&mut self, path: &str) -> Vec { + let mut arguments = Writer::new(); + arguments.opaque(path.as_bytes()); + let (accept, rest) = self.call(MOUNT, 1, arguments.as_bytes()); + assert_eq!(accept, 0); + let mut reader = Reader::new(&rest); + assert_eq!(reader.u32().unwrap(), 0, "MNT3_OK"); + reader.opaque(64).unwrap().to_vec() + } + pub fn getattr(&mut self, handle: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(1, &arguments); + if status != OK { + return Err(status); + } + Ok(attr(&mut Reader::new(&rest))) + } + pub fn lookup(&mut self, directory: &[u8], name: &[u8]) -> Result<(Vec, Attr), u32> { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let handle = reader.opaque(64).unwrap().to_vec(); + let attributes = post_op(&mut reader).expect("LOOKUP returns object attributes"); + Ok((handle, attributes)) + } + /// LOOKUP with both halves of its answer: the status, the object's + /// handle and attributes when found, and the directory's attributes. + pub fn lookup_with_directory(&mut self, directory: &[u8], name: &[u8]) -> Lookup { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + let mut reader = Reader::new(&rest); + let object = (status == OK).then(|| { + let handle = reader.opaque(64).unwrap().to_vec(); + ( + handle, + post_op(&mut reader).expect("LOOKUP returns object attributes"), + ) + }); + let parent = post_op(&mut reader); + (status, object, parent) + } + pub fn access(&mut self, handle: &[u8], requested: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle).u32(requested); + let (status, rest) = self.nfs(4, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.u32().unwrap()) + } + pub fn readlink(&mut self, handle: &[u8]) -> Result, u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(5, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.opaque(4096).unwrap().to_vec()) + } + /// One READ: data and EOF. + pub fn read(&mut self, handle: &[u8], offset: u64, count: u32) -> Result<(Vec, bool), u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(offset).u32(count); + let (status, rest) = self.nfs(6, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let count = reader.u32().unwrap(); + let eof = reader.bool().unwrap(); + let data = reader.opaque(1 << 20).unwrap().to_vec(); + assert_eq!(count as usize, data.len()); + Ok((data, eof)) + } + /// READs until EOF in windows of `count` bytes. + pub fn read_all(&mut self, handle: &[u8], count: u32) -> Vec { + let mut bytes = Vec::new(); + loop { + let (data, eof) = self.read(handle, bytes.len() as u64, count).unwrap(); + bytes.extend_from_slice(&data); + if eof { + return bytes; + } + assert!(!data.is_empty(), "a short non-EOF read makes progress"); + } + } + /// One READDIR (or READDIRPLUS): entries, verifier, EOF. + pub fn readdir( + &mut self, + handle: &[u8], + cookie: u64, + verifier: u64, + budget: u32, + plus: bool, + ) -> Result<(Vec, u64, bool), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(cookie) + .u64(verifier) + .u32(budget); + if plus { + arguments.u32(budget); + } + let (status, rest) = self.nfs(if plus { 17 } else { 16 }, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let verifier = reader.u64().unwrap(); + let mut entries = Vec::new(); + while reader.bool().unwrap() { + let fileid = reader.u64().unwrap(); + let name = reader.opaque(4096).unwrap().to_vec(); + let cookie = reader.u64().unwrap(); + if plus { + assert!(post_op(&mut reader).is_none()); + assert!(!reader.bool().unwrap(), "no handle in READDIRPLUS"); + } + entries.push(Entry { + name, + fileid, + cookie, + }); + } + let eof = reader.bool().unwrap(); + Ok((entries, verifier, eof)) + } + /// A whole enumeration as a kernel client runs it, and the calls it took. + pub fn list(&mut self, handle: &[u8], budget: u32, plus: bool) -> (Vec, u64) { + let (mut cookie, mut verifier, mut calls) = (0, 0, 0); + let mut all = Vec::new(); + loop { + let (entries, next, eof) = self + .readdir(handle, cookie, verifier, budget, plus) + .unwrap(); + calls += 1; + verifier = next; + if let Some(last) = entries.last() { + cookie = last.cookie; + } + all.extend(entries); + if eof { + return (all, calls); + } + } + } +} + +pub const PERM: u32 = 1; +pub const EXIST: u32 = 17; +pub const NOTDIR: u32 = 20; +pub const ISDIR: u32 = 21; +pub const INVAL: u32 = 22; +pub const NAMETOOLONG: u32 = 63; +pub const NOTEMPTY: u32 = 66; +pub const ACCES: u32 = 13; +pub const NOT_SYNC: u32 = 10002; +pub const NOTSUPP: u32 = 10004; +pub const JUKEBOX: u32 = 10008; + +/// `wcc_attr`: size, mtime and ctime before a change. +pub type Before = (u64, (u32, u32), (u32, u32)); +/// `wcc_data`: the pre-op (size, mtime, ctime) and post-op attributes. +#[derive(Clone, Debug, Default)] +pub struct Wcc { + pub before: Option, + pub after: Option, +} +pub fn wcc(reader: &mut Reader<'_>) -> Wcc { + let before = reader.bool().unwrap().then(|| { + let size = reader.u64().unwrap(); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + (size, mtime, ctime) + }); + Wcc { + before, + after: post_op(reader), + } +} +/// A `sattr3` to send. +#[derive(Clone, Copy, Debug, Default)] +pub struct Set { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + /// None: don't change; Some(None): server time; Some(Some(t)): client time. + pub mtime: Option>, +} +pub fn sattr(out: &mut Writer, set: Set) { + let field = |out: &mut Writer, value: Option| match value { + Some(value) => { + out.bool(true).u32(value); + } + None => { + out.bool(false); + } + }; + field(out, set.mode); + field(out, set.uid); + field(out, set.gid); + match set.size { + Some(size) => { + out.bool(true).u64(size); + } + None => { + out.bool(false); + } + } + out.u32(0); // atime: don't change + match set.mtime { + None => { + out.u32(0); + } + Some(None) => { + out.u32(1); + } + Some(Some((seconds, nanoseconds))) => { + out.u32(2).u32(seconds).u32(nanoseconds); + } + } +} +/// The results of CREATE, MKDIR and SYMLINK. +#[derive(Clone, Debug)] +pub struct Made { + pub handle: Vec, + pub attr: Attr, + pub directory: Wcc, +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Set), + Guarded(Set), + Exclusive([u8; 8]), +} +impl Client { + fn made(&mut self, procedure: u32, arguments: &Writer) -> Result { + let (status, rest) = self.nfs(procedure, arguments); + let mut reader = Reader::new(&rest); + if status != OK { + return Err((status, wcc(&mut reader))); + } + assert!(reader.bool().unwrap(), "a created handle"); + let handle = reader.opaque(64).unwrap().to_vec(); + let attr = post_op(&mut reader).expect("created attributes"); + let directory = wcc(&mut reader); + Ok(Made { + handle, + attr, + directory, + }) + } + pub fn create(&mut self, directory: &[u8], name: &[u8], how: How) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + match how { + How::Unchecked(set) => { + arguments.u32(0); + sattr(&mut arguments, set); + } + How::Guarded(set) => { + arguments.u32(1); + sattr(&mut arguments, set); + } + How::Exclusive(verifier) => { + arguments.u32(2).fixed(&verifier); + } + } + self.made(8, &arguments).map_err(|(status, _)| status) + } + /// CREATE UNCHECKED with a mode, the common case. + pub fn touch(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.create(directory, name, How::Unchecked(set)) + } + pub fn mkdir(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + sattr(&mut arguments, set); + self.made(9, &arguments).map_err(|(status, _)| status) + } + pub fn symlink(&mut self, directory: &[u8], name: &[u8], target: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + sattr(&mut arguments, Set::default()); + arguments.opaque(target); + self.made(10, &arguments).map_err(|(status, _)| status) + } + pub fn mknod_fifo(&mut self, directory: &[u8], name: &[u8]) -> u32 { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name).u32(7); + sattr(&mut arguments, Set::default()); + self.nfs(11, &arguments).0 + } + /// SETATTR, optionally guarded by a ctime. + pub fn setattr( + &mut self, + handle: &[u8], + set: Set, + guard: Option<(u32, u32)>, + ) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + sattr(&mut arguments, set); + match guard { + Some((seconds, nanoseconds)) => { + arguments.bool(true).u32(seconds).u32(nanoseconds); + } + None => { + arguments.bool(false); + } + } + let (status, rest) = self.nfs(2, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err((status, found)) + } + } + /// WRITE: (count, committed, verifier, file wcc). + pub fn write( + &mut self, + handle: &[u8], + offset: u64, + data: &[u8], + ) -> Result<(u32, u32, [u8; 8], Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(offset) + .u32(data.len() as u32) + .u32(0) + .opaque(data); + let (status, rest) = self.nfs(7, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let file = wcc(&mut reader); + let count = reader.u32().unwrap(); + let committed = reader.u32().unwrap(); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok((count, committed, verifier, file)) + } + /// WRITE of everything, in pieces of `piece` bytes. + pub fn write_all(&mut self, handle: &[u8], offset: u64, data: &[u8], piece: usize) { + let mut at = 0; + while at < data.len() { + let end = (at + piece).min(data.len()); + let (count, _, _, _) = self + .write(handle, offset + at as u64, &data[at..end]) + .unwrap(); + assert!(count > 0); + at += count as usize; + } + } + pub fn commit(&mut self, handle: &[u8]) -> Result<[u8; 8], u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(0).u32(0); + let (status, rest) = self.nfs(21, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + wcc(&mut reader); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok(verifier) + } + /// REMOVE (`directory` false) or RMDIR. + pub fn unlink(&mut self, directory: &[u8], name: &[u8], rmdir: bool) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(if rmdir { 13 } else { 12 }, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn remove(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, false) + } + pub fn rmdir(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, true) + } + pub fn rename( + &mut self, + from: &[u8], + name: &[u8], + to: &[u8], + new_name: &[u8], + ) -> Result<(Wcc, Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(from) + .opaque(name) + .opaque(to) + .opaque(new_name); + let (status, rest) = self.nfs(14, &arguments); + let mut reader = Reader::new(&rest); + let found = (wcc(&mut reader), wcc(&mut reader)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn link(&mut self, file: &[u8], directory: &[u8], name: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(file).opaque(directory).opaque(name); + let (status, rest) = self.nfs(15, &arguments); + let mut reader = Reader::new(&rest); + let object = post_op(&mut reader); + wcc(&mut reader); + if status == OK { + Ok(object.expect("LINK returns the file's attributes")) + } else { + Err(status) + } + } + /// A path of names below `root`, looked up one by one. + pub fn walk_to(&mut self, root: &[u8], path: &str) -> Result<(Vec, Attr), u32> { + let mut handle = root.to_vec(); + let mut attr = self.getattr(root)?; + for name in path.split('/').filter(|name| !name.is_empty()) { + let (next, next_attr) = self.lookup(&handle, name.as_bytes())?; + handle = next; + attr = next_attr; + } + Ok((handle, attr)) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/rig.rs b/engine/crates/layerfs-nfs/tests/support/rig.rs new file mode 100644 index 0000000000..3c7b5ff4d1 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/rig.rs @@ -0,0 +1,295 @@ +//! A committed Branch from a host tree, bound to one Workspace and one engine +//! kernel mount, with no kernel connection: the NFS export is the only client. +use layerfs_daemon::{ + bootstrap::open_store, + store::{BindRequest, BoundWorkspace, Store}, + Mounted, Owner, OwnerClient, OwnerConfig, +}; +use layerfs_history::{ + BranchId, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryName, + LayerStackId, WorkspaceId, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{KernelMount, ProfileConfig}; +use layerfs_persistence::{ + Handles, PersistenceConfig, SqliteAcquisitionSchema, SqlitePersistenceProfile, +}; +use layerfs_project::{init, InitRequest}; +use layerfs_storage::{Storage, StoragePolicy}; +use layerfs_telemetry::timer::Timing; +use std::{ + net::TcpListener, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; + +const BINDING: &[u8] = b"layerfs-nfs-rig"; +const CURSOR: [u8; 32] = [71; 32]; +const CACHE: usize = 32 * 1024 * 1024; + +pub struct Rig { + pub directory: PathBuf, + store: Arc, + owner: Owner, + client: OwnerClient, + pub bound: Arc, + pub mount: KernelMount, + pub branch: BranchId, + /// Paths below the root that Init acquired. + pub entries: u64, + /// The reference bound exports of this rig are configured with. + pub lookups_held: usize, +} +impl Rig { + /// Initializes a Store from `source`, publishes Branch `main` at that + /// root, binds one Workspace and creates one engine kernel mount. + /// `directory` must not exist; everything the rig writes goes there. + pub fn new(source: &Path, directory: &Path, timeout: Duration) -> Self { + std::fs::create_dir_all(directory).unwrap(); + let sealed = directory.join("sealed.sqlite"); + let config = PersistenceConfig::sqlite(&sealed) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); + let handles = Handles::create( + config.clone(), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: CURSOR, + incarnation: 1, + }, + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let initialized = Timing::disabled("nfs.rig.init", |timing| { + init( + &storage, + &handles.history, + InitRequest { + source, + acquisition: &handles.acquisition, + stack: LayerStackId::from_authority([72; 16]), + name: HistoryName::new("nfs").unwrap(), + scope_seed: [73; 32], + deadline: Instant::now() + timeout, + }, + timing, + ) + }) + .0 + .unwrap(); + let branch = BranchId::from_authority([74; 16]); + handles + .history + .fork(&ForkRequest { + stack: initialized.stack.id, + branch, + name: HistoryName::new("main").unwrap(), + source: ForkSource::Layer(initialized.stack.head_layer), + }) + .unwrap(); + drop(storage); + let sealed = handles.seal().unwrap(); + let store = open_store( + PersistenceConfig { + path: sealed.path.clone(), + ..config + }, + BINDING, + CURSOR, + 4, + CACHE, + Default::default(), + ) + .unwrap(); + let owner = Owner::start( + &directory.join("overlay"), + ProfileConfig::default(), + OwnerConfig::default(), + ) + .unwrap(); + let client = owner.client(); + let bound = Arc::new( + store + .bind( + client.clone(), + BindRequest { + branch, + workspace: WorkspaceId::from_authority([75; 32]).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = client.kernel_mount(bound.route(), root).unwrap(); + Self { + directory: directory.to_owned(), + store, + owner, + client, + bound, + mount, + branch, + entries: initialized.entries, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } + /// Serves this rig's engine mount read-only on `listener`. + pub fn serve(&self, listener: TcpListener, uid: u32, gid: u32) -> Server { + self.export(listener, uid, gid, true) + } + /// Serves this rig's engine mount on `listener`, writable or not. + pub fn export(&self, listener: TcpListener, uid: u32, gid: u32, read_only: bool) -> Server { + self.export_services( + listener, + Arc::new(Mounted(self.bound.clone())), + uid, + gid, + read_only, + ) + } + /// Serves this rig's engine mount over `services` (a wrapper of the + /// rig's own ports, for example). + pub fn export_services( + &self, + listener: TcpListener, + services: Arc, + uid: u32, + gid: u32, + read_only: bool, + ) -> Server { + Server::start( + listener, + services, + self.mount, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// A daemon control Service over this rig's Store and overlay owner whose + /// Attach serves NFS (the native route's mount entry). + pub fn service( + &self, + export: Arc, + ) -> Arc { + Arc::new(layerfs_daemon::control::Service::with_export( + self.store.clone(), + &self.owner, + export, + )) + } + /// The product Commit of everything the Workspace published locally. + pub fn commit(&self) -> layerfs_daemon::store::CommitSuccess { + self.bound + .commit_captured(&layerfs_daemon::store::CommitOptions::default()) + .unwrap_or_else(|failure| panic!("Commit: {:?}", failure.error)) + } + /// A second Workspace bound to Branch `main` as it is now, with its own + /// engine mount: an independent view of what was committed. + pub fn view(&self, workspace: [u8; 32]) -> (Arc, KernelMount) { + let bound = Arc::new( + self.store + .bind( + self.client.clone(), + BindRequest { + branch: self.branch, + workspace: WorkspaceId::from_authority(workspace).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = self.client.kernel_mount(bound.route(), root).unwrap(); + (bound, mount) + } + /// Serves a view read-only. + pub fn serve_view( + &self, + listener: TcpListener, + view: &(Arc, KernelMount), + uid: u32, + gid: u32, + ) -> Server { + Server::start( + listener, + Arc::new(Mounted(view.0.clone())), + view.1, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only: true, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// Revokes and closes a view. + pub fn close_view(&self, view: (Arc, KernelMount)) { + self.client.kernel_revoke(view.0.route(), view.1).unwrap(); + view.0.close().unwrap(); + } + /// A second engine mount of the same Workspace, as a restarted export + /// would create: handles of the first are stale under it. + pub fn remount(&mut self) { + self.client + .kernel_revoke(self.bound.route(), self.mount) + .unwrap(); + self.mount = self + .client + .kernel_mount(self.bound.route(), self.mount.root_serial()) + .unwrap(); + } + /// Revokes the engine mount, closes the Workspace and stops the owner. + /// Returns the Store readers still open or quarantined afterwards. + pub fn stop(self, served: Released) -> (Released, u64) { + let Self { + directory, + store, + owner, + client, + bound, + mount, + .. + } = self; + client.kernel_revoke(bound.route(), mount).unwrap(); + bound.close().unwrap(); + drop(bound); + // A closed Workspace keeps no Store reader and none was quarantined; + // the Store's own control reader stays open. + let work = store.read_work(); + assert!(!work.poisoned && work.opening_failures == 0, "{work:?}"); + let outstanding = work.readers.saturating_sub(1) + work.quarantined; + owner.stop().unwrap(); + drop(store); + std::fs::remove_dir_all(directory).unwrap(); + (served, outstanding as u64) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/spy.rs b/engine/crates/layerfs-nfs/tests/support/spy.rs new file mode 100644 index 0000000000..0f329a20f3 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/spy.rs @@ -0,0 +1,199 @@ +//! A wrapper of the rig's request ports that watches the publication +//! ticket contract from outside the export: every `reply_attempted` waits +//! until the test's client has received the reply of the call that made the +//! publication, +//! and records whether that happened (a release before the reply reached the +//! wire would wait forever on a synchronous client, so it times out and is +//! counted as a violation). It can also refuse serial reservations, as a +//! contended allocator does. +use super::client::Wire; +use layerfs_fuse::ports::{ + MountServices, RequestServices, ServiceError, ServiceFuture, ServiceReply, +}; +use layerfs_overlay::{KernelCookieOffer, KernelMount, Publication}; +use layerfs_workspace::{ + BaseView, KernelDirectoryWindow, KernelMutationOutcome, KernelReadOperation, KernelReadOutcome, + KernelVisitRequest, KernelWindow, VisitFacts, WriteData, +}; +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +/// What the spy saw. +#[derive(Default)] +pub struct Watch { + pub wire: Arc, + /// Releases made after the reply was received, and before it. + pub released_after: AtomicU64, + pub released_before: AtomicU64, + /// Publishing visits (`Applied` outcomes) and reservations refused. + pub published: AtomicU64, + /// Reservations still to refuse. + pub refuse_serials: AtomicU64, + pub refused: AtomicU64, + pub log: Mutex>, + /// Each publication and the call count when it was made: the client was + /// then waiting for exactly that call's reply. + pub owed: Mutex>, +} +pub struct Spy { + pub inner: Arc, + pub watch: Arc, +} +impl MountServices for Spy { + fn request( + &self, + fence: &layerfs_fuse::ports::Fence, + ) -> Result, ServiceError> { + Ok(Arc::new(Request { + inner: self.inner.request(fence)?, + watch: self.watch.clone(), + })) + } +} +struct Request { + inner: Arc, + watch: Arc, +} +impl RequestServices for Request { + fn directory_visit( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + offset: u64, + after: Option>, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .directory_visit(mount, serial, handle, offset, after) + } + fn publish_cookies( + &self, + offer: KernelCookieOffer, + names: Vec>, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.publish_cookies(offer, names) + } + fn close_directory( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_directory(mount, serial, handle) + } + fn read_visit( + &self, + mount: KernelMount, + serial: u64, + handle: Option, + offset: u64, + length: u32, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner.read_visit(mount, serial, handle, offset, length) + } + fn close_file( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_file(mount, serial, handle) + } + fn provider_failure(&self) -> Result, ServiceError> { + self.inner.provider_failure() + } + fn failed_base_read(&self, step: ServiceError) -> ServiceError { + self.inner.failed_base_read(step) + } + fn observe_visit( + &self, + mount: KernelMount, + request: u64, + serial: u64, + handle: Option, + operation: KernelReadOperation, + facts: Arc, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .observe_visit(mount, request, serial, handle, operation, facts) + } + fn base(&self) -> ServiceFuture<'_, BaseView> { + self.inner.base() + } + fn mutate_visit( + &self, + request: KernelVisitRequest, + ) -> ServiceFuture<'_, ServiceReply>> { + let watch = self.watch.clone(); + let future = self.inner.mutate_visit(request); + Box::pin(async move { + let reply = future.await?; + if let Ok(layerfs_workspace::JobOutcome::Applied { publication, .. }) = + &reply.get().result + { + watch.published.fetch_add(1, Ordering::AcqRel); + let sent = watch.wire.counts.lock().unwrap().0; + watch.owed.lock().unwrap().push((*publication, sent)); + } + Ok(reply) + }) + } + fn recognize(&self, serial: u64, offset: u64, data: WriteData) -> ServiceFuture<'_, WriteData> { + self.inner.recognize(serial, offset, data) + } + fn forget( + &self, + mount: KernelMount, + serial: u64, + count: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.forget(mount, serial, count) + } + fn reserve_serial(&self) -> Result, ServiceError> { + let refuse = self + .watch + .refuse_serials + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |left| { + left.checked_sub(1) + }) + .is_ok(); + if refuse { + self.watch.refused.fetch_add(1, Ordering::AcqRel); + return Ok(None); + } + self.inner.reserve_serial() + } + fn reply_attempted(&self, publication: Publication) -> ServiceFuture<'_, ServiceReply<()>> { + let wire = &self.watch.wire; + let owed = { + let mut owed = self.watch.owed.lock().unwrap(); + let at = owed.iter().position(|(made, _)| *made == publication); + at.map_or(u64::MAX, |at| owed.remove(at).1) + }; + let counts = wire.counts.lock().unwrap(); + // The reply of the call that published this ticket must reach the + // client before the release, whatever the client sends next. + let (counts, timeout) = wire + .changed + .wait_timeout_while(counts, Duration::from_secs(5), |(_, received)| { + *received < owed + }) + .unwrap(); + let (sent, received) = *counts; + drop(counts); + if timeout.timed_out() { + self.watch.released_before.fetch_add(1, Ordering::AcqRel); + self.watch.log.lock().unwrap().push(format!( + "released before reply: sent {sent} received {received}" + )); + } else { + self.watch.released_after.fetch_add(1, Ordering::AcqRel); + } + self.inner.reply_attempted(publication) + } +} diff --git a/engine/crates/layerfs-nfs/tests/support/tree.rs b/engine/crates/layerfs-nfs/tests/support/tree.rs new file mode 100644 index 0000000000..e1bb648bc8 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/support/tree.rs @@ -0,0 +1,136 @@ +//! A host tree as an oracle, and a protocol walk that checks an export +//! against it: every path, kind, mode, byte, link target, listing, `..`, +//! directory link count and the identity behind hard links. +use super::client::Client; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +/// What the host tree holds, read before the export runs. +#[derive(Debug)] +pub enum Node { + File { mode: u32, bytes: Vec, ino: u64 }, + Directory { mode: u32, names: BTreeSet> }, + Link { target: Vec }, +} +pub fn oracle(root: &Path) -> BTreeMap { + use std::os::unix::ffi::OsStrExt; + let mut nodes = BTreeMap::new(); + let mut pending = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + let path = root.join(&relative); + let meta = fs::symlink_metadata(&path).unwrap(); + let node = if meta.file_type().is_symlink() { + Node::Link { + target: fs::read_link(&path) + .unwrap() + .as_os_str() + .as_bytes() + .to_vec(), + } + } else if meta.is_dir() { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(&path).unwrap() { + let name = entry.unwrap().file_name(); + pending.push(relative.join(&name)); + names.insert(name.as_bytes().to_vec()); + } + Node::Directory { + mode: meta.mode() & 0o7777, + names, + } + } else { + Node::File { + mode: meta.mode() & 0o7777, + bytes: fs::read(&path).unwrap(), + ino: meta.ino(), + } + }; + nodes.insert(relative, node); + } + nodes +} + +/// Walks the export from `handle` and checks every node against the oracle. +/// Returns (paths, regular bytes read) and the fileid of every host inode. +pub fn walk( + client: &mut Client, + oracle: &BTreeMap, + budget: u32, + owner: (u32, u32), +) -> (usize, usize, BTreeMap>) { + use std::os::unix::ffi::OsStrExt; + let root = client.mnt("/"); + let root_attr = client.getattr(&root).unwrap(); + let mut identities: BTreeMap> = BTreeMap::new(); + let (mut paths, mut bytes) = (0, 0); + let mut pending = vec![(PathBuf::new(), root, root_attr.clone(), root_attr.fileid)]; + while let Some((relative, handle, attr, parent)) = pending.pop() { + paths += 1; + assert_eq!((attr.uid, attr.gid), owner, "{relative:?}"); + match &oracle[&relative] { + Node::File { + mode, + bytes: expected, + ino, + } => { + assert_eq!(attr.kind, 1, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + assert_eq!(attr.size, expected.len() as u64, "{relative:?} size"); + let actual = client.read_all(&handle, 128 * 1024); + assert_eq!(&actual, expected, "{relative:?} bytes"); + bytes += actual.len(); + identities.entry(*ino).or_default().insert(attr.fileid); + } + Node::Link { target } => { + assert_eq!(attr.kind, 5, "{relative:?}"); + assert_eq!(&client.readlink(&handle).unwrap(), target, "{relative:?}"); + } + Node::Directory { mode, names } => { + assert_eq!(attr.kind, 2, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + let (entries, _) = client.list(&handle, budget, false); + let mut listed = BTreeSet::new(); + for entry in &entries { + assert!(listed.insert(entry.name.clone()), "{relative:?} duplicate"); + } + let dot = entries.iter().find(|entry| entry.name == b".").unwrap(); + let dotdot = entries.iter().find(|entry| entry.name == b".."); + assert_eq!(dot.fileid, attr.fileid, "{relative:?} ."); + if let Some(dotdot) = dotdot { + assert_eq!(dotdot.fileid, parent, "{relative:?} .."); + } + listed.remove(&b"."[..]); + listed.remove(&b".."[..]); + assert_eq!(&listed, names, "{relative:?} complete listing"); + let (up, up_attr) = client.lookup(&handle, b"..").unwrap(); + assert_eq!(up_attr.fileid, parent, "{relative:?} LOOKUP .."); + drop(up); + // Subdirectory link counts: two plus child directories. + let subdirectories = names + .iter() + .filter(|name| { + matches!( + oracle.get(&relative.join(std::ffi::OsStr::from_bytes(name))), + Some(Node::Directory { .. }) + ) + }) + .count(); + assert_eq!( + attr.nlink as usize, + subdirectories + 2, + "{relative:?} nlink" + ); + for name in names { + let (child, child_attr) = client.lookup(&handle, name).unwrap(); + let child_path = relative.join(std::ffi::OsStr::from_bytes(name)); + pending.push((child_path, child, child_attr, attr.fileid)); + } + } + } + } + (paths, bytes, identities) +} diff --git a/engine/crates/layerfs-nfs/tests/write.rs b/engine/crates/layerfs-nfs/tests/write.rs new file mode 100644 index 0000000000..ba11401ff3 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/write.rs @@ -0,0 +1,687 @@ +//! A writable Workspace served over loopback NFSv3: every changing +//! procedure and its error statuses, the publication-ticket contract (each +//! ticket released only after its reply was received, contended serial +//! reservations), the export's own handling of client silly renames and +//! AppleDouble names, and a product Commit whose tree equals what the same +//! operations made of a host directory. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/spy.rs"] +mod spy; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{ + Client, How, Set, ACCES, EXIST, INVAL, ISDIR, JUKEBOX, NAMETOOLONG, NOENT, NOTDIR, NOTEMPTY, + NOTSUPP, NOT_SYNC, OK, PERM, STALE, +}; +use layerfs_daemon::Mounted; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; +const FILE_SYNC: u32 = 2; +const LINUX_SILLY: &[u8] = b".nfs00000000000012ab00000001"; +const MAC_SILLY: &[u8] = b".nfs.20051025.b896"; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-w{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested")).unwrap(); + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep.txt"), b"deep\n").unwrap(); + fs::write(root.join("docs/guide.md"), b"guide\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); +} +fn pattern(seed: u32, length: usize) -> Vec { + (0..length) + .map(|n| (n as u32).wrapping_mul(31).wrapping_add(seed * 97) as u8) + .collect() +} + +#[test] +fn every_changing_procedure_and_its_statuses() { + let base = temp("procedures"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + + // CREATE: UNCHECKED, GUARDED over an existing name, UNCHECKED again. + let made = c.touch(&root, b"new.txt", 0o640).unwrap(); + assert_eq!( + (made.attr.kind, made.attr.mode, made.attr.size), + (1, 0o640, 0) + ); + assert_eq!((made.attr.uid, made.attr.gid), (UID, GID)); + assert!(made.directory.before.is_some() && made.directory.after.is_some()); + let guarded = How::Guarded(Set::default()); + assert_eq!(c.create(&root, b"new.txt", guarded).map(|_| ()), Err(EXIST)); + let again = c.touch(&root, b"new.txt", 0o600).unwrap(); + assert_eq!(again.attr.fileid, made.attr.fileid); + assert_eq!( + again.attr.mode, 0o640, + "UNCHECKED keeps an existing file's mode" + ); + let file = made.handle.clone(); + + // WRITE: FILE_SYNC with this export's verifier; a hole reads as zeros; + // a call larger than one engine window is split and fully written. + let (count, committed, verifier, wcc) = c.write(&file, 0, b"hello").unwrap(); + assert_eq!( + (count, committed, verifier), + (5, FILE_SYNC, server.verifier()) + ); + assert_eq!(wcc.after.unwrap().size, 5); + let big = pattern(1, 300_000); + let (count, _, _, wcc) = c.write(&file, 10, &big).unwrap(); + assert_eq!(count as usize, big.len()); + assert_eq!(wcc.after.unwrap().size, 300_010); + let mut expected = b"hello\0\0\0\0\0".to_vec(); + expected.extend_from_slice(&big); + assert_eq!(c.read_all(&file, 128 * 1024), expected); + assert_eq!(c.commit(&file), Ok(server.verifier())); + + // SETATTR: truncate, mode, client mtime, owner no-op and refusal, guard. + let set = |size, mode| Set { + size, + mode, + ..Set::default() + }; + let after = c + .setattr(&file, set(Some(3), None), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.size, 3); + assert_eq!(c.read_all(&file, 4096), b"hel"); + let after = c + .setattr(&file, set(None, Some(0o600)), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.mode, 0o600); + let stamp = Set { + mtime: Some(Some((1_000_000_000, 5))), + ..Set::default() + }; + let after = c.setattr(&file, stamp, None).unwrap().after.unwrap(); + assert_eq!(after.mtime, (1_000_000_000, 5)); + let same = Set { + uid: Some(UID), + gid: Some(GID), + ..Set::default() + }; + assert!( + c.setattr(&file, same, None).is_ok(), + "chown to the same identity" + ); + let other = Set { + uid: Some(0), + ..Set::default() + }; + assert_eq!( + c.setattr(&file, other, None).map(|_| ()).unwrap_err().0, + PERM + ); + let wrong = c.setattr(&file, set(Some(1), None), Some((7, 7))); + assert_eq!(wrong.map(|_| ()).unwrap_err().0, NOT_SYNC); + assert_eq!( + c.getattr(&file).unwrap().size, + 3, + "a failed guard changes nothing" + ); + let right = c.setattr(&file, set(Some(1), None), Some(after.mtime)); + assert_eq!(right.unwrap().after.unwrap().size, 1); + + // MKDIR, SYMLINK, MKNOD. + let dir = c.mkdir(&root, b"d", 0o750).unwrap(); + assert_eq!( + (dir.attr.kind, dir.attr.mode, dir.attr.nlink), + (2, 0o750, 2) + ); + assert_eq!(c.mkdir(&root, b"d", 0o750).map(|_| ()), Err(EXIST)); + let (_, up) = c.lookup(&dir.handle, b"..").unwrap(); + assert_eq!(up.fileid, c.getattr(&root).unwrap().fileid); + let link = c.symlink(&root, b"s", b"new.txt").unwrap(); + assert_eq!(link.attr.kind, 5); + assert_eq!(c.readlink(&link.handle).unwrap(), b"new.txt"); + let long = vec![b'x'; 4096]; + assert_eq!( + c.symlink(&root, b"long", &long).map(|_| ()), + Err(NAMETOOLONG) + ); + assert_eq!(c.mknod_fifo(&root, b"fifo"), NOTSUPP); + assert_eq!(c.lookup(&root, b"fifo").map(|_| ()), Err(NOENT)); + + // LINK: one more name and link count; directories and taken names refused. + let linked = c.link(&file, &dir.handle, b"alias").unwrap(); + assert_eq!((linked.fileid, linked.nlink), (made.attr.fileid, 2)); + assert_ne!(c.link(&dir.handle, &root, b"dirlink").map(|_| ()), Ok(())); + assert_eq!(c.link(&file, &root, b"README.md").map(|_| ()), Err(EXIST)); + + // RENAME: across directories, of a directory, replacing, into itself. + c.rename(&root, b"new.txt", &dir.handle, b"moved").unwrap(); + assert_eq!(c.lookup(&root, b"new.txt").map(|_| ()), Err(NOENT)); + let (_, moved) = c.lookup(&dir.handle, b"moved").unwrap(); + assert_eq!(moved.fileid, made.attr.fileid); + let sub = c.mkdir(&dir.handle, b"sub", 0o755).unwrap(); + c.rename(&dir.handle, b"sub", &root, b"top").unwrap(); + let (_, up) = c.lookup(&sub.handle, b"..").unwrap(); + assert_eq!( + up.fileid, + c.getattr(&root).unwrap().fileid, + ".. follows a move" + ); + assert_eq!(c.getattr(&dir.handle).unwrap().nlink, 2); + assert_ne!( + c.rename(&root, b"d", &dir.handle, b"inside").map(|_| ()), + Ok(()) + ); + let (readme, readme_attr) = c.lookup(&root, b"README.md").unwrap(); + c.rename(&root, b"run.sh", &root, b"README.md").unwrap(); + assert_eq!(c.getattr(&readme), Err(STALE), "the replaced inode is gone"); + let (_, replaced) = c.lookup(&root, b"README.md").unwrap(); + assert_ne!(replaced.fileid, readme_attr.fileid); + assert_eq!(replaced.mode, 0o755); + let missing = c.rename(&root, b"absent", &root, b"x").map(|_| ()); + assert_eq!(missing, Err(NOENT)); + + // REMOVE and RMDIR. + c.remove(&dir.handle, b"alias").unwrap(); + assert_eq!( + c.getattr(&file).unwrap().nlink, + 1, + "the other name keeps it" + ); + assert_eq!(c.remove(&root, b"absent").map(|_| ()), Err(NOENT)); + assert_eq!(c.remove(&root, b"d").map(|_| ()), Err(ISDIR)); + assert_eq!(c.rmdir(&root, b"d").map(|_| ()), Err(NOTEMPTY)); + assert_eq!(c.rmdir(&root, b"README.md").map(|_| ()), Err(NOTDIR)); + assert_eq!(c.rmdir(&root, b".").map(|_| ()), Err(INVAL)); + c.remove(&dir.handle, b"moved").unwrap(); + assert_eq!(c.getattr(&file), Err(STALE), "its last name is gone"); + c.rmdir(&root, b"d").unwrap(); + assert_eq!(c.getattr(&dir.handle), Err(STALE)); + + // Names the export refuses or the protocol forbids. + // An AppleDouble file is created detached (see the test below); other + // kinds under such a name are refused. + c.touch(&root, b"._new.txt", 0o644).unwrap(); + c.remove(&root, b"._new.txt").unwrap(); + assert_eq!(c.mkdir(&root, b"._d", 0o755).map(|_| ()), Err(ACCES)); + let refused = c.rename(&root, b"s", &root, b"._s").map(|_| ()); + assert_eq!(refused, Err(ACCES)); + assert_eq!(c.touch(&root, b".", 0o644).map(|_| ()), Err(EXIST)); + let long_name = vec![b'n'; 256]; + assert_eq!( + c.touch(&root, &long_name, 0o644).map(|_| ()), + Err(NAMETOOLONG) + ); + + // EXCLUSIVE: a retransmission finds its own file; another verifier does not. + let first = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + let again = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + assert_eq!(first.attr.fileid, again.attr.fileid); + let other = c.create(&root, b"excl", How::Exclusive(*b"another!")); + assert_eq!(other.map(|_| ()), Err(EXIST)); + + // ACCESS on a writable export grants changes by mode. + let (guide, _) = c.walk_to(&root, "docs/guide.md").unwrap(); + assert_eq!(c.access(&guide, 0x3f), Ok(0x01 | 0x04 | 0x08)); + let (docs, _) = c.lookup(&root, b"docs").unwrap(); + assert_eq!(c.access(&docs, 0x3f), Ok(0x01 | 0x02 | 0x04 | 0x08 | 0x10)); + + // Every serial handed out holds one reference: removed ones gave theirs back. + let held = server.held(); + assert_eq!( + ( + held.silly, + held.retained, + held.names_refused, + held.companions, + held.names_detached + ), + (0, 0, 2, 0, 1) + ); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn silly_renames_stay_out_of_the_workspace() { + let base = temp("silly"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + for (silly, name) in [(LINUX_SILLY, &b"open.bin"[..]), (MAC_SILLY, b"open2.bin")] { + let made = c.touch(&root, name, 0o644).unwrap(); + c.write(&made.handle, 0, b"still open").unwrap(); + assert_eq!(c.lookup(&root, silly).map(|_| ()), Err(NOENT)); + c.rename(&root, name, &root, silly).unwrap(); + assert_eq!(c.lookup(&root, name).map(|_| ()), Err(NOENT)); + let (handle, attr) = c.lookup(&root, silly).unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 10); + // The open file stays readable, writable and truncatable. + c.write(&handle, 10, b"!").unwrap(); + assert_eq!(c.read_all(&handle, 4096), b"still open!"); + let truncate = Set { + size: Some(5), + ..Set::default() + }; + assert_eq!( + c.setattr(&handle, truncate, None) + .unwrap() + .after + .unwrap() + .size, + 5 + ); + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| e.name != silly && e.name != name)); + assert_eq!(server.held().silly, 1); + c.remove(&root, silly).unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().silly, 0); + } + // A user's own names that only resemble the patterns are ordinary. + let made = c.touch(&root, b".nfsrc", 0o644).unwrap(); + c.rename(&root, b".nfsrc", &root, b".nfs-old").unwrap(); + assert!(c.lookup(&root, b".nfs-old").is_ok()); + drop(made); + // A silly name still held at Commit is not captured; neither is the + // name it replaced. + let kept = c.touch(&root, b"kept.bin", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + c.rename(&root, b"kept.bin", &root, LINUX_SILLY).unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([76; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b".nfs-old".to_vec())); + assert!(!names + .iter() + .any(|n| n.starts_with(b".nfs0") || n == b"kept.bin")); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + // Release gives the silly inode back. + drop(c); + let released = server.stop(); + assert_eq!((released.silly, released.silly_renamed), (1, 3)); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn apple_double_files_stay_out_of_the_workspace() { + let base = temp("appledouble"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + // What macOS copyfile does for a source carrying extended attributes: + // the data file, then its `._` companion with the attributes. + let data = c.touch(&root, b"copied.rs", 0o644).unwrap(); + c.write(&data.handle, 0, b"fn main() {}").unwrap(); + let made = c.touch(&root, b"._copied.rs", 0o644).unwrap(); + c.write(&made.handle, 0, b"AppleDouble header").unwrap(); + // It reads back under its name for the export's life ... + let (handle, attr) = c.lookup(&root, b"._copied.rs").unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 18); + assert_eq!(c.read_all(&handle, 4096), b"AppleDouble header"); + // ... opening it again truncates it, GUARDED finds it taken ... + let truncate = Set { + size: Some(0), + ..Set::default() + }; + let again = c + .create(&root, b"._copied.rs", How::Unchecked(truncate)) + .unwrap(); + assert_eq!( + (again.handle.clone(), again.attr.size), + (made.handle.clone(), 0) + ); + let guarded = c.create(&root, b"._copied.rs", How::Guarded(Set::default())); + assert_eq!(guarded.map(|_| ()), Err(EXIST)); + // ... READDIR does not list it, it moves only to another AppleDouble name, + // and a real file cannot take such a name. + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| !e.name.starts_with(b"._"))); + c.rename(&root, b"._copied.rs", &root, b"._moved.rs") + .unwrap(); + assert_eq!(c.lookup(&root, b"._copied.rs").map(|_| ()), Err(NOENT)); + let back = c + .rename(&root, b"._moved.rs", &root, b"plain.rs") + .map(|_| ()); + assert_eq!(back, Err(ACCES)); + let onto = c.rename(&root, b"copied.rs", &root, b"._x").map(|_| ()); + assert_eq!(onto, Err(ACCES)); + assert_eq!(server.held().companions, 1); + // REMOVE gives it back. + c.remove(&root, b"._moved.rs").unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().companions, 0); + // One still held at Commit is not captured; the data file is. + let kept = c.touch(&root, b"._kept", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b"copied.rs".to_vec())); + assert!(!names.iter().any(|n| n.starts_with(b"._"))); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + drop(c); + let released = server.stop(); + assert_eq!( + ( + released.silly, + released.companions, + released.names_detached, + released.names_refused + ), + (0, 1, 2, 1) + ); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn tickets_are_released_only_after_the_reply() { + let base = temp("tickets"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let watch = Arc::new(spy::Watch::default()); + let services = Arc::new(spy::Spy { + inner: Arc::new(Mounted(rig.bound.clone())), + watch: watch.clone(), + }); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let server = rig.export_services(listener, services, UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + c.wire = Some(watch.wire.clone()); + let root = c.mnt("/"); + + let made = c.touch(&root, b"f", 0o644).unwrap(); + // 300 000 bytes are three engine windows: three publications, three + // tickets, all released after the one reply. + c.write(&made.handle, 0, &pattern(2, 300_000)).unwrap(); + let truncate = Set { + size: Some(10), + ..Set::default() + }; + c.setattr(&made.handle, truncate, None).unwrap(); + c.mkdir(&root, b"dir", 0o755).unwrap(); + c.rename(&root, b"f", &root, b"g").unwrap(); + c.remove(&root, b"g").unwrap(); + let published = watch.published.load(Ordering::Acquire); + assert_eq!(published, 1 + 3 + 1 + 1 + 1 + 1); + // The last call's tickets are released just after its reply arrived. + let settled = std::time::Instant::now() + Duration::from_secs(5); + while watch.released_after.load(Ordering::Acquire) < published + && std::time::Instant::now() < settled + { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(watch.released_after.load(Ordering::Acquire), published); + assert_eq!( + watch.released_before.load(Ordering::Acquire), + 0, + "{:?}", + watch.log.lock().unwrap() + ); + + // A contended allocator: nothing is reserved and nothing created; the + // client is told to retry later, and the retry succeeds. + watch.refuse_serials.store(1, Ordering::Release); + assert_eq!(c.touch(&root, b"later", 0o644).map(|_| ()), Err(JUKEBOX)); + assert_eq!(watch.refused.load(Ordering::Acquire), 1); + assert_eq!(c.lookup(&root, b"later").map(|_| ()), Err(NOENT)); + assert_eq!(watch.published.load(Ordering::Acquire), published); + assert!(c.touch(&root, b"later", 0o644).is_ok()); + + // No ticket is left pending: the Workspace captures and commits. + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + assert_eq!(server.held().retained, 0); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +/// The same operations, over NFS and on a host directory. +struct Twin<'a> { + client: &'a mut Client, + root: Vec, + host: PathBuf, +} +impl Twin<'_> { + fn handle(&mut self, path: &str) -> Vec { + self.client.walk_to(&self.root, path).unwrap().0 + } + fn split(path: &str) -> (&str, &str) { + path.rsplit_once('/').unwrap_or(("", path)) + } + fn mkdir(&mut self, path: &str, mode: u32) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client.mkdir(&parent, name.as_bytes(), mode).unwrap(); + fs::create_dir(self.host.join(path)).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn write(&mut self, path: &str, offset: u64, data: &[u8]) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let handle = match self.client.lookup(&parent, name.as_bytes()) { + Ok((handle, _)) => handle, + Err(_) => { + let made = self.client.touch(&parent, name.as_bytes(), 0o644).unwrap(); + fs::write(self.host.join(path), b"").unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(0o644)) + .unwrap(); + made.handle + } + }; + self.client.write_all(&handle, offset, data, 64 * 1024); + use std::os::unix::fs::FileExt; + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.write_all_at(data, offset).unwrap(); + } + fn truncate(&mut self, path: &str, size: u64) { + let handle = self.handle(path); + let set = Set { + size: Some(size), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.set_len(size).unwrap(); + } + fn chmod(&mut self, path: &str, mode: u32) { + let handle = self.handle(path); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn rename(&mut self, from: &str, to: &str) { + let (a, an) = Self::split(from); + let (b, bn) = Self::split(to); + let (a, b) = (self.handle(a), self.handle(b)); + self.client + .rename(&a, an.as_bytes(), &b, bn.as_bytes()) + .unwrap(); + fs::rename(self.host.join(from), self.host.join(to)).unwrap(); + } + fn link(&mut self, existing: &str, new: &str) { + let file = self.handle(existing); + let (parent, name) = Self::split(new); + let parent = self.handle(parent); + self.client.link(&file, &parent, name.as_bytes()).unwrap(); + fs::hard_link(self.host.join(existing), self.host.join(new)).unwrap(); + } + fn symlink(&mut self, path: &str, target: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .symlink(&parent, name.as_bytes(), target.as_bytes()) + .unwrap(); + std::os::unix::fs::symlink(target, self.host.join(path)).unwrap(); + } + fn remove(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let directory = fs::symlink_metadata(self.host.join(path)).unwrap().is_dir(); + self.client + .unlink(&parent, name.as_bytes(), directory) + .unwrap(); + if directory { + fs::remove_dir(self.host.join(path)).unwrap(); + } else { + fs::remove_file(self.host.join(path)).unwrap(); + } + } + /// A file the client still has open when it is unlinked: renamed to a + /// silly name over NFS, simply gone on the host. + fn unlink_open(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .rename(&parent, name.as_bytes(), &parent, LINUX_SILLY) + .unwrap(); + fs::remove_file(self.host.join(path)).unwrap(); + } +} + +#[test] +fn a_commit_holds_what_the_same_operations_make_of_a_host_tree() { + let base = temp("commit"); + let tree = base.join("tree"); + source(&tree); + let host = base.join("host"); + fs::create_dir_all(&host).unwrap(); + assert!(std::process::Command::new("cp") + .arg("-a") + .arg(format!("{}/.", tree.display())) + .arg(&host) + .status() + .unwrap() + .success()); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let mut t = Twin { + client: &mut client, + root, + host: host.clone(), + }; + t.mkdir("a", 0o755); + t.mkdir("a/b", 0o700); + t.mkdir("a/b/c", 0o755); + t.write("a/b/c/big.bin", 0, &pattern(3, 400_000)); + t.write("a/b/c/big.bin", 123_456, &pattern(4, 70_000)); + t.truncate("a/b/c/big.bin", 350_001); + t.write("a/empty", 0, b""); + t.write("a/hello.txt", 0, b"hello\n"); + t.chmod("a/hello.txt", 0o755); + t.write("README.md", 0, b"# rewritten\n"); + t.truncate("src/nested/deep.txt", 0); + t.rename("a/hello.txt", "a/b/hello.txt"); + t.rename("a/b/c", "c-moved"); + t.link("a/b/hello.txt", "hello-alias"); + t.symlink("a/to-hello", "b/hello.txt"); + t.remove("src/alias.rs"); + t.remove("docs/guide.md"); + t.remove("docs"); + t.write("tmp-old", 0, b"old\n"); + t.write("tmp-new", 0, b"new\n"); + t.rename("tmp-new", "tmp-old"); + t.write("unicod\u{e9}.txt", 0, "caf\u{e9}\n".as_bytes()); + t.write("open-at-commit", 0, b"open\n"); + t.unlink_open("open-at-commit"); + drop(t); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + + // An independent Workspace bound to the committed Branch, read over NFS, + // equals the host tree path for path and byte for byte. + let expected = tree::oracle(&host); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let (paths, _, identities) = tree::walk(&mut v, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + let hello = fs::metadata(host.join("hello-alias")).unwrap(); + use std::os::unix::fs::MetadataExt; + assert_eq!(identities[&hello.ino()].len(), 1, "both names, one inode"); + drop(v); + viewer.stop(); + rig.close_view(view); + assert_eq!(OK, 0); + drop(client); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-node/Cargo.toml b/engine/crates/layerfs-node/Cargo.toml index 44fca52204..ef0420fa5c 100644 --- a/engine/crates/layerfs-node/Cargo.toml +++ b/engine/crates/layerfs-node/Cargo.toml @@ -22,6 +22,7 @@ layerfs-sandbox = { path = "../layerfs-sandbox" } layerfs-storage = { path = "../../../core/crates/layerfs-storage" } layerfs-store-service = { path = "../layerfs-store-service" } layerfs-telemetry = { path = "../../../core/crates/layerfs-telemetry" } +libc = "=0.2.189" napi = "=2.16.17" napi-derive = "=2.16.13" diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..95083db61d 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -1,4 +1,7 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +#[cfg(target_os = "linux")] +use crate::runtime::NAMESPACES; +use crate::runtime::{LOCAL, NATIVE}; use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, @@ -9,10 +12,11 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; -use layerfs_sandbox::{ - backend::docker::{Docker, Sandbox, SandboxRequest}, - CommandIdentity, ContainerId, -}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider, SandboxRequest}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::backend::native::{self, LocalProcess}; +use layerfs_sandbox::provider::{LocalRecord, SandboxRecord}; +use layerfs_sandbox::{CommandIdentity, ContainerId, Sandbox, WireFailure}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, @@ -26,6 +30,7 @@ use std::{ }; extern "C" { fn geteuid() -> u32; + fn getegid() -> u32; fn flock(fd: i32, operation: i32) -> i32; } const LOCK_EX: i32 = 2; @@ -45,7 +50,7 @@ pub(crate) fn home(args: &SdkValue) -> Result { directory(&home)?; Ok(home) } -fn private(path: &Path, dir: bool) -> Result<()> { +pub(crate) fn private(path: &Path, dir: bool) -> Result<()> { let meta = fs::symlink_metadata(path).map_err(failed)?; if meta.file_type().is_symlink() || meta.is_dir() != dir @@ -68,6 +73,11 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } +/// This process's effective uid and gid: a native Sandbox's command +/// identity (its commands run as this user). +pub(crate) fn user() -> (u32, u32) { + unsafe { (geteuid(), getegid()) } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -127,6 +137,8 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. +// Unmount exchanges run outside this guard; one shared readiness deadline +// covers the in-process turn and the cross-process discovery lock. const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); @@ -328,6 +340,20 @@ pub(crate) fn project(args: &SdkValue) -> Result { installed, }) } +/// The labels a record keeps, as the value map its reader expects. +fn label_values(labels: &[(String, String)]) -> SdkValue { + SdkValue::List( + (labels.iter()) + .map(|(key, value)| { + SdkValue::map([ + ("key", SdkValue::text(key)), + ("value", SdkValue::text(value)), + ]) + }) + .collect(), + ) +} + pub(crate) fn save_sandbox( directory: &Path, state: &SandboxState, @@ -339,41 +365,97 @@ pub(crate) fn save_sandbox( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "deploy"))?; + let mut fields = vec![ + ("id", SdkValue::text(&state.id)), + ("project", SdkValue::text(&state.project)), + ("name", SdkValue::text(&state.name)), + ( + "container", + SdkValue::text(state.runtime.identity().to_string()), + ), + ]; + match state.runtime.as_ref() { + Sandbox::Docker(runtime) => { + let record = runtime.request(); + fields.extend([ + ("docker", SdkValue::text(registry::text(deploy, "docker")?)), + ("image", SdkValue::text(&record.image)), + ("volume", SdkValue::text(&record.store_volume)), + ("labels", label_values(&record.labels)), + ]); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Sandbox::Native(runtime) => { + let record = runtime + .local_record() + .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + fields.extend(local_fields(args, state, &record, NATIVE, LOCAL)?); + } + #[cfg(target_os = "linux")] + Sandbox::Namespaces(runtime) => { + let Some(SandboxRecord::Namespaces(record)) = runtime.record() else { + return Err(refused("Failed", "namespace Sandbox has no init process")); + }; + let image = record.image().as_str().to_owned(); + fields.extend(local_fields( + args, + state, + record.local(), + NAMESPACES, + &image, + )?); + } + } + fields.extend([ + ("secret", SdkValue::Bytes(secret.to_vec())), + ("peer", SdkValue::Bytes(peer.to_vec())), + ("instance", SdkValue::Bytes(instance.to_vec())), + ]); save( &directory.join("sandbox.record"), - SdkValue::map([ - ("id", SdkValue::text(&state.id)), - ("project", SdkValue::text(&state.project)), - ("name", SdkValue::text(&state.name)), - ( - "container", - SdkValue::text(state.runtime.identity().to_string()), - ), - ("docker", SdkValue::text(registry::text(deploy, "docker")?)), - ("image", SdkValue::text(&state.runtime.request().image)), - ( - "volume", - SdkValue::text(&state.runtime.request().store_volume), - ), - ( - "labels", - SdkValue::List( - (state.runtime.request().labels.iter()) - .map(|(key, value)| { - SdkValue::map([ - ("key", SdkValue::text(key)), - ("value", SdkValue::text(value)), - ]) - }) - .collect(), - ), - ), - ("secret", SdkValue::Bytes(secret.to_vec())), - ("peer", SdkValue::Bytes(peer.to_vec())), - ("instance", SdkValue::Bytes(instance.to_vec())), - ]), + SdkValue::Map( + (fields.into_iter()) + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ), ) } +/// The fields a local Sandbox record keeps: its directory, its daemon or init +/// process identity, its listener port and the image it runs, under `runtime` +/// `native` (whose image is `local`) or `namespaces` (whose image is the +/// digest its root is an overlay of). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_fields( + args: &SdkValue, + state: &SandboxState, + record: &LocalRecord, + runtime: &'static str, + image: &str, +) -> Result> { + let boot = record + .boot + .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; + Ok(vec![ + ("runtime", SdkValue::text(runtime)), + ("uid", SdkValue::Int(i64::from(state.identity.uid))), + ("gid", SdkValue::Int(i64::from(state.identity.gid))), + ( + "directory", + SdkValue::text( + crate::local::directory(args, &state.id)? + .display() + .to_string(), + ), + ), + ("pid", SdkValue::Int(i64::from(record.pid))), + ("start", SdkValue::text(record.start_time.to_string())), + ("boot", SdkValue::text(format!("{boot:032x}"))), + ("port", SdkValue::Int(i64::from(record.port))), + ("image", SdkValue::text(image)), + ("labels", label_values(&record.labels)), + ]) +} + /// A Sandbox whose container no longer runs it: the Engine no longer has the /// container, or has it stopped (it closed when idle, or Docker restarted), /// which lost its daemon and every mount in it. Only its records, and a @@ -406,30 +488,260 @@ pub(crate) fn gone_container(refusal: &SdkRefusal) -> Option<(&str, bool)> { _ => None, } } -/// The Engine, container and deployment a Sandbox record names. -pub(crate) fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { +/// The labels a Sandbox record keeps; a record written before #373 has none. +fn labels(value: &SdkValue) -> Result> { + match value.get("labels") { + Some(SdkValue::List(items)) => items + .iter() + .map(|item| { + Ok(( + registry::text(item, "key")?.to_owned(), + registry::text(item, "value")?.to_owned(), + )) + }) + .collect(), + _ => Ok(Vec::new()), + } +} +/// The Engine, container and deployment a Docker Sandbox record names. +fn runtime(value: &SdkValue) -> Result<(DockerProvider, ContainerId, SandboxRecord)> { let docker = Docker::new(registry::text(value, "docker")?, Duration::from_secs(30)).map_err(failed)?; let request = SandboxRequest { image: registry::text(value, "image")?.to_owned(), store_volume: registry::text(value, "volume")?.to_owned(), port: 30421, - // a record written before #373 has no labels, nor has its container - labels: match value.get("labels") { - Some(SdkValue::List(items)) => items - .iter() - .map(|item| { - Ok(( - registry::text(item, "key")?.to_owned(), - registry::text(item, "value")?.to_owned(), - )) - }) - .collect::>()?, - _ => Vec::new(), - }, + labels: labels(value)?, }; let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; - Ok((docker, container, request)) + let provider = DockerProvider::new(docker); + Ok((provider, container, SandboxRecord::Docker(request.into()))) +} +/// What a Sandbox record names: a Docker container, a local daemon process +/// (`runtime` `native`) with its directory and port, or a namespace Sandbox +/// (`runtime` `namespaces`) with its own runtime, record and directory. +pub(crate) enum Recorded { + Docker(DockerProvider, ContainerId, SandboxRecord), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Local { + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + }, + #[cfg(target_os = "linux")] + Namespaces(layerfs_sandbox::Runtime, ContainerId, SandboxRecord), +} +/// Reads Sandbox `id`'s record's runtime. +pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + #[cfg(target_os = "linux")] + if value.get_text("runtime") == Some(NAMESPACES) { + return namespaces_record(args, id, value); + } + if matches!(value.get_text("runtime"), Some(LOCAL | NATIVE)) { + return local_record(args, id, value); + } + let (docker, container, request) = runtime(value)?; + Ok(Recorded::Docker(docker, container, request)) +} +/// A local record is used only where this deployment puts that Sandbox: +/// its directory must be exactly `/-` +/// and named so, so a damaged, edited or stale record (another +/// LAYERFS_LOCAL_DIR or prefix) never points a delete elsewhere. A record +/// without `boot` was written before boot ids were kept: its daemon is +/// matched by id and start time as then, and never signalled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, record) = local_record_of(args, id, value)?; + // with no placement (#428), a mount names its Sandbox: a record is used + // only by a native deployment, and only a native record is. A record of + // the `local` runtime (an earlier, unmerged build) is never used, so + // nothing it names is deleted + let native = args.get("deploy").is_some_and(crate::runtime::native); + if !native || value.get_text("runtime") != Some(NATIVE) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a {} Sandbox, but only a native deployment \ + (LAYERFS_SANDBOX=native) uses a native record and none uses another; \ + Sandboxes of other runtimes are never mixed, so the record is neither used \ + nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + value.get_text("runtime").unwrap_or_default(), + registry::text(args, "project")? + ), + )); + } + Ok(Recorded::Local { + directory, + id: ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + process: LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + }, + port: record.port, + labels: record.labels, + }) +} +/// A record's directory and physical facts, checked against where this +/// deployment puts that Sandbox. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record_of(args: &SdkValue, id: &str, value: &SdkValue) -> Result<(PathBuf, LocalRecord)> { + let directory = PathBuf::from(registry::text(value, "directory")?); + let expected = crate::local::directory(args, id)?; + // the Sandbox's own directory name, exactly as this deployment names it + let name = registry::text(value, "name")?; + if directory != expected || expected.file_name().and_then(|n| n.to_str()) != Some(name) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id}'s local record names {} (as {name}), but this deployment puts it at \ + {}: the record is damaged or from another LAYERFS_LOCAL_DIR or prefix, so it is \ + neither used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + directory.display(), + expected.display(), + registry::text(args, "project")? + ), + )); + } + let number = |key: &str| { + value + .get_int(key) + .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) + }; + let boot = match value.get_text("boot") { + None => None, + Some(hex) => Some( + (hex.len() == 32) + .then(|| u128::from_str_radix(hex, 16).ok()) + .flatten() + .ok_or_else(|| refused("InvalidInput", "local Sandbox record boot"))?, + ), + }; + let record = LocalRecord { + name: name.to_owned(), + pid: u32::try_from(number("pid")?).map_err(failed)?, + start_time: registry::text(value, "start")? + .parse::() + .map_err(failed)?, + boot, + port: u16::try_from(number("port")?).map_err(failed)?, + labels: labels(value)?, + }; + Ok((directory, record)) +} +/// A namespaces record is used only where this backend puts that Sandbox and +/// only by a deployment that selects it: another backend never reopens or +/// deletes what this one made. +#[cfg(target_os = "linux")] +fn namespaces_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, local) = local_record_of(args, id, value)?; + let image = + layerfs_sandbox::Image::parse(registry::text(value, "image")?).map_err(|error| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox whose record names no image digest \ + ({error}); it cannot be given a root, so it is neither reopened nor deleted" + ), + ) + })?; + let record = layerfs_sandbox::provider::NamespaceRecord { local, image }; + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + if crate::runtime::backend(deploy)? != crate::runtime::Backend::Namespaces { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox, but this deployment runs the other local \ + backend; Sandboxes of other runtimes are never mixed, so the record is neither \ + used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + registry::text(args, "project")? + ), + )); + } + let root = (directory.parent()) + .ok_or_else(|| refused("InvalidInput", "namespaces Sandbox directory"))?; + let provider = + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root).map_err(|error| { + refused( + "Unsupported", + format!( + "namespaces Sandbox at {} cannot run on this host: {error}", + directory.display() + ), + ) + })?; + Ok(Recorded::Namespaces( + layerfs_sandbox::Runtime::Namespaces(provider), + ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + SandboxRecord::Namespaces(record), + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { + Err(refused( + "Unsupported", + format!( + "Sandbox {id} is a native Sandbox (LAYERFS_SANDBOX=native), which needs Linux or macOS" + ), + )) +} +impl Recorded { + /// The runtime, observed running once (it starts nothing). + pub(crate) fn reopen(self) -> std::result::Result, Box> { + match self { + Self::Docker(provider, id, record) => provider + .reopen(id, &record) + .map(|s| Box::new(Sandbox::Docker(s))), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => native::Sandbox::reopen(directory, id, process, port, labels) + .map(|s| Box::new(Sandbox::Native(s))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => runtime.reopen(id, &record).map(Box::new), + } + } + /// The runtime found stopped: only its one Delete is admitted. + pub(crate) fn stopped(self) -> Box { + match self { + Self::Docker(provider, id, record) => { + let instance = provider + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"); + Box::new(Sandbox::Docker(instance)) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => Box::new(Sandbox::Native(native::Sandbox::stopped( + directory, id, process, port, labels, + ))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => Box::new( + runtime + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"), + ), + } + } } pub(crate) fn reopen(args: &SdkValue) -> Result> { let id = registry::text(args, "sandbox")?; @@ -447,8 +759,16 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let (docker, selector, request) = runtime(&value)?; - let runtime = Sandbox::reopen(docker, selector, request).map_err(|failure| { + let recorded = recorded(args, id, &value)?; + // a local daemon listens on this kernel's loopback; no other host reaches it + let host = match recorded { + Recorded::Docker(..) => crate::pool::daemon_host(args), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Recorded::Local { .. } => None, + #[cfg(target_os = "linux")] + Recorded::Namespaces(..) => None, + }; + let runtime = recorded.reopen().map_err(|failure| { if failure.container_gone() || failure.container_stopped() { gone(id, container, failure.container_stopped()) } else { @@ -457,7 +777,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { })?; let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, - host: crate::pool::daemon_host(args), + host, secret: bytes(&value, "secret")?, peer: bytes(&value, "peer")?, instance: bytes(&value, "instance")?, @@ -470,7 +790,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { name: value.get_text("name").unwrap_or_default().to_owned(), directory, runtime, - identity: CommandIdentity { uid: 501, gid: 20 }, + identity: recorded_identity(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -478,8 +798,23 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { refresh(&state)?; Ok(state) } +/// The command identity a Sandbox record names: a native Sandbox's is the +/// user who started it (`uid`, `gid`); every other runs as 501:20. +fn recorded_identity(value: &SdkValue) -> CommandIdentity { + let id = |key: &str| value.get_int(key).and_then(|n| u32::try_from(n).ok()); + match (id("uid"), id("gid")) { + (Some(uid), Some(gid)) => CommandIdentity { uid, gid }, + _ => CommandIdentity { uid: 501, gid: 20 }, + } +} /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { + refresh_except(state, &[]) +} +/// Explicit Sandbox close may dispose retained unmounts without asking their +/// daemon for a Ready receipt that no longer exists. Other records still +/// undergo the ordinary refresh and MountsActive admission. +pub(crate) fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); let mut processes = std::collections::BTreeMap::new(); for item in fs::read_dir(&state.directory).map_err(failed)? { @@ -490,6 +825,9 @@ pub(crate) fn refresh(state: &SandboxState) -> Result<()> { continue; }; let path = registry::text(&value, "path")?; + if retained.iter().any(|kept| kept == path) { + continue; + } let token = layerfs_bridge::control::WorkspaceToken { workspace: layerfs_history::WorkspaceId::from_slice(&bytes::<32>( &value, diff --git a/engine/crates/layerfs-node/src/dispatch.rs b/engine/crates/layerfs-node/src/dispatch.rs index 99c80b12b6..365004dc7f 100644 --- a/engine/crates/layerfs-node/src/dispatch.rs +++ b/engine/crates/layerfs-node/src/dispatch.rs @@ -75,6 +75,9 @@ fn listed(answer: SdkValue) -> Outcome { /// the daemon, so it is not sent on. fn daemon_backend(op: &str, args: SdkValue) -> Outcome { let state = registry::sandbox_live(&args)?; + if let Some(path) = args.get_text("path") { + crate::unmount::check(&state, path)?; + } let args = match args { SdkValue::Map(fields) => SdkValue::Map( fields diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index 80a3120b2a..3982d74648 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -5,46 +5,38 @@ //! daemon is not involved. A background command is started in its own session //! with its output in a log file, and this binding keeps its process id. The //! Engine offers no per-exec cancel, so `cancel` signals the process group -//! from a second command and waits (bounded) until it is gone. -use crate::job::{failed, refused, Outcome}; +//! from a second command and waits (bounded) until it is gone. A local +//! Sandbox (`layerfs_sandbox::Sandbox::Native`) runs the same commands as +//! processes of this kernel, so the same probes and signals apply. +use crate::job::{refused, Outcome}; use crate::output::Spool; use crate::registry::{lock, naming, sandbox, sandbox_live, text, Process, SandboxState}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::time::Duration; -/// Home of the command identity, which has no account in the image. -const HOME: &str = "HOME=/tmp"; /// Checks that a cancelled process group is gone. const GONE_CHECKS: u32 = 50; /// Pause between those checks. const GONE_PAUSE: Duration = Duration::from_millis(100); /// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. +/// non-empty stream to its file instead of keeping it inline. Its home is +/// the Sandbox's scratch directory (`/tmp` in a container), since the +/// command identity has no account there. fn run( state: &SandboxState, directory: &str, arguments: Vec, spool: bool, ) -> Result<(i32, Spool, Spool), SdkRefusal> { - let created = layerfs_api::internal::exec( + crate::runtime::run( &state.runtime, state.identity, arguments, - vec![HOME.to_owned()], - directory.to_owned(), - false, + vec![format!("HOME={}", crate::runtime::scratch(&state.runtime))], + directory, + spool, ) - .map_err(failed)?; - let attached = created.start().map_err(failed)?; - let (_input, output, handle) = attached.into_parts().map_err(failed)?; - let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); - output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; - let inspected = handle.inspect().map_err(failed)?; - let code = inspected - .known_root_exit() - .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))?; - Ok((code, stdout, stderr)) } /// `workspace.exec(ws, command, { background?, spool? })`; with @@ -91,11 +83,17 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { state.id, layerfs_bridge::sdk_values::hex(&crate::registry::random()?[..12]) ); - let script = format!("setsid bash -c \"$0\" >/tmp/layerfs-{id}.log 2>&1 \"$1\" 2>&1 Outcome { "background process exited before its identity was observed", ) })?; + // its own session leaves the command's process group; named, it still + // ends with a local Sandbox. Unnamed it would outlive it: it ends here, once. + if let Err(refusal) = crate::runtime::track(&state.runtime, pid, &start) { + let end = format!("kill -KILL -- -{pid} 2>/dev/null; true"); + let _ = run( + &state, + &directory, + vec!["bash".into(), "-c".into(), end], + false, + ); + return Err(refusal); + } let process = Process { path: path.to_owned(), command: command.to_owned(), @@ -136,7 +146,7 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { /// The container's first process is the daemon, which does not reap orphans, /// so a finished member can stay a zombie that `kill -0` still finds. fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { - let (code, out, _) = run(state, "/tmp", vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; + let (code, out, _) = run(state, &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; if code == 0 { Ok(Some(String::from_utf8_lossy(out.kept()).into_owned())) } else { @@ -151,7 +161,7 @@ fn alive(state: &SandboxState, pid: u32) -> Result { ); let (code, _, _) = run( state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), probe], false, )?; @@ -283,7 +293,7 @@ pub(crate) fn cancel(args: &SdkValue) -> Outcome { } run( &state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec![ "bash".into(), "-c".into(), diff --git a/engine/crates/layerfs-node/src/lib.rs b/engine/crates/layerfs-node/src/lib.rs index 298d5020af..f7f89ae788 100644 --- a/engine/crates/layerfs-node/src/lib.rs +++ b/engine/crates/layerfs-node/src/lib.rs @@ -5,12 +5,16 @@ mod discovery; mod dispatch; mod exec; mod job; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local; mod output; mod pool; mod project; mod registry; +mod runtime; mod sandbox; mod trace; +mod unmount; mod workspace; use job::Job; diff --git a/engine/crates/layerfs-node/src/local.rs b/engine/crates/layerfs-node/src/local.rs new file mode 100644 index 0000000000..1556ed7efa --- /dev/null +++ b/engine/crates/layerfs-node/src/local.rs @@ -0,0 +1,440 @@ +//! Native Sandboxes (`deploy.sandbox` `native`, LAYERFS_SANDBOX=native): +//! the project's daemon runs as a process of this user instead of in a +//! container. It isolates nothing beyond the caller's own sandbox. +//! +//! Each Sandbox gets a directory `/-` +//! with what a container held at fixed paths: its setup +//! (`/layerfs-local/config`), overlay template (`/layerfs-local/overlay`) and +//! Workspace mounts (`/workspaces`). The daemon opens the installed Store +//! where this process opens it (`storePath`), so the daemon setup and the +//! Store's locator name that path instead of `/layerfs-store`. The daemon +//! listens on 127.0.0.1 only. macOS serves loopback NFS as this user; Linux +//! keeps the root FUSE daemon and nonroot commands (501:20) from #402. Start, +//! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's +//! daemon stopped and its directory deleted, each once. +use crate::job::{failed, refused}; +use crate::registry::{lock, text, SandboxState, SANDBOXES}; +use layerfs_api::{sandbox::ProviderCreate, SandboxApi}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::{CommandIdentity, Runtime, Sandbox}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +/// Starts the project's next Sandbox as a native daemon. Linux requires +/// root before any effect. Called by +/// [`crate::sandbox::start`] under the project's guard, with the project +/// and its installed manifest already read. +pub(crate) fn start( + args: &SdkValue, + directory: &Path, + project: (&crate::registry::Project, StoreManifest), +) -> Result, SdkRefusal> { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + // Inputs are read first, so a refused request leaves no creation intent. + if cfg!(target_os = "linux") && crate::discovery::user().0 != 0 { + return Err(refused( + "Unsupported", + "LAYERFS_SANDBOX=native on Linux needs root for FUSE and commands as uid 501, gid 20; run inside the VM or CI box that isolates it", + )); + } + let root = deploy.get_text("localDir").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a native Sandbox needs it", + ) + })?; + let backend = crate::runtime::backend(deploy)?; + let runtime = native_runtime(root, backend) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let executable = PathBuf::from(text(deploy, "daemon")?); + let store = match args.get_text("storePath") { + Some(path) if Path::new(path).is_absolute() => path.to_owned(), + _ => { + return Err(refused( + "InvalidInput", + "a local Sandbox's daemon opens the client's storePath (LAYERFS_STORE_PATH), an absolute path", + )) + } + }; + crate::sandbox::unresolved( + directory, + &format!("a directory under {root} named for it, and its daemon process"), + )?; + let (id, saved_directory) = crate::discovery::claim_next(args)?; + let made = start_claimed( + args, + directory, + &saved_directory, + &id, + project, + (runtime, backend, executable, store), + ); + if made.is_err() { + let _ = std::fs::remove_dir(&saved_directory); + } + made +} + +/// Starts the claimed Sandbox `id`: directory and daemon on the installed +/// Store, its first control connection, then its records. +fn start_claimed( + args: &SdkValue, + directory: &Path, + saved_directory: &Path, + id: &str, + (project, installed): (&crate::registry::Project, StoreManifest), + (runtime, backend, executable, store): (Runtime, crate::runtime::Backend, PathBuf, String), +) -> Result, SdkRefusal> { + let name = text(args, "project")?; + let sandbox_name = sandbox_name(args, id)?; + let paths = runtime.native_paths(&sandbox_name).map_err(failed)?; + let path_text = |path: &Path| { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_LOCAL_DIR must be UTF-8")) + }; + let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let mut setup = crate::sandbox::setup(daemon.private, controller.public); + // macOS uses the caller; Linux keeps #402's separate nonroot identity. + #[cfg(target_os = "macos")] + { + (setup.command_uid, setup.command_gid) = crate::discovery::user(); + } + setup.listen = "127.0.0.1:0".into(); + setup.store = store.clone(); + setup.overlay = path_text(&paths.overlay)?; + setup.mounts = path_text(&paths.mounts)?; + let mut existing = project.manifest.clone(); + existing.daemon_sqlite = installed.daemon_sqlite; + // the Store's path in the daemon's filesystem, which is this one + existing.locator = store; + setup.existing_store = Some(existing); + let identity = CommandIdentity { + uid: setup.command_uid, + gid: setup.command_gid, + }; + setup.encode().map_err(failed)?; + // Read only if this process ends before the intent is resolved. + crate::discovery::save( + &directory.join("create.intent"), + SdkValue::text(format!("creation of Sandbox {id} began and never finished")), + )?; + let labels = crate::sandbox::labels(name, id); + let api = SandboxApi::with_runtime(Arc::new(runtime)); + let options = launch_options(backend, identity, &setup.store, args)?; + let mut manifest = project.manifest.clone(); + manifest.locator = setup.store.clone(); + let created = api + .create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: sandbox_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), + labels, + startup_timeout: crate::sandbox::STARTUP, + options, + }, + setup, + controller_private: controller.private, + daemon_peer: daemon.public, + }, + &project.store, + &manifest, + ) + .map_err(|mut failure| { + drop(failure.control.take()); + abandon( + directory, + id, + format!( + "native sandbox stopped at {:?}: {:?}", + failure.phase, failure.cause + ), + failure.runtime.take(), + ) + })?; + let instance = created.startup.instance; + let (sandbox, control, identity) = layerfs_api::internal::split_provider(created); + let endpoint = match sandbox.endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + drop(control); + return Err(abandon( + directory, + id, + format!("native endpoint after startup: {error:?}"), + Some(sandbox), + )); + } + }; + let dial = crate::pool::Dial { + endpoint, + host: None, + secret: controller.private, + peer: daemon.public, + instance, + }; + let state = SandboxState { + id: id.to_owned(), + project: name.to_owned(), + name: sandbox_name, + directory: saved_directory.to_owned(), + runtime: sandbox, + identity, + controls: crate::pool::Controls::new(control, dial), + mounts: Default::default(), + processes: Default::default(), + }; + let recorded = (|| { + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed) + })(); + if let Err(refusal) = recorded { + // The daemon runs and is this start's own: it is stopped and its + // directory deleted like any failed start's, and the records written + // so far are removed, so its number is free again. + let SandboxState { runtime, .. } = state; + for record in ["sandbox.record", "container"] { + let _ = std::fs::remove_file(saved_directory.join(record)); + } + return Err(abandon( + directory, + id, + format!("local sandbox records: {}", refusal.message), + Some(runtime), + )); + } + let state = Arc::new(state); + lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); + Ok(state) +} + +/// The native runtime this host offers, over `root`: macOS loopback NFS as +/// this user, or Linux FUSE with a root daemon. The other platform's +/// provider, and another OS, is refused before root access. +#[cfg(target_os = "macos")] +fn native_runtime( + root: &str, + _: crate::runtime::Backend, +) -> Result { + layerfs_sandbox::backend::native::NativeMacosProvider::new(root).map(Runtime::MacosNative) +} + +#[cfg(target_os = "linux")] +fn native_runtime( + root: &str, + backend: crate::runtime::Backend, +) -> Result { + match backend { + crate::runtime::Backend::Native => { + layerfs_sandbox::backend::native::NativeLinuxProvider::new(root) + .map(Runtime::LinuxNative) + } + crate::runtime::Backend::Namespaces => { + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root) + .map(Runtime::Namespaces) + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn native_runtime( + _: &str, + _: crate::runtime::Backend, +) -> Result { + Err(layerfs_sandbox::RuntimeError::Unsupported( + "native Sandbox runtime platform", + )) +} + +/// How the daemon of `backend` is launched: the same local options, and for +/// the namespace backend the Store file its private root binds and the image +/// archive its root is unpacked from. +#[cfg(target_os = "linux")] +fn launch_options( + backend: crate::runtime::Backend, + identity: CommandIdentity, + store: &str, + args: &SdkValue, +) -> Result { + Ok(match backend { + crate::runtime::Backend::Native => ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }, + crate::runtime::Backend::Namespaces => { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + let (image, archive) = crate::runtime::namespaces_image(deploy)?; + ProviderLaunchOptions::Namespaces { + command_identity: identity, + environment: native_environment(), + store: PathBuf::from(store), + image, + archive, + } + } + }) +} + +#[cfg(not(target_os = "linux"))] +fn launch_options( + _: crate::runtime::Backend, + identity: CommandIdentity, + _: &str, + _: &SdkValue, +) -> Result { + Ok(ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }) +} + +/// What a native Sandbox's daemon starts with besides `PATH`: the NFS +/// route, and the mount helper and reference overrides this process was +/// given (`LAYERFS_NFS_*`), so a test host can name its own client. +#[cfg(target_os = "macos")] +fn native_environment() -> Vec<(String, String)> { + let mut environment = vec![("LAYERFS_KERNEL".to_owned(), "nfs".to_owned())]; + for name in [ + "LAYERFS_NFS_MOUNT", + "LAYERFS_NFS_UNMOUNT", + "LAYERFS_NFS_FORCE", + "LAYERFS_NFS_FLUSH", + "LAYERFS_NFS_REFRESH", + ] { + if let Ok(value) = std::env::var(name) { + environment.push((name.to_owned(), value)); + } + } + environment +} + +#[cfg(target_os = "linux")] +fn native_environment() -> Vec<(String, String)> { + vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())] +} + +/// A local Sandbox's directory name: `-`, the name its +/// container would have. +fn sandbox_name(args: &SdkValue, id: &str) -> Result { + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + Ok(format!("{prefix}{}-{id}", text(args, "project")?)) +} + +/// Where this deployment puts Sandbox `id`: +/// `/-`, computed from the +/// configuration alone, as a record's directory must be. +pub(crate) fn directory(args: &SdkValue, id: &str) -> Result { + let root = (args.get("deploy")) + .and_then(|deploy| deploy.get_text("localDir")) + .map(Path::new) + .ok_or_else(|| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a local Sandbox; its record is used only with \ + LAYERFS_LOCAL_DIR (deploy.localDir) configured" + ), + ) + })?; + let name = sandbox_name(args, id)?; + let plain = |path: &Path| { + !path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + }; + let single = matches!( + Path::new(&name).components().collect::>()[..], + [Component::Normal(_)] + ); + if !root.is_absolute() || !plain(root) || !single { + return Err(refused( + "InvalidInput", + format!("local Sandbox directory {}/{name}", root.display()), + )); + } + Ok(root.join(name)) +} + +/// The refusal for a failed start. Everything a local start makes is its +/// own (a directory and a child process), so each effect is known: the +/// daemon's last log lines are read once, then it is stopped and its +/// directory deleted, each once, and the intent removed. If that cleanup +/// fails, the intent stays and says what is left. +fn abandon( + directory: &Path, + id: &str, + mut cause: String, + sandbox: Option>, +) -> SdkRefusal { + let container = sandbox.as_ref().map(|s| s.identity().to_string()); + let mut log = None; + let mut left = None; + if let Some(mut sandbox) = sandbox { + match sandbox.log_tail(crate::sandbox::LOG_LINES, crate::sandbox::LOG_BYTES) { + Ok(bytes) => { + let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); + cause = format!("{cause}; its daemon's last log lines:\n{text}"); + log = Some(text); + } + Err(error) => cause = format!("{cause}; its daemon log was unreadable: {error:?}"), + } + let removed = sandbox.stop(1).and_then(|()| sandbox.delete()); + left = removed.err().map(|error| { + format!( + "its local Sandbox {} could not be removed: {error:?}", + sandbox.identity() + ) + }); + } + let intent = directory.join("create.intent"); + let (message, removed) = match left { + None => match std::fs::remove_file(&intent) { + Ok(()) => (cause, container.is_some()), + Err(error) => ( + format!("{cause}; its creation intent stays: {error}"), + container.is_some(), + ), + }, + Some(left) => { + let state = format!("creation of Sandbox {id} failed ({cause}); {left}"); + let kept = crate::discovery::save(&intent, SdkValue::text(&state)) + .err() + .map(|error| format!(" (recording it failed: {})", error.message)) + .unwrap_or_default(); + (format!("{state}; its creation intent stays{kept}"), false) + } + }; + SdkRefusal { + code: "Failed".to_owned(), + message, + detail: SdkValue::map([ + ("container", SdkValue::opt_text(container)), + ("removed", SdkValue::Bool(removed)), + ("log", SdkValue::opt_text(log)), + ]), + } +} diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..8bbbc2081c 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -2,7 +2,8 @@ //! project itself is only its home entry (`//`), so a project //! removed there can be initialized again without a restart. //! -//! A Sandbox is one container and its daemon. Its runtime runs commands +//! A Sandbox is one container and its daemon (or, with the local runtime, +//! one daemon process: `runtime`). Its runtime runs commands //! directly; its control connections (`pool`) each carry one exchange at a //! time, and concurrent calls take separate connections. The Sandbox runtime owner (this //! binding) also keeps the background processes it started; the daemon holds @@ -11,7 +12,7 @@ use crate::job::refused; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; +use layerfs_sandbox::CommandIdentity; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -42,10 +43,11 @@ pub(crate) struct Process { pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, - /// Its container's name, `-`. + /// Its container's name, `-` (a local Sandbox's + /// directory name). pub(crate) name: String, pub(crate) directory: std::path::PathBuf, - pub(crate) runtime: Sandbox, + pub(crate) runtime: Box, pub(crate) identity: CommandIdentity, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, @@ -76,7 +78,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// tables as they stand: no daemon call (#352 F-029). A call that names a /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes -/// [`sandbox_refreshed`]. +/// `discovery::refresh`. pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -129,14 +131,6 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { SdkValue::Map(fields) } -/// The Sandbox an operation names, its tables read again from the shared -/// records and the daemon (one status call per mount). -pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { - let state = sandbox(args)?; - crate::discovery::refresh(&state)?; - Ok(state) -} - impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { @@ -154,6 +148,7 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { + crate::unmount::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs new file mode 100644 index 0000000000..a4757f26ab --- /dev/null +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -0,0 +1,156 @@ +//! A Sandbox's runtime: Docker (default), or `LAYERFS_SANDBOX=native` on +//! Linux (root FUSE daemon, nonroot commands) and macOS (loopback NFS, +//! caller identity). Both use the same authenticated daemon control pipeline +//! and the same concrete `layerfs_sandbox::Sandbox` handle. +use crate::job::{failed, refused}; +use crate::output::Spool; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::provider::SandboxRecord; +#[cfg(target_os = "linux")] +use layerfs_sandbox::Image; +use layerfs_sandbox::{CommandIdentity, Sandbox}; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The image a native Sandbox records and reports, and the `runtime` of a +/// record from the former `local` runtime, which is never used (a Docker +/// record has no `runtime`). +pub(crate) const LOCAL: &str = "local"; +/// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local +/// daemon serving FUSE on Linux or loopback NFS on macOS. +pub(crate) const NATIVE: &str = "native"; +/// A namespace Sandbox record's `runtime` (LAYERFS_SANDBOX_BACKEND=namespaces, +/// Linux): a local daemon in its own mount, PID and IPC namespaces and cgroup. +pub(crate) const NAMESPACES: &str = "namespaces"; + +/// Which local backend a native deployment runs inside its Sandbox directory: +/// the plain native one, or the Linux namespace one. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Backend { + Native, + Namespaces, +} + +/// The local backend the deployment selects: `deploy.sandboxBackend` +/// (LAYERFS_SANDBOX_BACKEND), `native` when unset. The namespace backend is +/// Linux only; anywhere else selecting it is refused before any effect. +pub(crate) fn backend(deploy: &SdkValue) -> Result { + match deploy.get_text("sandboxBackend") { + None | Some(NATIVE) => Ok(Backend::Native), + Some(NAMESPACES) if cfg!(target_os = "linux") => Ok(Backend::Namespaces), + Some(NAMESPACES) => Err(refused( + "Unsupported", + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) namespaces is Linux only; use native", + )), + Some(other) => Err(refused( + "InvalidInput", + format!( + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) is native or namespaces, not {other}" + ), + )), + } +} + +/// Whether the deployment selects a local daemon: `deploy.sandbox` unset or +/// `docker` is Docker, `native` is a local daemon; anything else (the +/// former `local` included) is refused before any effect. +pub(crate) fn local(deploy: &SdkValue) -> Result { + match deploy.get_text("sandbox") { + None | Some("docker") => Ok(false), + Some(NATIVE) => Ok(true), + Some(other) => Err(refused( + "InvalidInput", + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not {other}"), + )), + } +} + +/// Whether the deployment selects the native route: FUSE on Linux, +/// loopback NFS on macOS, with no Docker. +pub(crate) fn native(deploy: &SdkValue) -> bool { + deploy.get_text("sandbox") == Some(NATIVE) +} + +/// The image a Sandbox reports: its container's, the digest a namespaces +/// Sandbox's root is an overlay of, or `local` for a native one. +pub(crate) fn image(runtime: &Sandbox) -> String { + match runtime.record() { + Some(SandboxRecord::Docker(docker)) => docker.image, + #[cfg(target_os = "linux")] + Some(SandboxRecord::Namespaces(record)) => record.image().as_str().to_owned(), + _ => LOCAL.to_owned(), + } +} + +/// The image a namespaces deployment's Sandbox root is an overlay of: +/// `deploy.image` (LAYERFS_IMAGE), the same `sha256:<64 hex>` digest the +/// Docker backend uses, and `deploy.imageArchive` (LAYERFS_IMAGE_ARCHIVE), +/// the docker-archive in the bundle that holds it. Both are read before any +/// effect, so a bundle without its image refuses the start. +#[cfg(target_os = "linux")] +pub(crate) fn namespaces_image(deploy: &SdkValue) -> Result<(Image, PathBuf), SdkRefusal> { + let digest = deploy.get_text("image").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.image is not configured (LAYERFS_IMAGE); a namespaces Sandbox's root is the image that digest names", + ) + })?; + let image = Image::parse(digest) + .map_err(|error| refused("InvalidInput", format!("deploy.image: {error}")))?; + let archive = deploy.get_text("imageArchive").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.imageArchive is not configured (LAYERFS_IMAGE_ARCHIVE); a namespaces Sandbox needs the docker-archive the bundle carries", + ) + })?; + let archive = PathBuf::from(archive); + if !archive.is_absolute() { + return Err(refused( + "InvalidInput", + "deploy.imageArchive (LAYERFS_IMAGE_ARCHIVE) must be an absolute path", + )); + } + Ok((image, archive)) +} + +/// Where its commands keep scratch files: `/tmp` in a container, the +/// Sandbox's own `tmp` directory for a native one (deleted with it), so a +/// native Sandbox leaves nothing in this machine's `/tmp`. +pub(crate) fn scratch(runtime: &Sandbox) -> String { + runtime.scratch_directory() +} + +/// Names a background process a command started in a session of its own +/// (`pid`, started at `start`), so it ends with a native Sandbox as it ends +/// with a container. A container's teardown already owns its descendants. +pub(crate) fn track(runtime: &Sandbox, pid: u32, start: &str) -> Result<(), SdkRefusal> { + if matches!(runtime, Sandbox::Docker(_)) { + return Ok(()); + } + let start = start.parse::().map_err(failed)?; + runtime.track(pid, start).map_err(failed) +} + +/// One command to its end: exit code, stdout, stderr. `spool` sends every +/// non-empty stream to its file instead of keeping it inline. +pub(crate) fn run( + runtime: &Sandbox, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, + spool: bool, +) -> Result<(i32, Spool, Spool), SdkRefusal> { + let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); + let request = layerfs_sandbox::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + let exit = runtime + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)?; + Ok((exit.code, stdout, stderr)) +} diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index edf23889a2..45e164f3b3 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -14,15 +14,15 @@ //! the caller's behalf. use crate::job::{failed, refused, Outcome}; use crate::registry::{lock, text, SandboxState, SANDBOXES}; -use layerfs_api::internal::split; -use layerfs_api::sandbox::values; -use layerfs_api::{ - OperationFailure, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase, -}; +use layerfs_api::internal::split_provider; +use layerfs_api::sandbox::{values, ProviderCreate, ProviderFailure}; +use layerfs_api::{OperationFailure, SandboxApi, SandboxCause, SandboxPhase}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::backend::docker::{CreatePhase, Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{CreatePhase, Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -30,12 +30,12 @@ use std::time::Duration; /// Port the daemon listens on inside its container. const PORT: u16 = 30421; /// Wait for the daemon's listener after Start. -const STARTUP: Duration = Duration::from_secs(30); +pub(crate) const STARTUP: Duration = Duration::from_secs(30); /// Wait for one Engine control response. const ENGINE: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. -const LOG_LINES: u16 = 20; -const LOG_BYTES: usize = 2048; +pub(crate) const LOG_LINES: u16 = 20; +pub(crate) const LOG_BYTES: usize = 2048; pub(crate) fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +pub(crate) fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -84,7 +84,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { } /// A container name prefix when the deployment sets none. -const PREFIX: &str = "layerfs-"; +pub(crate) const PREFIX: &str = "layerfs-"; /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. @@ -115,6 +115,15 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result state, - _ => "its record is unreadable".to_owned(), - }; - return Err(refused( - "Unknown", - format!( - "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ - is unknown, so it is neither retried nor taken as done. Remove any container it \ - left (docker ps -a --filter volume={store_volume}), then delete {}", - intent.display() - ), - )); - } + unresolved( + directory, + &format!("docker ps -a --filter volume={store_volume}"), + )?; // The number is claimed before anything is made; a failed start gives it // back (its directory holds no record yet), so the number is used again. let (id, saved_directory) = crate::discovery::claim_next(args)?; @@ -174,6 +170,29 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result<(), SdkRefusal> { + let intent = directory.join("create.intent"); + if intent.exists() { + let state = match crate::discovery::load(&intent) { + Ok(SdkValue::Text(state)) => state, + _ => "its record is unreadable".to_owned(), + }; + return Err(refused( + "Unknown", + format!( + "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ + is unknown, so it is neither retried nor taken as done. Remove any container it \ + left ({left}), then delete {}", + intent.display() + ), + )); + } + Ok(()) +} + /// Starts the claimed Sandbox `id`: container, daemon on the installed /// Store, then its records. fn start_claimed( @@ -204,25 +223,30 @@ fn start_claimed( .map_err(failed)? .naming(&container_name) .map_err(|_| refused("InvalidInput", format!("container name {container_name}")))?; + let provider = Arc::new(Runtime::Docker(DockerProvider::new(docker))); // Read only if this process ends before the intent is resolved. crate::discovery::save( &directory.join("create.intent"), SdkValue::text(format!("creation of Sandbox {id} began and never finished")), )?; - let created = SandboxApi::new(docker).create_file( - SandboxCreate { - deployment: SandboxRequest { - image, - store_volume, - port: PORT, + let created = SandboxApi::with_runtime(provider).create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: container_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), labels: labels(name, id), + startup_timeout: STARTUP, + options: ProviderLaunchOptions::Docker { + image, + store_volume, + port: PORT, + connect_host: crate::pool::daemon_host(args), + }, }, setup: daemon_setup, - executable, controller_private: controller.private, daemon_peer: daemon.public, - startup_wait: STARTUP, - connect_host: crate::pool::daemon_host(args), }, &project.store, &project.manifest, @@ -230,7 +254,7 @@ fn start_claimed( let owner = created.map_err(|failure| abandon(directory, id, failure))?; let container = owner.runtime.identity().to_string(); let instance = owner.startup.instance; - let (runtime, control, identity) = split(owner); + let (runtime, control, identity) = split_provider(owner); let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, host: crate::pool::daemon_host(args), @@ -267,33 +291,46 @@ fn start_claimed( /// Whether a failed create's every effect is known: nothing reached the /// Store volume, and any container it made is acknowledged by identity. -fn settled(failure: &SandboxFailure) -> bool { +fn settled(failure: &ProviderFailure) -> bool { // A Create the Engine refused (409: the container name is taken) made no container. - let unknown_container = matches!( - &failure.cause, - SandboxCause::Create(create) if create.phase == CreatePhase::Create - && !matches!(create.transfer.cause, layerfs_sandbox::RuntimeError::Http(409)) - ); + let unknown_container = match &failure.cause { + SandboxCause::Provider(launch) => match &launch.cause { + layerfs_sandbox::RuntimeError::Docker(docker_failure) => { + match docker_failure.as_ref() { + layerfs_sandbox::backend::docker::ProviderFailure::Create(create) => { + create.phase == CreatePhase::Create + && !matches!( + create.transfer.cause, + layerfs_sandbox::RuntimeError::Http(409) + ) + } + _ => false, + } + } + _ => false, + }, + _ => false, + }; failure.phase != SandboxPhase::Install && failure.installed.is_none() && !unknown_container } /// The refusal for a failed create. A settled one reads its container's log /// tail once, removes the container (stop, then delete, each once) and its /// intent; otherwise the intent stays, rewritten to say what is left. -fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { +fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { let mut cause = format!( "sandbox create stopped at {:?}: {:?}", failure.phase, failure.cause ); let container = failure - .sandbox + .runtime .as_ref() .map(|sandbox| sandbox.identity().to_string()); drop(failure.control.take()); let mut log = None; let left = if !settled(&failure) { Some("its effect on the Store volume or the Engine is unknown".to_owned()) - } else if let Some(sandbox) = failure.sandbox.as_mut() { + } else if let Some(sandbox) = failure.runtime.as_mut() { match sandbox.log_tail(LOG_LINES, LOG_BYTES) { Ok(bytes) => { let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); @@ -344,7 +381,8 @@ fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkR /// `sandbox.create({ image })`: under the project's guard, reclaims the /// project's idle-closed Sandboxes, then starts the lowest free number on /// the call's image (`deploy.image`) and waits for its daemon: -/// `{ sandbox, name, image }`. The only call that starts a Sandbox. Failures +/// `{ sandbox, name, image }`. With `deploy.sandbox` `native` (macOS) it +/// starts a daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures /// are `Failed` or `Unknown`, never retried. pub(crate) fn create(args: &SdkValue) -> Outcome { let project = crate::discovery::project_dir(args)?; @@ -356,7 +394,7 @@ pub(crate) fn create(args: &SdkValue) -> Outcome { ("name", SdkValue::text(&state.name)), ( "image", - SdkValue::text(state.runtime.request().image.as_str()), + SdkValue::text(crate::runtime::image(&state.runtime)), ), ])) } @@ -381,8 +419,16 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { } let state = crate::registry::sandbox_live(args)?; let _guard = crate::discovery::guard(&state.directory)?; - crate::discovery::refresh(&state)?; - let mounted = lock(&state.mounts)?.keys().cloned().collect::>(); + let recoverable = crate::unmount::check_all(&state)?; + crate::discovery::refresh_except(&state, &recoverable)?; + // Explicit close can take over an orphaned/retained unmount, but must + // still refuse every unrelated live Workspace. Stop detaches before + // killing the daemon; Delete independently checks physical detach. + let mounted = lock(&state.mounts)? + .keys() + .filter(|path| !recoverable.contains(path)) + .cloned() + .collect::>(); if !mounted.is_empty() { return Err(SdkRefusal { code: "MountsActive".to_owned(), @@ -397,8 +443,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { }); } let value = crate::discovery::load(&record)?; - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let mut owner = layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request) + let mut owner = crate::discovery::recorded(args, &id, &value)? + .reopen() .map_err(failed)?; owner.stop(STOP_S).map_err(failed)?; owner.delete().map_err(failed)?; @@ -413,7 +459,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { /// `backend.daemon.sandbox.get_log({ sandbox, lines })`: the last `lines` /// lines (1..=2000) of Sandbox `sandbox`'s daemon log, at most -/// [`GET_LOG_BYTES`], read once through the Engine: `{ sandbox, container, +/// [`GET_LOG_BYTES`], read once through the Engine (a local Sandbox's from +/// its log file): `{ sandbox, container, /// text, bytes }`. A stopped container's log is readable until its records /// are reclaimed. Refused `NoSuchSandbox` with no record or no container. /// Read-only: it starts, stops and reclaims nothing. @@ -437,8 +484,7 @@ pub(crate) fn get_log(args: &SdkValue) -> Outcome { crate::discovery::load(&record)? }; let container = text(&value, "container")?.to_owned(); - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let sandbox = layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request); + let sandbox = crate::discovery::recorded(args, &id, &value)?.stopped(); let bytes = match sandbox.log_tail(lines, GET_LOG_BYTES) { Ok(bytes) => bytes, Err(failure) if failure.container_gone() => { @@ -460,7 +506,7 @@ const GET_LOG_BYTES: usize = 64 * 1024; /// Seconds a closing Sandbox's daemon has to exit before the Engine kills it. const STOP_S: u16 = 5; -/// Reclaims Sandbox `id`, whose container a reopen found gone or stopped +/// Reclaims Sandbox `id`, whose container (or local daemon) a reopen found gone or stopped /// (`container`, `stopped`): one unforced Delete of a stopped container, then /// its records, so its number is free again. Called under the project's /// guard; the Sandbox's own guard is taken too. The record is read again @@ -485,8 +531,9 @@ pub(crate) fn reclaim(args: &SdkValue, id: &str, container: &str, stopped: bool) )); } if stopped { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - match layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request).delete() + match crate::discovery::recorded(args, id, &value)? + .stopped() + .delete() { Err(failure) if !failure.container_gone() => return Err(failed(failure)), _ => {} diff --git a/engine/crates/layerfs-node/src/unmount.rs b/engine/crates/layerfs-node/src/unmount.rs new file mode 100644 index 0000000000..34b0c34b95 --- /dev/null +++ b/engine/crates/layerfs-node/src/unmount.rs @@ -0,0 +1,282 @@ +//! Workspace-scoped unmount ownership; Sandbox discovery is held only at +//! admission and shared-record cleanup, never during the daemon's helper wait. +//! Mount paths are unique within one daemon lifetime; Sandbox removal clears +//! the terminal records before that directory can be reused. +use crate::{ + discovery, + job::{failed, refused, Outcome}, + registry::SandboxState, +}; +use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; +use std::{ + fs::{File, OpenOptions}, + os::unix::{fs::OpenOptionsExt, io::AsRawFd}, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +extern "C" { + fn flock(fd: i32, operation: i32) -> i32; +} +const LOCK_SH: i32 = 1; +const LOCK_EX: i32 = 2; +const LOCK_NB: i32 = 4; +const LOCK_UN: i32 = 8; +fn millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(i64::MAX as u128) as i64 +} +fn record(state: &SandboxState, token: WorkspaceToken) -> PathBuf { + discovery::workspace_file(state, token).with_extension("unmount") +} +struct Lease(File); +impl Drop for Lease { + fn drop(&mut self) { + unsafe { flock(self.0.as_raw_fd(), LOCK_UN) }; + } +} +fn lease(path: &std::path::Path, operation: i32) -> Result, SdkRefusal> { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .map_err(failed)?; + discovery::private(path, false)?; + if unsafe { flock(file.as_raw_fd(), operation | LOCK_NB) } == 0 { + return Ok(Some(Lease(file))); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::WouldBlock { + Ok(None) + } else { + Err(failed(error)) + } +} +/// Explicit unlock matters when a concurrent fork inherited the descriptor. +pub(crate) struct Attempt { + _lease: Lease, + path: PathBuf, + started: i64, +} +fn result(value: &SdkValue) -> Outcome { + if value.get_text("state") == Some("Unmounted") { + return Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])); + } + let detail = SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "elapsedMs", + SdkValue::Int( + millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0), + ), + ), + ("original", value.get("detail").cloned().unwrap_or_default()), + ]); + Err(SdkRefusal { + code: value.get_text("code").unwrap_or("Unknown").into(), + message: value + .get_text("message") + .unwrap_or("unmount outcome retained") + .into(), + detail, + }) +} +fn progress(value: &SdkValue, active: bool) -> SdkRefusal { + let elapsed = millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0); + SdkRefusal { + code: if active { "Busy" } else { "Unknown" }.into(), + message: if active { + format!("Workspace is Unmounting; waited {elapsed}ms") + } else { + format!("unmount caller ended without publishing its result after {elapsed}ms; inspect daemon custody, do not resend") + }, + detail: SdkValue::map([ + ( + "state", + SdkValue::text(if active { "Unmounting" } else { "Retained" }), + ), + ("elapsedMs", SdkValue::Int(elapsed)), + ]), + } +} +fn observed(path: &std::path::Path) -> Result, SdkRefusal> { + match discovery::load(path) { + Ok(value) => Ok(Some(value)), + Err(_) if !path.exists() => Ok(None), + Err(error) => Err(error), + } +} +/// Consulted before token lookup, including when another process removed +/// the Workspace record. Does not acquire the Sandbox guard or dial daemon. +fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { + let Some(value) = observed(path)? else { + return Ok(None); + }; + if value.get_text("state") != Some("Unmounting") { + return Ok(Some(value)); + } + let available = lease(&path.with_extension("unmount-lock"), LOCK_SH)?; + // Publication may have completed between our read and lease acquisition. + let latest = observed(path)?; + match latest { + Some(latest) if latest.get_text("state") != Some("Unmounting") => Ok(Some(latest)), + None => Ok(None), + Some(_) if available.is_none() => Err(progress(&value, true)), + Some(_) => { + let error = progress(&value, false); + Ok(Some(SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "startedMs", + value.get("startedMs").cloned().unwrap_or_default(), + ), + ("path", value.get("path").cloned().unwrap_or_default()), + ("code", SdkValue::text(error.code)), + ("message", SdkValue::text(error.message)), + ("detail", error.detail), + ]))) + } + } +} +pub(crate) fn check(state: &SandboxState, path: &str) -> Result<(), SdkRefusal> { + if let Some(value) = find(state, path)? { + if value.get_text("state") == Some("Unmounted") { + return Err(refused("NotMounted", "Workspace already unmounted")); + } + return result(&value).map(drop); + } + Ok(()) +} +pub(crate) fn find(state: &SandboxState, path: &str) -> Result, SdkRefusal> { + // Steady-state token/directory lookups already have a token: avoid a + // directory scan proportional to the lifetime's completed unmounts. + let known = crate::registry::lock(&state.mounts)? + .get(path) + .map(|mount| mount.workspace.bound.token); + if let Some(token) = known { + return inspect(&record(state, token)); + } + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + if let Some(value) = observed(&entry.path())? { + if value.get_text("path") == Some(path) { + return inspect(&entry.path()); + } + } + } + } + Ok(None) +} +/// Closing a running Sandbox may not race a registered terminal operation. +pub(crate) fn check_all(state: &SandboxState) -> Result, SdkRefusal> { + let mut recoverable = Vec::new(); + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + // Even a published result may still be cleaning its Workspace + // record. Close must not delete/reuse the Sandbox directory in + // that short interval while the original lease is held. + let probe = lease(&entry.path().with_extension("unmount-lock"), LOCK_SH)?; + if probe.is_none() { + if let Some(value) = observed(&entry.path())? { + return Err(progress(&value, true)); + } + } + if let Some(value) = inspect(&entry.path())? { + if value.get_text("state") == Some("Retained") { + recoverable.push(crate::registry::text(&value, "path")?.to_owned()); + } + } + } + } + Ok(recoverable) +} +/// Called under the Sandbox guard, after all before-effect checks. +pub(crate) fn begin( + state: &SandboxState, + token: WorkspaceToken, + path: &str, +) -> Result { + let path_record = record(state, token); + let file = lease(&path_record.with_extension("unmount-lock"), LOCK_EX)? + .ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?; + let attempt = Attempt { + _lease: file, + path: path_record, + started: millis(), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text("Unmounting")), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ]), + )?; + Ok(attempt) +} +/// Publish while the Workspace lease prevents another terminal request. +/// No Sandbox guard is needed to replace this attempt's own result file. +pub(crate) fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> { + match outcome { + Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => { + std::fs::remove_file(&attempt.path).map_err(failed)?; + } + _ => { + let (state, code, message, detail) = match outcome { + Ok(_) => ("Unmounted", "", "", SdkValue::Null), + Err(error) => ( + "Retained", + error.code.as_str(), + error.message.as_str(), + error.detail.clone(), + ), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text(state)), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ("code", SdkValue::text(code)), + ("message", SdkValue::text(message)), + ("detail", detail), + ]), + )?; + } + } + Ok(()) +} +pub(crate) fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> { + std::fs::remove_file(&attempt.path).map_err(failed) +} +pub(crate) fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome { + // Do not replace a known daemon success with a cleanup Busy/Failed. + match &mut outcome { + Ok(SdkValue::Map(fields)) => { + fields.push(("cleanupError".into(), SdkValue::text(error.message))); + } + Err(original) => { + original.detail = SdkValue::map([ + ("original", original.detail.clone()), + ("cleanupError", SdkValue::text(error.message)), + ]); + } + _ => {} + } + outcome +} +pub(crate) fn repeat(value: &SdkValue) -> Outcome { + result(value) +} diff --git a/engine/crates/layerfs-node/src/workspace.rs b/engine/crates/layerfs-node/src/workspace.rs index 829df67f71..8dcb0c4ca9 100644 --- a/engine/crates/layerfs-node/src/workspace.rs +++ b/engine/crates/layerfs-node/src/workspace.rs @@ -8,9 +8,7 @@ //! by a mount and reclaimed by the next `sandbox.create` or //! `runtime.sandbox.reclaim`. use crate::job::{failed, Outcome}; -use crate::registry::{ - lock, naming, random, sandbox, sandbox_live, sandbox_refreshed, text, Mount, -}; +use crate::registry::{lock, naming, random, sandbox, sandbox_live, text, Mount}; use crate::sandbox::{operation, WORKSPACES}; use layerfs_api::internal::workspace; use layerfs_api::sandbox::{values, Unmounted}; @@ -96,8 +94,10 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { if let SdkValue::Map(fields) = &mut value { fields.push(("sandbox".to_owned(), SdkValue::text(state.id.as_str()))); fields.push(("container".to_owned(), SdkValue::text(state.name.as_str()))); - let image = state.runtime.request().image.as_str(); - fields.push(("image".to_owned(), SdkValue::text(image))); + fields.push(( + "image".to_owned(), + SdkValue::text(crate::runtime::image(&state.runtime)), + )); } crate::discovery::save_workspace(&state, &mounted, branch)?; lock(&state.mounts)?.insert( @@ -112,15 +112,21 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { /// One recorded Sandbox of the project: running (`Ok(true)`), or found /// gone or stopped (`Ok(false)`, with its container and whether it stopped). -/// Its container is inspected once; no daemon is dialed. -fn observed(record: &SdkValue, directory: &Path) -> Result, SdkRefusal> { +/// Its container (or local daemon process) is inspected once; no daemon +/// is dialed. +fn observed( + args: &SdkValue, + id: &str, + record: &SdkValue, + directory: &Path, +) -> Result, SdkRefusal> { let cached = lock(&crate::registry::SANDBOXES)?.get(directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => { - let (docker, selector, request) = crate::discovery::runtime(record)?; - layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request).map(drop) - } + // a container, or with the local runtime a daemon process + None => crate::discovery::recorded(args, id, record)? + .reopen() + .map(drop), }; match observed { Ok(()) => Ok(None), @@ -152,7 +158,7 @@ fn records(args: &SdkValue) -> Result Result<(), SdkRefusal> { for (id, directory, value) in records(args)? { - if let Some((container, stopped)) = observed(&value, &directory)? { + if let Some((container, stopped)) = observed(args, &id, &value, &directory)? { crate::sandbox::reclaim(args, &id, &container, stopped)?; } } @@ -168,7 +174,7 @@ pub(crate) fn reclaim_gone(args: &SdkValue) -> Result<(), SdkRefusal> { pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { let mut items = Vec::new(); for (id, directory, value) in records(args)? { - if observed(&value, &directory)?.is_some() { + if observed(args, &id, &value, &directory)?.is_some() { continue; } items.push(SdkValue::map([ @@ -198,7 +204,7 @@ pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { pub(crate) fn reclaim(args: &SdkValue) -> Outcome { let mut reclaimed = Vec::new(); for (id, directory, value) in records(args)? { - let Some((container, stopped)) = observed(&value, &directory)? else { + let Some((container, stopped)) = observed(args, &id, &value, &directory)? else { continue; }; let _placing = crate::discovery::guard(&crate::discovery::project_dir(args)?)?; @@ -252,10 +258,19 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { /// `workspace.unmount(ws, { discard?, force? })`: `Busy` while a background /// process this binding started still runs there, then the daemon's answer. pub(crate) fn unmount(args: &SdkValue) -> Outcome { - let state = sandbox_refreshed(args)?; - let _guard = crate::discovery::guard(&state.directory)?; + let state = sandbox(args)?; let path = text(args, "path")?; + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } + let guard = crate::discovery::guard(&state.directory)?; + // Another process may have registered while this call waited for the + // short admission section. Check again before sending anything. + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } let token = state.token(path)?; + crate::discovery::refresh(&state)?; // The runtime owner knows its own processes; the daemon's kernel probe // still answers Busy for any other holder. let running = crate::exec::running_in(&state, path)?; @@ -273,22 +288,59 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { )]), }); } - let value = { + let attempt = crate::unmount::begin(&state, token, path)?; + drop(guard); + // Only this Workspace's lease remains held. Other Workspaces and + // processes can take Sandbox ownership while the daemon helper waits. + let mut sent = false; + let outcome = (|| { let mut control = crate::trace::control(&state.controls)?; + sent = true; let mut api = workspace(&mut control); // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. if args.get_bool("force") { match api.unmount(token, true).map_err(operation)? { - Some(outcome) => values::force_unmounted(&outcome), - None => values::unmounted(&Unmounted::Closed)?, + Some(outcome) => Ok(values::force_unmounted(&outcome)), + None => values::unmounted(&Unmounted::Closed), } } else { - let discard = args.get_bool("discard"); - values::unmounted(&api.unmount_checked(token, discard).map_err(operation)?)? + match api.unmount_checked(token, args.get_bool("discard")) { + Ok(answer) => values::unmounted(&answer), + Err(error) => Err(operation(error)), + } } + })(); + let outcome = match outcome { + Err(error) if error.code == "Missing" => { + Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])) + } + other => other, }; - std::fs::remove_file(crate::discovery::workspace_file(&state, token)).map_err(failed)?; - lock(&state.mounts)?.remove(path); - Ok(value) + // Preserve the daemon's known outcome before any discovery-lock wait. + // A file/cleanup failure is separate from an already successful detach. + let publication = if !sent { + crate::unmount::cancel(&attempt) + } else { + crate::unmount::finish(&attempt, path, &outcome) + }; + if let Err(error) = publication { + return crate::unmount::cleanup_failure(outcome, error); + } + if outcome.is_ok() { + let cleanup = (|| { + let _guard = crate::discovery::guard(&state.directory)?; + match std::fs::remove_file(crate::discovery::workspace_file(&state, token)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + lock(&state.mounts)?.remove(path); + Ok(()) + })(); + if let Err(error) = cleanup { + return crate::unmount::cleanup_failure(outcome, error); + } + } + outcome } diff --git a/engine/crates/layerfs-node/tests/README.md b/engine/crates/layerfs-node/tests/README.md index 75af6d4e58..1c5c9e54fd 100644 --- a/engine/crates/layerfs-node/tests/README.md +++ b/engine/crates/layerfs-node/tests/README.md @@ -12,6 +12,7 @@ cargo +1.85.1 build --manifest-path Cargo.toml --locked -p layerfs-node cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --tests --no-run timeout 30 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_discovery timeout 90 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_admission +timeout 60 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_local ``` Use the worktree's registered Cargo resources for these commands. Node 20 or @@ -24,6 +25,15 @@ number replacement during inspection, and discovery-lock release while a forked child retains the descriptor. Its C fixture interposes `flock` only in the child Node process; the fixture's fork is bounded and released on test unwinding. +`native_local` covers native Sandbox records (`LAYERFS_SANDBOX=native`) through +`runtime.sandbox.reclaim`: a record naming any directory other than where its +deployment puts that Sandbox is refused and nothing is deleted, the configured +`localDir` must be absolute and plain, a stopped native Sandbox is deleted and its +number freed, a damaged boot or port is refused, a native record is used only by +a native deployment, and a record of the former `local` runtime is never used or +deleted. Record handling is the same code on Linux and macOS, so it runs here; +starting a native Sandbox is supported on macOS (caller identity, loopback NFS) and Linux (root daemon, nonroot 501:20 commands, FUSE). Linux lifecycle qualification also runs `layerfs-sandbox --test native_provider` as root and the SDK smoke with `LAYERFS_SANDBOX=native`. + `native_admission` covers initial connection admission and its `Busy` deadline. The deadline test pays the product's full 60-second readiness wait. The mock daemon has bounded socket waits and stops independently of a test assertion. diff --git a/engine/crates/layerfs-node/tests/native_discovery.rs b/engine/crates/layerfs-node/tests/native_discovery.rs index f0afde037f..8498ae9424 100644 --- a/engine/crates/layerfs-node/tests/native_discovery.rs +++ b/engine/crates/layerfs-node/tests/native_discovery.rs @@ -32,11 +32,19 @@ fn another_404_is_a_failure_that_removes_nothing() { let home = Home::new(); let engine = engine(&home.0, 404, r#"{"message":"page not found"}"#); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(404\)/); + assert.match(answer.refused.message, /page not found/); + "#, None, Duration::from_secs(5), ); - engine.join().unwrap(); + assert!(engine + .join() + .unwrap() + .starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } @@ -113,11 +121,21 @@ fn a_stopped_container_the_engine_will_not_delete_keeps_its_records() { ], ); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(409\)/); + assert.match(answer.refused.message, /cannot remove container: container is running/); + "#, None, Duration::from_secs(5), ); - assert_eq!(engine.join().unwrap().len(), 2); + let requests = engine.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); + assert!(requests[1].starts_with(&format!( + "DELETE /v1.54/containers/{CID}?force=false&v=false HTTP/1.1\r\n" + ))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs new file mode 100644 index 0000000000..b6c8a7ba2d --- /dev/null +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -0,0 +1,274 @@ +//! External tests of the built Node binding's native Sandbox records +//! (LAYERFS_SANDBOX=native). No private Rust source is included: every case +//! writes a Sandbox record as a previous call would have, then calls the +//! addon's public `runtime.sandbox.reclaim` and observes the reply and the +//! files it left. Neither Docker, FUSE, nor a Store is needed. +#![cfg(target_os = "linux")] +mod support; +use layerfs_bridge::control::{Answer, Reply, SdkValue}; +use std::{ + io::Write, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + time::Duration, +}; +use support::*; + +/// This boot's id as a record keeps it: 32 lowercase hex digits. +fn boot() -> String { + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .unwrap() + .trim() + .replace('-', "") +} + +/// Sandbox 1 of `proj` recorded as a `runtime` (native, or the former local) Sandbox +/// in `directory` named `name`, its daemon long gone (no process has id +/// u32::MAX). `edit` changes the fields before the record is written. +fn record( + home: &Home, + runtime: &str, + directory: &str, + name: &str, + edit: impl FnOnce(&mut Vec<(String, SdkValue)>), +) { + let bytes = |n: u8| SdkValue::Bytes(vec![n; 32]); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("id".into(), SdkValue::text("1")), + ("project".into(), SdkValue::text("proj")), + ("name".into(), SdkValue::text(name)), + ("container".into(), SdkValue::text(CID)), + ("runtime".into(), SdkValue::text(runtime)), + ("directory".into(), SdkValue::text(directory)), + ("pid".into(), SdkValue::Int(i64::from(u32::MAX))), + ("start".into(), SdkValue::text("1")), + ("boot".into(), SdkValue::text(boot())), + ("port".into(), SdkValue::Int(40404)), + ("image".into(), SdkValue::text("local")), + ("secret".into(), bytes(1)), + ("peer".into(), bytes(2)), + ("instance".into(), bytes(3)), + ]; + edit(&mut fields); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(home.0.join("proj/sandboxes/1/sandbox.record")) + .unwrap(); + file.write_all( + &Answer { + id: 1, + reply: Reply::Sdk(SdkValue::Map(fields)), + } + .encode() + .unwrap(), + ) + .unwrap(); +} + +/// Replaces (or with `None` removes) one field of a record. +fn set(key: &'static str, value: Option) -> impl FnOnce(&mut Vec<(String, SdkValue)>) { + move |fields| { + fields.retain(|(k, _)| k != key); + if let Some(value) = value { + fields.push((key.into(), value)); + } + } +} + +/// `LAYERFS_LOCAL_DIR` of these tests, created like a deployment's. +fn root(home: &Home) -> PathBuf { + let root = home.0.join("local"); + std::fs::create_dir_all(&root).unwrap(); + root +} + +/// Where a `root` deployment puts Sandbox 1 of `proj`, created with a file. +fn own(root: &Path) -> PathBuf { + let own = root.join("layerfs-proj-1"); + std::fs::create_dir_all(own.join("tmp")).unwrap(); + std::fs::write(own.join("daemon.log"), b"log").unwrap(); + own +} + +/// The public reclaim of a `sandbox` (native or docker) deployment, with +/// `localDir` when given, followed by `then` (JavaScript, `r` is the reply). +fn reclaim(home: &Home, sandbox: &str, local_dir: Option<&str>, then: &str) { + let deploy = match local_dir { + Some(dir) => format!("{{sandbox:{sandbox:?}, localDir:{dir:?}}}"), + None => format!("{{sandbox:{sandbox:?}}}"), + }; + home.node( + &format!( + "const r = await native.backendCall('runtime.sandbox.reclaim', {{...args, deploy:{deploy}}});\n{then}" + ), + None, + Duration::from_secs(10), + ); +} + +const NEITHER: &str = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /neither used nor deleted/);"; +const RECLAIMED: &str = "assert.deepEqual(r, {value:{kind:'ok',reclaimed:['1']}});"; + +fn text(path: &Path) -> &str { + path.to_str().unwrap() +} + +#[test] +fn a_native_record_naming_another_directory_is_neither_used_nor_deleted() { + let home = Home::new(); + let root = root(&home); + // a data directory a damaged or edited record names by its own name + let data = home.0.join("data"); + std::fs::create_dir(&data).unwrap(); + std::fs::write(data.join("keep"), b"x").unwrap(); + let old = home.0.join("old/layerfs-proj-1"); + std::fs::create_dir_all(&old).unwrap(); + let r = text(&root); + for (directory, name) in [ + (text(&data).to_owned(), "data"), + (format!("{r}/data"), "data"), + // another LAYERFS_LOCAL_DIR, another Sandbox, another name + (text(&old).to_owned(), "layerfs-proj-1"), + (format!("{r}/layerfs-proj-2"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1/sub"), "sub"), + ("relative/layerfs-proj-1".to_owned(), "layerfs-proj-1"), + (format!("{r}/../etc/layerfs-proj-1"), "layerfs-proj-1"), + ("/".to_owned(), ""), + ] { + record(&home, "native", &directory, name, |_| {}); + reclaim(&home, "native", Some(r), NEITHER); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "the record of {directory} was removed" + ); + } + assert!( + data.join("keep").exists(), + "a delete reached {}", + data.display() + ); + assert!(old.exists(), "a delete reached {}", old.display()); +} + +#[test] +fn a_native_record_is_used_only_with_its_deployment_s_absolute_local_dir() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + let refused = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r));"; + reclaim(&home, "native", None, refused); + reclaim(&home, "native", Some("relative"), refused); + reclaim( + &home, + "native", + Some(&format!("{}/../local", text(&root))), + refused, + ); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} + +#[test] +fn a_stopped_native_sandbox_where_its_deployment_puts_it_is_deleted_and_reclaimed() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!own.exists(), "the stopped Sandbox's directory is deleted"); + assert!( + !home.0.join("proj/sandboxes/1").exists(), + "its number is free again" + ); + assert!(root.exists(), "LAYERFS_LOCAL_DIR itself stays"); +} + +#[test] +fn a_record_from_before_boot_ids_is_read_without_one() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record( + &home, + "native", + text(&own), + "layerfs-proj-1", + set("boot", None), + ); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} + +#[test] +fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + for (key, value, code) in [ + ("boot", SdkValue::text("not-hex"), "InvalidInput"), + ("port", SdkValue::Int(70000), "Failed"), + ] { + record( + &home, + "native", + text(&own), + "layerfs-proj-1", + set(key, Some(value)), + ); + reclaim( + &home, + "native", + Some(text(&root)), + &format!("assert.equal(r.refused?.code, {code:?}, JSON.stringify(r));"), + ); + assert!(own.join("daemon.log").exists(), "{key}"); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "{key}" + ); + } +} + +#[test] +fn a_docker_record_stays_docker_under_a_native_deployment() { + // the default record of Home is a Docker one: with no Engine at its + // socket it is neither reclaimed nor read as a native Sandbox + let home = Home::new(); + let root = root(&home); + reclaim( + &home, + "native", + Some(text(&root)), + "assert.equal(r.refused?.code, 'Failed', JSON.stringify(r));", + ); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} + +#[test] +fn a_record_of_the_former_local_runtime_is_never_used_nor_deleted() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + let mixed = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /never mixed/); + assert.match(r.refused.message, /neither used nor deleted/);"; + // a record of the former `local` runtime is used by no deployment + record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), mixed); + reclaim(&home, "docker", Some(text(&root)), mixed); + // nor is a native record by any deployment but a native one + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "docker", Some(text(&root)), mixed); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); + // a native deployment uses its own record: stopped, it is reclaimed + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!own.exists()); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} diff --git a/engine/crates/layerfs-node/tests/support/engine.rs b/engine/crates/layerfs-node/tests/support/engine.rs index 6fb656c2d4..302c2f8e38 100644 --- a/engine/crates/layerfs-node/tests/support/engine.rs +++ b/engine/crates/layerfs-node/tests/support/engine.rs @@ -1,5 +1,5 @@ use super::*; -/// The Engine at `home/docker.sock`, answering one request with `status` and `body`. +/// The Engine at `home/docker.sock`, answering one business request with `status` and `body`. pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { let engine = engines(home, vec![(status, body.to_owned())]); thread::spawn(move || engine.join().unwrap().remove(0)) @@ -7,6 +7,7 @@ pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { /// The Engine at `home/docker.sock`, answering each request, one connection /// each, with the next `(status, body)`; it returns their request heads. +/// Version probes get API 1.54 without consuming a response or calling `before`. pub fn engines(home: &Path, responses: Vec<(u16, String)>) -> JoinHandle> { engines_before(home, responses, || {}) } @@ -22,11 +23,9 @@ pub fn engines_before( let mut before = Some(before); let deadline = Instant::now() + Duration::from_secs(3); let mut requests = Vec::new(); - for (status, body) in responses { - let response = format!( - "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); + let mut responses = responses.into_iter().peekable(); + while responses.peek().is_some() { + assert!(Instant::now() < deadline, "bounded engine request deadline"); let stream = loop { match listener.accept() { Ok((stream, _)) => break stream, @@ -43,6 +42,9 @@ pub fn engines_before( stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); let mut request = String::new(); loop { @@ -52,11 +54,23 @@ pub fn engines_before( } request.push_str(&line); } - if let Some(before) = before.take() { - before(); - } + let probe = request.split_whitespace().nth(1) == Some("/version"); + let (status, body) = if probe { + (200, r#"{"ApiVersion":"1.54"}"#.to_owned()) + } else { + if let Some(before) = before.take() { + before(); + } + responses.next().unwrap() + }; + let response = format!( + "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); (&stream).write_all(response.as_bytes()).unwrap(); - requests.push(request); + if !probe { + requests.push(request); + } } requests }) diff --git a/engine/crates/layerfs-overlay/Cargo.toml b/engine/crates/layerfs-overlay/Cargo.toml index a0de581508..842ac36348 100644 --- a/engine/crates/layerfs-overlay/Cargo.toml +++ b/engine/crates/layerfs-overlay/Cargo.toml @@ -11,5 +11,5 @@ blake3 = { version = "=1.8.5", default-features = false } nix = { version = "=0.31.3", default-features = false, features = ["fs"] } rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 747630e708..ff9c423fc5 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -22,5 +22,5 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs"] } [target.'cfg(target_os = "linux")'.dev-dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthread", "signal"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/engine/crates/layerfs-sandbox/Cargo.toml b/engine/crates/layerfs-sandbox/Cargo.toml index e75b3fd4ce..a1f1141f86 100644 --- a/engine/crates/layerfs-sandbox/Cargo.toml +++ b/engine/crates/layerfs-sandbox/Cargo.toml @@ -7,4 +7,13 @@ rust-version.workspace = true license.workspace = true [dependencies] -nix = { version = "=0.31.3", features = ["poll", "socket", "fs"] } +nix = { version = "=0.31.3", features = [ + "poll", + "socket", + "fs", + "signal", + "mount", + "user", + "sched", + "process", +] } diff --git a/engine/crates/layerfs-sandbox/PROVIDERS.md b/engine/crates/layerfs-sandbox/PROVIDERS.md new file mode 100644 index 0000000000..85c740797c --- /dev/null +++ b/engine/crates/layerfs-sandbox/PROVIDERS.md @@ -0,0 +1,134 @@ +# Sandbox runtimes + +Status: Current general guide. + +`backend::runtime` holds the concrete runtime choice and the launched owner. +There are no provider or instance traits: each backend is an ordinary concrete +type and the only dispatch is one `match` per method. + +- `Runtime` is one deployment's selection and is kept for every Sandbox it + starts: `Docker(DockerProvider)` (the default), `MacosNative(NativeMacosProvider)`, + `LinuxNative(NativeLinuxProvider)`, `Namespaces(NamespaceProvider)` (Linux + only) and `Windows`, a placeholder with no backend. + `Runtime::{capabilities, local_directory, native_paths, launch, reopen, + reopen_stopped}` dispatch to them. +- `Sandbox` is what `Runtime::launch`/`reopen` return: `Docker(docker::Sandbox)`, + `Native(native::Sandbox)` or `Namespaces(Box)`, with + `identity`, `endpoint`, `run`, `track`, `log_tail`, `inspect`, `stop`, + `delete`, `scratch_directory`, `local_record` and `record`. + +`layerfs_sandbox::{Runtime, Sandbox}` re-export both. `provider` keeps the +inputs and observations they share: `LaunchRequest`, `ProviderLaunchOptions`, +`LocalRecord`, `DockerRecord`, `SandboxRecord`, `ExecExit`, +`SandboxInspection`, `ProviderCapabilities` and the `LaunchPhase`/`LaunchFailure` +pair. The runtime layer has no daemon control dependency. `backend::HelperSpec` +is the namespace helper's line format; it is compiled and tested on every +platform, while only the helper that reads it is Linux only. + +## Docker + +`Runtime::Docker(DockerProvider::new(docker))` adapts the existing Engine client +and `SandboxRequest`. It shares its lifecycle admission, API 1.48–1.54 +negotiation cache, streamed private archive, logs, exact deployment inspection, +and explicit stop/delete. Call `runtime.launch(LaunchRequest { .. })` with the +executable, private configuration, name, explicit startup wait, and +`ProviderLaunchOptions::Docker { image, store_volume, port, connect_host }`; +Docker validates the immutable image digest, existing volume name, port, +connect_host format and labels before any Engine effect. The runtime does not +decode daemon configuration or perform authentication. + +`Sandbox::run` streams stdout/stderr to caller sinks with bounded windows and +returns `ExecExit` only after one inspection knows the root process exit. It +rejects `stdin=true` before effects: interactive users keep the existing +concrete `create_exec/start/into_parts` stream API. EOF alone is not an exit and +no automatic poll follows an unknown exit. `track` needs no host registration +because container teardown owns descendants; scratch is `/tmp` and +`local_record()` is absent. `inspect` checks the recorded deployment but neither +adopts it nor changes lifecycle admission; unknown PID/exit/storage fields stay +absent. + +## Native + +`Runtime::MacosNative(NativeMacosProvider::new(root))` runs the daemon as this +user and serves loopback NFS; `Runtime::LinuxNative(NativeLinuxProvider::new(root))` +runs a root daemon serving FUSE with ordinary commands as a nonroot identity. +Each refuses the wrong platform before root access, and Linux refuses a non-root +caller the same way. `backend::native` owns the one lifecycle both use; only the +platform facts differ: + +- `backend::macos`: `darwin.rs` (`proc_pidinfo`, `sysctl`, the `mount` table and + `umount -f`) and `mounts.rs` (force detach, deepest first). +- `backend::linux`: `process.rs` (`/proc//stat`, the boot id and the + process table) and `mounts.rs` (`/proc/self/mountinfo`, lazy `MNT_DETACH`). + +The provider validates the command identity, the environment it is given and +the daemon executable, then starts one daemon in a private directory with the +platform's kernel marker (`LAYERFS_KERNEL=fuse` on Linux, `nfs` on macOS); it +rejects conflicting overrides. Launch preserves the original single-attempt +process lifecycle and partial owner on failure. `inspect` reports exact process +liveness; unknown exit/storage facts stay absent. Records remain the Node +binding's responsibility; runtimes neither scan directories nor infer project +ownership. + +## Namespaces (Linux) + +`Runtime::Namespaces(NamespaceProvider::new(root))` is the same local root and +daemon control as `LinuxNative`, with each Sandbox's daemon in its own mount, +PID and IPC namespaces and its own cgroup v2 directory. The VM that holds this +kernel is the boundary (one per Store); the namespaces keep Sandboxes on one +Store out of each other. + +- `backend::namespaces::create` makes the Sandbox's private directory, copies + the daemon to `/bin/layerfs-daemon` (0755), writes the setup (0600) + and a `HelperSpec`, makes `/sys/fs/cgroup/layerfs/`, then spawns the + helper's init stage. +- `backend::namespaces::sandbox` owns the launched handle: the init process's + identity, `cgroup.kill` for stop (a `SIGKILL` to everything the Sandbox + started, `populated 0` awaited), cgroup and directory removal on delete, and + reopen by the recorded process plus its cgroup membership. +- `backend::namespaces::exec` runs one command through the helper's exec stage. +- `src/bin/layerfs-sandbox-helper` performs the calls that act on the calling + process, so the product crate stays `unsafe`-free: the init stage creates the + namespaces, joins the cgroup and starts the inner stage (PID 1), which mounts + the private root (the VM's userland read-only, a private `/proc`, `/dev`, + `/dev/pts`, `/dev/shm`, `/tmp`, plus the Sandbox's directory and the Store's + directory at their own paths), pivots, starts the daemon and reaps; the exec + stage enters the namespaces, sets no-new-privs and runs the command as the + requested uid:gid (its own exit is a signal, never a command's status). + +Preconditions are checked before any effect and refused with the existing +`RuntimeError::Unsupported`, never replaced by a weaker Sandbox: root, the +unified cgroup v2 hierarchy (and `cgroup.kill`), `/dev/fuse`, and +`user.max_user_namespaces=0` — without the last one a command could unshare a +user namespace and hold its own copy of the Workspace mount. The runtime never +sets that limit itself. There is no image: the private root binds the VM's own +userland, so the Sandbox runs the daemon and libraries the VM already has. +`ProviderLaunchOptions::Namespaces { command_identity, environment, store }` +names the Store file whose directory is bound in; `SandboxRecord::Namespaces` +keeps the same physical facts as `LocalRecord`. + +## Windows + +`Runtime::Windows` is a placeholder: `capabilities`, `local_directory`, +`native_paths`, `launch`, `reopen` and `reopen_stopped` all refuse with the +existing `RuntimeError::Unsupported` before any effect, and no `Sandbox` variant +exists for it. No Windows host runtime is implemented; see +[`src/backend/windows/README.md`](src/backend/windows/README.md) for the planned +routes. + +## Failure and records + +A `LaunchFailure` retains the acknowledged `Sandbox`, when any. Its cause may be +`RuntimeError::Docker(Box)`: match the typed variant for +the original Create/Upload/Start/Listener receipt, partial IDs, upload work and +pending source bytes. Upload is within existing `LaunchPhase::Create`; inspect +the typed cause to distinguish it. A partially decoded Create ID is never +adopted, and retaining a handle permits an explicit later observation, never a +replay of creation or Start. + +`Runtime::reopen(id, &SandboxRecord)` validates the exact existing deployment or +process and endpoint without starting it; `reopen_stopped` requires the caller's +prior exact stopped observation and admits only non-forcing Delete. Both accept +`SandboxRecord`, which represents `LocalRecord`, `Namespaces(LocalRecord)` or +`DockerRecord`; each runtime refuses another runtime's record before effects. `Sandbox::record()` exposes the persisted deployment facts. No disk +format changes occur: a record without a `runtime` field is Docker's. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs index 0b1ceaa8a6..88ba0e4eb1 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs @@ -1,11 +1,11 @@ //! One-attempt create/start/stop/delete, with no implicit adoption or volume deletion. use super::{ container_types::{CreatePhase, Sandbox, SandboxCreateFailure, SandboxRequest}, - json::{once, Json}, - strings::quoted, topology::SECCOMP, Docker, }; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; use crate::{ContainerId, RuntimeError, WireFailure}; use std::io::{self, Read, Write}; impl SandboxRequest { diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs index 0d207f1049..16ddfe2e62 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs @@ -1,8 +1,6 @@ //! Structurally observed exact container and loopback-published daemon endpoint. -use super::{ - json::{once, Json}, - Sandbox, -}; +use super::Sandbox; +use crate::backend::json::{once, Json}; use crate::{ContainerId, RuntimeError, WireFailure}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; impl Sandbox { @@ -61,6 +59,20 @@ impl Sandbox { if !self.started || self.stop_attempted || self.delete_attempted { return Err(self.invalid("running endpoint admission")); } + let (_, endpoint) = self.observe(true)?; + endpoint.ok_or_else(|| self.invalid("missing daemon endpoint")) + } + /// One exact deployment/state observation; absent optional facts remain absent. + pub fn inspect(&self) -> Result> { + self.observe(false).map(|(inspection, _)| inspection) + } + fn observe( + &self, + require_running: bool, + ) -> Result<(crate::provider::SandboxInspection, Option), Box> { + if self.deleted { + return Err(self.invalid("deleted container observation")); + } let mut r = self .docker .exchange( @@ -80,6 +92,8 @@ impl Sandbox { let mut topology = true; let mut running = None; let mut endpoint = None; + let mut exit_code = None; + let mut process_id = None; let mut j = Json::new(&mut r.body); j.object(|j, key| { if key.equals("Id") { @@ -108,6 +122,20 @@ impl Sandbox { if key.equals("Running") { once(&mut bits, 1)?; running = Some(j.boolean()?); + } else if key.equals("ExitCode") { + once(&mut bits, 16)?; + if !j.null()? { + exit_code = + Some(i32::try_from(j.integer()?).map_err(|_| { + RuntimeError::Protocol("container exit width") + })?); + } + } else if key.equals("Pid") { + once(&mut bits, 32)?; + process_id = Some( + u32::try_from(j.integer()?) + .map_err(|_| RuntimeError::Protocol("container PID width"))?, + ); } else if key.equals("Paused") || key.equals("Restarting") || key.equals("Dead") @@ -126,7 +154,7 @@ impl Sandbox { } Ok(()) })?; - topology &= bits == 15; + topology &= bits & 15 == 15; } else if key.equals("NetworkSettings") { once(&mut fields, 4)?; let mut bits = 0; @@ -137,6 +165,9 @@ impl Sandbox { j.object(|j, key| { if key.equals(&format!("{}/tcp", self.request.port)) { once(&mut selected, 1)?; + if j.null()? { + return Ok(()); + } let mut count = 0; j.array(|j| { count += 1; @@ -213,27 +244,36 @@ impl Sandbox { Ok(()) })?; j.finish()?; - // the exact recorded container, intact but not running: its daemon - // and every mount in it are gone (Docker restarted, for instance) - if observed == Some(self.id) - && fields == 127 - && image - && topology - && running == Some(false) - { - return Err(RuntimeError::Protocol(crate::types::STOPPED)); - } if observed != Some(self.id) || fields != 127 || !image || !topology - || running != Some(true) + || running.is_none() { return Err(RuntimeError::Protocol( "exact running container observation", )); } - endpoint.ok_or(RuntimeError::Protocol("missing daemon endpoint")) + if require_running && running == Some(false) { + return Err(RuntimeError::Protocol(crate::types::STOPPED)); + } + if running == Some(true) && endpoint.is_none() { + return Err(RuntimeError::Protocol("missing daemon endpoint")); + } + Ok(( + crate::provider::SandboxInspection { + identity: self.id, + running: running.expect("required"), + exit_code: if running == Some(false) { + exit_code + } else { + None + }, + process_id: process_id.filter(|pid| *pid != 0), + scratch_bytes: None, + }, + endpoint, + )) })(); result.map_err(|e| { let mut f = self.selected(r.fail(e, None)); diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs index f4d5e88b4a..1a81fc2baf 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs @@ -1,8 +1,6 @@ //! One exact runtime observation, structurally decoded without retaining Arguments. -use super::{ - json::{once, Json}, - Docker, -}; +use super::Docker; +use crate::backend::json::{once, Json}; use crate::{ContainerId, ExecId, ExecInspection, RuntimeError, WireFailure}; impl Docker { /// Reads one coherent runtime snapshot. PID is diagnostic, never a signal target. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs index e0eedb87b1..e02b4a03b2 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs @@ -7,13 +7,14 @@ mod endpoint; mod headers; mod http; mod inspect; -mod json; mod listener; mod mux; +mod provider; +mod provider_exec; +mod provider_failure; mod request; mod socket; mod streams; -mod strings; mod topology; mod version; pub use archive::{UploadFailure, UploadWork}; @@ -22,6 +23,9 @@ pub use container_types::{ }; pub use http::Docker; pub use listener::{ListenerFailure, ListenerObservation}; +pub use provider::{DockerInstance, DockerProvider}; +pub(crate) use provider_exec::run as run_command; +pub use provider_failure::ProviderFailure; pub use request::{ExecCreateFailure, ExecCreated, ExecRequest}; pub use streams::{ Attached, ExecHandle, InputCloseFailure, InputEnd, OutputFailure, OutputProgress, RuntimeInput, diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs new file mode 100644 index 0000000000..d4171e4002 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs @@ -0,0 +1,124 @@ +//! Docker's existing one-attempt runtime as a concrete backend. +use super::{Docker, Sandbox, SandboxRequest}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::fs::File; + +/// Negotiation and all transfers use the supplied client's original policy and version cache. +#[derive(Clone, Debug)] +pub struct DockerProvider { + docker: Docker, +} +/// The original container owner, including its exact attempted/acknowledged lifecycle state. +pub type DockerInstance = Sandbox; +impl DockerProvider { + /// Negotiation and all transfers use the supplied client's original policy and version cache. + pub fn new(docker: Docker) -> Self { + Self { docker } + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "docker", + isolation: IsolationModel::Container, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let mut phase = LaunchPhase::Validate; + let mut instance = None; + let result = (|| { + let ProviderLaunchOptions::Docker { + image, + store_volume, + port, + connect_host, + } = request.options + else { + return Err(RuntimeError::Unsupported( + "Docker requires Docker launch options", + )); + }; + if let Some(host) = &connect_host { + if host.is_empty() || host.contains(['\0', ' ', '\t', '\r', '\n']) { + return Err(RuntimeError::Protocol("Docker connect_host format")); + } + } + let deployment = SandboxRequest { + image, + store_volume, + port, + labels: request.labels, + }; + deployment.check()?; + if request.startup_timeout.is_zero() + || request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + { + return Err(RuntimeError::Protocol("Docker startup wait/private config")); + } + let docker = self.docker.clone().naming(&request.name)?; + // Opening the executable precedes any Engine effect, as in the original API route. + let mut file = File::open(request.daemon_executable)?; + phase = LaunchPhase::Create; + instance = + Some(docker.create_sandbox(deployment).map_err(|f| { + RuntimeError::Docker(Box::new(super::ProviderFailure::Create(f))) + })?); + let owner = instance.as_mut().expect("acknowledged container"); + owner + .upload(&mut file, &request.daemon_config) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Upload(f))))?; + phase = LaunchPhase::Start; + owner + .start() + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Start(f))))?; + phase = LaunchPhase::Listener; + owner + .wait_listener(request.startup_timeout) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Listener(f))))?; + Ok(()) + })(); + match result { + Ok(()) => Ok(instance.expect("launched")), + Err(cause) => Err(Box::new(LaunchFailure { + phase, + instance: instance + .map(|sandbox| Box::new(crate::backend::Sandbox::Docker(sandbox))), + cause, + })), + } + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + Sandbox::reopen(self.docker.clone(), id, record.clone().into()) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + let request: SandboxRequest = record.clone().into(); + request.check().map_err(WireFailure::local)?; + Ok(Sandbox::stopped(self.docker.clone(), id, request)) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs new file mode 100644 index 0000000000..59a415d3a5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs @@ -0,0 +1,100 @@ +//! Ordinary noninteractive provider execution, with streamed output and exact failure custody. +use super::{ProviderFailure, Sandbox}; +use crate::{provider::ExecExit, ExecRequest, RuntimeError, WireFailure}; +use std::io::Write; +fn failure(container: crate::ContainerId, cause: ProviderFailure) -> Box { + // Scalar facts describe only the deciding exchange. Owned pending bytes, + // diagnostics and fence errors remain in the original nested receipt. + let wire = match &cause { + ProviderFailure::ExecCreate(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecStart(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecInspect { failure, .. } => Some(failure.as_ref()), + _ => None, + }; + let mut outer = WireFailure::local(RuntimeError::Protocol("provider execution")); + if let Some(wire) = wire { + outer.attempted = wire.attempted; + outer.sent_bytes = wire.sent_bytes; + outer.request_header_bytes = wire.request_header_bytes; + outer.sent_body_bytes = wire.sent_body_bytes; + outer.status = wire.status; + outer.observed_exec = wire.observed_exec; + outer.observed_container = wire.observed_container; + outer.requested_exec = wire.requested_exec; + } else { + // Attachment/splitting, output and exit observation follow an attempted Start. + outer.attempted = true; + } + outer.requested_container = Some(container); + outer.requested_exec = outer.requested_exec.or_else(|| match &cause { + ProviderFailure::ExecStart(f) => Some(f.created.identity().1), + ProviderFailure::ExecOutput { handle, .. } + | ProviderFailure::ExecInspect { handle, .. } + | ProviderFailure::ExecExitUnknown { handle, .. } => Some(handle.identity().1), + _ => None, + }); + outer.cause = RuntimeError::Docker(Box::new(cause)); + outer +} +pub(crate) fn run( + sandbox: &Sandbox, + request: &ExecRequest, + mut stdout: &mut (dyn Write + Send), + mut stderr: &mut (dyn Write + Send), +) -> Result> { + if request.stdin { + return Err( + sandbox.invalid("provider run has no stdin source; use ordinary Docker streams") + ); + } + let id = sandbox.identity(); + let created = sandbox + .create_exec(request.clone()) + .map_err(|f| failure(id, ProviderFailure::ExecCreate(f)))?; + let attached = created + .start() + .map_err(|f| failure(id, ProviderFailure::ExecStart(f)))?; + let (input, output, handle) = attached + .into_parts() + .map_err(|f| failure(id, ProviderFailure::ExecSplit(f)))?; + // Stdin was disabled in the creation request; dropping it sends no cancellation. + drop(input); + let progress = match output.copy_to(&mut stdout, &mut stderr) { + Ok(progress) => progress, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecOutput { failure: f, handle }, + )) + } + }; + let inspection = match handle.inspect() { + Ok(inspection) => inspection, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + )) + } + }; + let Some(code) = inspection.known_root_exit() else { + return Err(failure( + id, + ProviderFailure::ExecExitUnknown { + inspection, + handle, + progress, + }, + )); + }; + Ok(ExecExit { + code, + pid: inspection.pid, + stdout_bytes: progress.stdout, + stderr_bytes: progress.stderr, + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs new file mode 100644 index 0000000000..c97c9a596d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs @@ -0,0 +1,32 @@ +//! Original Docker stage failures retained across the provider's common error port. +use super::{ + ExecCreateFailure, ExecHandle, ListenerFailure, OutputFailure, OutputProgress, + SandboxCreateFailure, SplitFailure, StartFailure, UploadFailure, +}; +use crate::{ExecInspection, WireFailure}; +/// No failed creation/Start/stream owner is reconstructed, replayed or silently cleaned up. +#[derive(Debug)] +pub enum ProviderFailure { + Create(Box), + Upload(Box), + Start(Box), + Listener(Box), + ExecCreate(Box), + ExecStart(Box), + ExecSplit(Box), + ExecOutput { + failure: Box, + handle: ExecHandle, + }, + ExecInspect { + failure: Box, + handle: ExecHandle, + progress: OutputProgress, + }, + /// EOF is not proof of process exit. The original handle permits explicit observation. + ExecExitUnknown { + inspection: ExecInspection, + handle: ExecHandle, + progress: OutputProgress, + }, +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs index b7622d7efc..5b283c9c85 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs @@ -1,58 +1,11 @@ //! One ordinary non-TTY Exec creation; exact acknowledged runtime custody. -use super::{ - json::{once, Json}, - strings::quoted, - Docker, -}; -use crate::{CommandIdentity, ContainerId, ExecId, RuntimeError, WireFailure}; -use std::{ - fmt, - io::{self, Write}, -}; -/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. -#[derive(Clone)] -pub struct ExecRequest { - pub identity: CommandIdentity, - pub arguments: Vec, - pub environment: Vec, - pub directory: String, - pub stdin: bool, -} -impl fmt::Debug for ExecRequest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ExecRequest") - .field("identity", &self.identity) - .field("arguments_count", &self.arguments.len()) - .field("environment_count", &self.environment.len()) - .field("directory", &self.directory) - .field("stdin", &self.stdin) - .finish() - } -} +use super::Docker; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; +pub use crate::ExecRequest; +use crate::{ContainerId, ExecId, RuntimeError, WireFailure}; +use std::io::{self, Write}; impl ExecRequest { - fn check(&self) -> Result<(), RuntimeError> { - self.identity.check()?; - if self.arguments.is_empty() - || self.arguments[0].is_empty() - || !std::path::Path::new(&self.directory).is_absolute() - || self - .arguments - .iter() - .chain(&self.environment) - .chain(std::iter::once(&self.directory)) - .any(|v| v.as_bytes().contains(&0)) - { - return Err(RuntimeError::Protocol("ordinary command fields")); - } - if self - .environment - .iter() - .any(|v| !v.contains('=') || v.starts_with('=')) - { - return Err(RuntimeError::Protocol("ordinary environment field")); - } - Ok(()) - } fn encode(&self, out: &mut dyn Write) -> io::Result<()> { out.write_all(b"{\"AttachStdout\":true,\"AttachStderr\":true,\"AttachStdin\":")?; out.write_all(if self.stdin { b"true" } else { b"false" })?; diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs index 9ee9ae628d..3626ab3bbe 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs @@ -1,8 +1,6 @@ //! Exact observed daemon identity, security and kernel-access settings, and volume placement. -use super::{ - json::{once, Json}, - SandboxRequest, -}; +use super::SandboxRequest; +use crate::backend::json::{once, Json}; use crate::RuntimeError; use std::io::Read; /// The container's whole system-call filter. It denies creation of a user diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs index ae3504b098..973623a478 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs @@ -1,9 +1,9 @@ //! The Engine API version a client speaks: the daemon's, up to the newest this client targets. use super::{ http::{failure, Mode}, - json::{once, Json}, Docker, }; +use crate::backend::json::{once, Json}; use crate::{ErrorDiagnostic, RuntimeError, WireFailure}; /// The oldest Engine API minor version (1.N) this client speaks: Engine 28.0. /// Before it, uploading the daemon fails (`getent unable to find entry "0:0"`: diff --git a/engine/crates/layerfs-sandbox/src/backend/identity.rs b/engine/crates/layerfs-sandbox/src/backend/identity.rs new file mode 100644 index 0000000000..8e1c174c73 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/identity.rs @@ -0,0 +1,16 @@ +//! A fresh random identity in the Engine's form (64 lowercase hex, not all +//! zero), so records, names and the cgroup directory treat a local Sandbox +//! like a container. +use crate::{ContainerId, RuntimeError}; +use std::{fs::File, io::Read}; + +pub(crate) fn random() -> Result { + let mut bytes = [0u8; 32]; + File::open("/dev/urandom")?.read_exact(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let text: String = (bytes.iter()) + .flat_map(|b| [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]]) + .map(char::from) + .collect(); + ContainerId::parse(&text) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/json.rs b/engine/crates/layerfs-sandbox/src/backend/json.rs similarity index 100% rename from engine/crates/layerfs-sandbox/src/backend/docker/json.rs rename to engine/crates/layerfs-sandbox/src/backend/json.rs diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs new file mode 100644 index 0000000000..46c121526d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs @@ -0,0 +1,7 @@ +//! Linux platform facts for the native runtime: the daemon runs as root and +//! serves FUSE, so its process facts come from `/proc` and its mounts are +//! detached lazily. Only facts live here; the Sandbox lifecycle is shared +//! with macOS in [`super::native`]. +pub(crate) mod mounts; +mod process; +pub(crate) use process::{boot, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs new file mode 100644 index 0000000000..35f67d00ed --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs @@ -0,0 +1,20 @@ +//! Linux mount table and lazy detach slot; implemented by the Linux native backend. +use std::{ + io, + path::{Path, PathBuf}, +}; + +const UNSUPPORTED: &str = "Linux native mount table"; + +pub(crate) fn under(_directory: &Path) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn filesystem(_path: &Path) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + let _ = filesystem(directory); + under(directory).map(drop) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/process.rs b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs new file mode 100644 index 0000000000..53b1661565 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs @@ -0,0 +1,17 @@ +//! Linux `/proc` process facts slot; implemented by the Linux native backend. +use super::super::native::process::Stat; +use std::io; + +const UNSUPPORTED: &str = "Linux native process table"; + +pub(crate) fn boot() -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn stat(_pid: u32) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn processes() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/listener.rs b/engine/crates/layerfs-sandbox/src/backend/listener.rs new file mode 100644 index 0000000000..c8fe639ce1 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/listener.rs @@ -0,0 +1,68 @@ +//! The daemon's startup line: a bounded wait for the loopback address it +//! prints once it listens. Both local backends spawn a daemon whose output is +//! a log file, so both read that same marker from it, each look taking only +//! what was appended since the last one. Nothing is retried: the wait ends at +//! the marker, at the daemon's exit, or at the caller's deadline. +use crate::RuntimeError; +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom}, + path::Path, + time::{Duration, Instant}, +}; + +/// The daemon's startup line; the address follows it. +pub(crate) const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; +/// Pause between two reads of a starting daemon's output. +const PAUSE: Duration = Duration::from_millis(10); + +/// Waits until `log` names the daemon's loopback address, the daemon is no +/// longer alive (as `alive` reports), or `wait` ends. +pub(crate) fn await_listener( + log: &Path, + wait: Duration, + alive: &dyn Fn() -> bool, +) -> Result { + let deadline = Instant::now() + wait; + let mut file = File::open(log)?; + let mut offset = 0u64; + let mut line = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + file.seek(SeekFrom::Start(offset))?; + loop { + let n = file.read(&mut chunk)?; + if n == 0 { + break; + } + offset += n as u64; + for &byte in &chunk[..n] { + if byte != b'\n' { + // a startup line is short; a longer one is not the marker + if line.len() < 256 { + line.push(byte); + } + continue; + } + if let Some(address) = line.strip_prefix(MARKER) { + let address = std::str::from_utf8(address) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|a| a.ip().is_loopback() && a.port() != 0) + .ok_or(RuntimeError::Protocol("local daemon loopback listener"))?; + return Ok(address.port()); + } + line.clear(); + } + } + if !alive() { + return Err(RuntimeError::Protocol( + "local daemon exited before its listener line", + )); + } + if Instant::now() >= deadline { + return Err(io::Error::from(io::ErrorKind::TimedOut).into()); + } + std::thread::sleep(PAUSE); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs new file mode 100644 index 0000000000..99ba3e4a52 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs @@ -0,0 +1,277 @@ +//! macOS facts the native runtime reads from `/proc` on Linux: a process's +//! state, group and start time (`proc_pidinfo`, a system call: `ps` is setuid +//! root, which a Seatbelt-sandboxed caller may not execute), this boot's +//! identity (`sysctl`), the mount table (`mount`) and a forced unmount +//! (`umount -f`). Each tool is one run with a fixed path and the C locale, +//! read once; nothing is retried. +use crate::backend::native::process::Stat; +use nix::{ + fcntl::{fcntl, FcntlArg, OFlag}, + libc, +}; +use std::{ + io::{self, Read}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, +}; + +const SYSCTL: &str = "/usr/sbin/sysctl"; +const MOUNT: &str = "/sbin/mount"; +const UMOUNT: &str = "/sbin/umount"; +const UNMOUNT_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const OUTPUT: usize = 2048; + +/// One tool's standard output, or its failure with what it said. +fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { + let out = Command::new(program) + .args(args) + .env_clear() + .env("LC_ALL", "C") + .env("PATH", "/usr/bin:/bin:/usr/sbin:/sbin") + .stdin(Stdio::null()) + .output()?; + Ok(( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + )) +} + +fn invalid(what: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, what) +} + +/// State, process group and start time of `pid` from the kernel +/// (`proc_pidinfo`, `PROC_PIDTBSDINFO`); `NotFound` when no such process +/// exists. Not `ps`: it is setuid root, and a sandboxed caller (dsh's +/// Seatbelt profile for agents' bash) may not execute it. +#[allow(unsafe_code)] +pub(crate) fn stat(pid: u32) -> io::Result { + let id = i32::try_from(pid).map_err(|_| invalid("process id width"))?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: the buffer is a proc_bsdinfo of exactly `size` bytes. + let got = unsafe { + libc::proc_pidinfo( + id, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if got <= 0 { + let error = io::Error::last_os_error(); + return Err(match error.raw_os_error() { + Some(libc::ESRCH) | Some(0) | None => io::Error::from(io::ErrorKind::NotFound), + _ => error, + }); + } + if got != size { + return Err(invalid("proc_bsdinfo size")); + } + // SAFETY: the kernel filled all `size` bytes. + let info = unsafe { info.assume_init() }; + if info.pbi_pid != pid { + return Err(invalid("proc_bsdinfo pid")); + } + Ok(from_info(&info)) +} + +/// `ps`'s state letters for `p_stat`, and the start time in microseconds: +/// equal for the same process, different for another under the same id. +fn from_info(info: &libc::proc_bsdinfo) -> Stat { + let state = match info.pbi_status { + 1 => b'I', + 2 => b'R', + 3 => b'S', + 4 => b'T', + libc::SZOMB => b'Z', + _ => b'?', + }; + Stat { + state, + group: info.pbi_pgid, + start: info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec, + } +} + +/// Every process now, with its stat (`proc_listallpids`); a process that +/// ends or is not readable between the listing and its read is left out. +#[allow(unsafe_code)] +pub(crate) fn processes() -> io::Result> { + // SAFETY: a null buffer asks only for the count. + let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + let mut pids = vec![0 as libc::pid_t; count as usize + 64]; + let bytes = (pids.len() * std::mem::size_of::()) as libc::c_int; + // SAFETY: `pids` holds `bytes` bytes of pid_t. + let listed = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + if listed < 0 { + return Err(io::Error::last_os_error()); + } + pids.truncate((listed as usize).min(pids.len())); + Ok(pids + .into_iter() + .filter(|pid| *pid > 0) + .filter_map(|pid| { + let pid = pid as u32; + stat(pid).ok().map(|stat| (pid, stat)) + }) + .collect()) +} + +/// This boot's identity as a number, read once per process: +/// `kern.bootsessionuuid`, else `kern.boottime` (seconds and microseconds). +pub(crate) fn boot() -> io::Result { + static BOOT: OnceLock = OnceLock::new(); + if let Some(boot) = BOOT.get() { + return Ok(*boot); + } + let uuid = output(SYSCTL, &["-n", "kern.bootsessionuuid"]) + .ok() + .and_then(|(ran, text)| if ran { uuid(&text) } else { None }); + let boot = match uuid { + Some(boot) => boot, + None => { + let (_, text) = output(SYSCTL, &["-n", "kern.boottime"])?; + boottime(&text).ok_or_else(|| invalid("kern.boottime"))? + } + }; + Ok(*BOOT.get_or_init(|| boot)) +} + +fn uuid(text: &str) -> Option { + let hex: String = text.trim().chars().filter(|c| *c != '-').collect(); + (hex.len() == 32) + .then(|| u128::from_str_radix(&hex, 16).ok()) + .flatten() +} + +/// `{ sec = 1729500000, usec = 123456 } Tue Oct 21 ...` as sec * 10^6 + usec. +fn boottime(text: &str) -> Option { + let field = |name: &str| -> Option { + let after = text.split(name).nth(1)?; + let digits: String = after + .trim_start_matches([' ', '=']) + .chars() + .take_while(char::is_ascii_digit) + .collect(); + digits.parse().ok() + }; + Some(field("sec")? * 1_000_000 + field("usec")?) +} + +/// Every mount point now, from `mount`'s ` on ()`. +pub(crate) fn mount_points() -> io::Result> { + let (ran, text) = output(MOUNT, &[])?; + if !ran { + return Err(io::Error::other("mount listing failed")); + } + Ok(text.lines().filter_map(mount_point).collect()) +} + +fn mount_point(line: &str) -> Option { + let (_, rest) = line.split_once(" on ")?; + let (point, _) = rest.rsplit_once(" (")?; + Some(PathBuf::from(point)) +} + +/// Forces the mount at `point` off once; a point no longer mounted is done. +pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { + static RETAINED: Mutex> = Mutex::new(Vec::new()); + { + let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let mut child = Command::new(UMOUNT) + .arg("-f") + .arg(point) + .env_clear() + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stderr = child.stderr.take().and_then(|pipe| { + // If nonblocking setup fails, drop the pipe rather than risk an + // unbounded read while stopping a Sandbox. + fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)) + .ok() + .map(|_| pipe) + }); + let mut output = Vec::new(); + let deadline = Instant::now() + UNMOUNT_WAIT; + let status = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) => { + break Err(io::Error::new( + io::ErrorKind::TimedOut, + "umount exceeded 60s", + )) + } + Err(error) => break Err(error), + } + }; + let status = match status { + Ok(status) => status, + Err(error) => { + let pid = child.id(); + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) | Err(_) => break false, + } + }; + let disposition = if reaped { + "exit reaped" + } else { + RETAINED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; child custody retained (possible kernel wait)" + }; + return Err(io::Error::new( + error.kind(), + format!( + "umount -f {}: {error}; pid={pid}; KILL={kill:?}; {disposition}: {}", + point.display(), + String::from_utf8_lossy(&output).trim() + ), + )); + } + }; + unmount_output(&mut stderr, &mut output); + if status.success() || !mount_points()?.iter().any(|p| p == point) { + return Ok(()); + } + Err(io::Error::other(format!( + "umount -f {}: {status}: {}", + point.display(), + String::from_utf8_lossy(&output).trim() + ))) +} + +fn unmount_output(stderr: &mut Option, output: &mut Vec) { + let Some(pipe) = stderr else { return }; + let mut bytes = [0; OUTPUT]; + if let Ok(count) = pipe.read(&mut bytes) { + output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs new file mode 100644 index 0000000000..f9bf26868c --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs @@ -0,0 +1,7 @@ +//! macOS platform facts for the native runtime: the daemon serves loopback +//! NFS as this user, so its process facts come from `proc_pidinfo`/`sysctl` +//! and its mounts from `mount`/`umount -f`. Only facts live here; the Sandbox +//! lifecycle is shared with Linux in [`super::native`]. +mod darwin; +pub(crate) mod mounts; +pub(crate) use darwin::{boot, force_unmount, mount_points, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs new file mode 100644 index 0000000000..23547afae8 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs @@ -0,0 +1,33 @@ +//! Kernel mounts left under a native Sandbox's directory once its daemon is +//! gone. A container's mounts end with its namespace; this daemon's stay in +//! the one kernel ("transport endpoint is not connected") until detached. The +//! table is `mount`'s output and a mount is forced off with `umount -f`. +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// Mount points at or under `directory`, deepest first, from this +/// process's mount table. +pub(crate) fn under(directory: &Path) -> io::Result> { + let mut found: Vec = (super::mount_points()?.into_iter()) + .filter(|point| point.starts_with(directory)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + +/// Unmounts every mount under `directory` once with `umount -f` (a dead +/// NFS export's mount is forced off; macOS has no lazy detach), then checks +/// that none is left. +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + for point in under(directory)? { + super::force_unmount(&point)?; + } + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::other("a mount under the local Sandbox remains")) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/mod.rs b/engine/crates/layerfs-sandbox/src/backend/mod.rs index 509d7e29fa..602a842926 100644 --- a/engine/crates/layerfs-sandbox/src/backend/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,2 +1,20 @@ -//! Concrete ordinary runtime providers. +//! Concrete ordinary runtime backends and the enum that selects one. pub mod docker; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod identity; +mod json; +#[cfg(target_os = "linux")] +pub(crate) mod linux; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod listener; +#[cfg(target_os = "macos")] +pub(crate) mod macos; +#[cfg(target_os = "linux")] +pub mod namespaces; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub mod native; +pub mod runtime; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod streams; +mod strings; +pub use runtime::{Runtime, Sandbox}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs new file mode 100644 index 0000000000..95e8e27e0b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs @@ -0,0 +1,6 @@ +//! Linux namespace Sandboxes slot; implemented by the namespaces backend. +mod provider; +mod sandbox; + +pub use provider::NamespaceProvider; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs new file mode 100644 index 0000000000..b57cd94a16 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs @@ -0,0 +1,74 @@ +//! Linux namespace runtime slot; implemented by the namespaces backend. +use super::sandbox::Sandbox; +use crate::backend::native::{Local, LocalPaths}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +const UNSUPPORTED: &str = "the Linux namespaces Sandbox backend"; + +/// A Linux namespace runtime over one local root directory. +#[derive(Clone, Debug)] +pub struct NamespaceProvider { + local: Local, +} + +impl NamespaceProvider { + /// Refuses with `Unsupported` before any effect in the interface-only build. + pub fn new(root: impl AsRef) -> Result { + let _ = ( + Local::new(root), + crate::backend::json::Json::<&[u8]>::text_into, + ); + Err(RuntimeError::Unsupported(UNSUPPORTED)) + } + + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + + /// What this backend supports. + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "namespaces-linux", + isolation: IsolationModel::MicroVm, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + + /// Starts one Sandbox once; a failed start keeps the acknowledged owner. + pub fn launch(&self, _request: LaunchRequest) -> Result> { + Err(Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(UNSUPPORTED), + })) + } + + /// Reopen starts nothing: it checks the recorded init process, its cgroup + /// and its listener. + pub fn reopen( + &self, + _id: ContainerId, + _record: &SandboxRecord, + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + _id: ContainerId, + _record: &SandboxRecord, + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs new file mode 100644 index 0000000000..c2f85776d9 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs @@ -0,0 +1,75 @@ +//! Linux namespace Sandbox handle slot; implemented by the namespaces backend. +use crate::backend::native::LocalProcess; +use crate::provider::{ExecExit, SandboxRecord}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +const UNSUPPORTED: &str = "the Linux namespaces Sandbox backend"; + +/// One acknowledged namespace Sandbox. +#[derive(Debug)] +pub struct Sandbox { + id: ContainerId, +} + +impl Sandbox { + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + PathBuf::new() + } + + /// Its init stage process, once started. + pub fn process(&self) -> Option { + None + } + + /// Its record, naming this backend so another one never reopens it. + pub fn record(&self) -> Option { + None + } + + /// The daemon's endpoint while its init stage runs. + pub fn endpoint(&self) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Runs one command to its end inside the Sandbox's namespaces. + pub fn run( + &self, + _request: &ExecRequest, + _stdout: &mut (dyn Write + Send), + _stderr: &mut (dyn Write + Send), + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Its cgroup owns every process the Sandbox started. + pub fn track(&self, _pid: u32, _start_time: u64) -> Result<(), Box> { + Ok(()) + } + + /// The daemon output's last `lines` lines, keeping only the final `cap` bytes. + pub fn log_tail(&self, _lines: u16, _cap: usize) -> Result, Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Reaps the init stage this process started, if it exited. + pub(crate) fn reap(&self) -> bool { + true + } + + /// Ends the Sandbox once. + pub fn stop(&mut self, _timeout_seconds: u16) -> Result<(), Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Deletes its directory and cgroup once, after a known stop. + pub fn delete(&mut self) -> Result<(), Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/create.rs b/engine/crates/layerfs-sandbox/src/backend/native/create.rs new file mode 100644 index 0000000000..c814869456 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/create.rs @@ -0,0 +1,284 @@ +//! One local daemon start: the Sandbox's private directory, its 0600 setup, +//! one spawn and its listener line. Nothing is retried. +use super::{ + process::LocalProcess, + sandbox::{Sandbox, LOG}, + search::search_path, +}; +use crate::backend::{identity, listener}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::Duration, +}; + +/// A local runtime root: one directory per Sandbox under it. +#[derive(Clone, Debug)] +pub struct Local { + root: PathBuf, + name: Option, + /// Variables the daemon starts with besides `PATH`. + environment: Vec<(String, String)>, +} + +/// Where one Sandbox keeps its parts, all under its directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalPaths { + /// The Sandbox directory, `/`, 0755. + pub directory: PathBuf, + /// Its daemon setup, `config/daemon.setup`, 0600 in a 0700 directory. + pub config: PathBuf, + /// Its daemon overlay template, `overlay/overlay.sqlite`, in a 0700 directory. + pub overlay: PathBuf, + /// Its Workspace mount home, `workspaces`, made 0755 by the daemon. + pub mounts: PathBuf, + /// Its daemon's output, `daemon.log`, 0600. + pub log: PathBuf, + /// Its commands' scratch directory, `tmp`, 1777 like a container's + /// `/tmp`, so what they leave there goes with the Sandbox. + pub tmp: PathBuf, +} + +/// The boundary one failed create reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LocalPhase { + /// Local checks; nothing made. + Validate, + /// The directory, setup and log file. + Create, + /// The daemon spawn. + Start, + /// The wait for the daemon's listener line. + Listener, +} + +/// A failed create. `sandbox` holds what it made (its directory, and its +/// daemon once spawned), for the caller to stop and delete once. +#[derive(Debug)] +pub struct LocalCreateFailure { + pub phase: LocalPhase, + pub sandbox: Option, + pub cause: RuntimeError, +} + +impl Local { + /// Selects `root`: an existing absolute directory owned by this user, + /// writable by it alone and traversable by others, as is every directory + /// above it (commands enter their Workspace below it only after taking + /// another identity, so one closed directory on the way, such as a 0700 + /// `/root`, would fail every command at spawn). + pub fn new(root: impl AsRef) -> Result { + let local = Self::native(root.as_ref())?; + let root = root.as_ref(); + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } + Ok(local) + } + /// Selects `root` for the native route, whose commands run as this user + /// (no other identity): the root's own checks as in [`Self::new`], but + /// the directories above it need not be traversable by others (a macOS + /// home is 0750). + pub fn native(root: impl AsRef) -> Result { + let root = root.as_ref(); + if !root.is_absolute() + || root + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return Err(RuntimeError::Protocol("absolute local runtime root")); + } + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() + || metadata.uid() != nix::unistd::Uid::effective().as_raw() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & 0o001 == 0 + { + return Err(RuntimeError::Protocol( + "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", + )); + } + Ok(Self { + root: root.to_owned(), + name: None, + environment: Vec::new(), + }) + } + /// Starts the next daemon with `environment` besides its search path (the + /// native route's `LAYERFS_KERNEL=nfs` and its mount helpers). Nothing + /// else of this process's environment reaches it. + pub fn environment(mut self, environment: Vec<(String, String)>) -> Self { + self.environment = environment; + self + } + /// Names the next Sandbox's directory `name`, under the Docker name rule + /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at + /// most 128), so it is one plain path component. + pub fn naming(mut self, name: &str) -> Result { + let bytes = name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 128 + || !bytes[0].is_ascii_alphanumeric() + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) + { + return Err(RuntimeError::Protocol("local Sandbox name")); + } + self.name = Some(name.to_owned()); + Ok(self) + } + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self) -> Result { + let name = self + .name + .as_deref() + .ok_or(RuntimeError::Protocol("local Sandbox name"))?; + let directory = self.root.join(name); + Ok(LocalPaths { + config: directory.join("config").join("daemon.setup"), + overlay: directory.join("overlay").join("overlay.sqlite"), + mounts: directory.join("workspaces"), + log: directory.join(LOG), + tmp: directory.join("tmp"), + directory, + }) + } + /// Makes the named Sandbox's directory, writes `config` (the encoded + /// daemon setup naming [`Self::paths`]) 0600, spawns `executable + /// --config ` in its own process group with no inherited + /// environment but the search path, and waits up to `wait` for its + /// listener line. Once anything is made, a failure returns it. + pub fn create( + &self, + executable: &Path, + config: &[u8], + wait: Duration, + labels: Vec<(String, String)>, + ) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(LocalCreateFailure { + phase, + sandbox, + cause, + }) + }; + let paths = self + .paths() + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let checked = (|| { + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("local daemon executable")); + } + if config.is_empty() || config.len() > 8192 || wait.is_zero() { + return Err(RuntimeError::Protocol("local daemon setup/startup wait")); + } + match fs::symlink_metadata(&paths.directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(_) => Err(RuntimeError::Protocol("local Sandbox directory exists")), + } + })(); + checked.map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted(id, paths.directory.clone(), labels); + let log = match prepare(&paths, config) { + Ok(log) => log, + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + }; + let spawned = log.try_clone().and_then(|output| { + Command::new(executable) + .arg("--config") + .arg(&paths.config) + .env_clear() + .env("PATH", search_path()) + .envs(self.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + let process = match LocalProcess::observe(child.id()) { + Ok(process) => process, + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + }; + sandbox.process = Some(process); + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.log, wait, &|| { + sandbox.reap(); + sandbox.process.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } + } +} + +/// The setup and overlay directories (0700), the scratch directory (1777), +/// the setup file (0600, written once) and the empty log file (0600) it +/// returns. +fn prepare(paths: &LocalPaths, config: &[u8]) -> io::Result { + // the process umask may narrow the create mode + fs::set_permissions(&paths.directory, Permissions::from_mode(0o755))?; + for file in [&paths.config, &paths.overlay] { + let parent = file + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "local path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.tmp)?; + fs::set_permissions(&paths.tmp, Permissions::from_mode(0o1777))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + private(&paths.config)?.write_all(config)?; + private(&paths.log) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/exec.rs b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs new file mode 100644 index 0000000000..1478054771 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs @@ -0,0 +1,133 @@ +//! One ordinary command to its end in a local Sandbox: the command identity, +//! the Workspace directory and a search path (on macOS this process's own); +//! nothing else of this process's environment, no shell of its own and no +//! implicit timeout. Each command leads a process group of its own, named in +//! the Sandbox until it is empty, so what it leaves running ends with the +//! Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox, search::search_path}; +use crate::backend::streams; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use nix::sys::signal::{killpg, Signal}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +/// One command's end: its exit code (128 + the signal for one a signal +/// ended, as the Engine reports it), its process id (diagnostic only) and the +/// bytes each stream delivered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalExit { + pub code: i32, + pub pid: u32, + pub stdout: u64, + pub stderr: u64, +} + +impl Sandbox { + /// Runs `request` once while this Sandbox's daemon runs: as its uid/gid + /// with no supplementary groups, in its directory, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. Standard input is not offered. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + self.endpoint()?; + let mut command = Command::new(&request.arguments[0]); + command + .args(&request.arguments[1..]) + .env_clear() + .env("PATH", search_path()) + .current_dir(Path::new(&request.directory)) + // std drops supplementary groups (setgroups 0) when it sets a uid as root + .uid(request.identity.uid) + .gid(request.identity.gid) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + // named before anything is waited for: the leader is not reaped + // before the wait below, so it is observable even once it exited + let named = LocalProcess::observe(pid) + .and_then(|leader| groups::name(self.directory(), leader).map(|()| leader)); + let leader = match named { + Ok(leader) => leader, + Err(error) => { + // an unnamed group would outlive the Sandbox: it ends here, once + if let Ok(group) = process::pid(pid) { + let _ = killpg(group, Signal::SIGKILL); + } + let _ = child.wait(); + let mut failure = self.io(error); + failure.attempted = true; + return Err(failure); + } + }; + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + // a name left behind is harmless: its group is checked again when ended + let _ = groups::forget_if_empty(self.directory(), leader); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("local command exit status"))?; + Ok(LocalExit { + code, + pid, + stdout: out_bytes, + stderr: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/groups.rs b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs new file mode 100644 index 0000000000..6f18869381 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs @@ -0,0 +1,149 @@ +//! The process groups a local Sandbox's processes run in. A container's +//! processes end with it; a local Sandbox's would outlive it on this kernel, +//! so each command starts its own process group, named in the Sandbox's +//! `groups` directory while it may have members, and every named group (and +//! the daemon's own) is killed when the Sandbox stops or is deleted. +//! +//! A group is named by its leader: id, start time and boot. Its id stays the +//! group's while any member lives, so a group whose leader is gone is still +//! known by it; a leader found with another start time means the id was +//! reused and the group is not this Sandbox's. A process that leaves its +//! group (`setsid`) is not followed unless it is named here too +//! ([`super::Sandbox::track`]). +use super::platform; +use super::process::{self, LocalProcess}; +use nix::{fcntl::OFlag, sys::signal::Signal}; +use std::{ + fs::{self, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::Path, + time::{Duration, Instant}, +}; + +/// The directory of a Sandbox that names its groups, one empty file each. +pub(super) const GROUPS: &str = "groups"; +/// Pause between two looks at a killed group. +const PAUSE: Duration = Duration::from_millis(20); + +/// Names the group `leader` leads (its id is the group's), once. +pub(super) fn name(directory: &Path, leader: LocalProcess) -> io::Result<()> { + let groups = directory.join(GROUPS); + // a Sandbox started before groups were named has no directory yet + match fs::DirBuilder::new().mode(0o700).create(&groups) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(groups.join(entry(leader)?)) + .map(drop) +} + +/// Forgets the group `leader` led once no member of it is left. Called after +/// the leader was reaped; a group that still has members stays named. +pub(super) fn forget_if_empty(directory: &Path, leader: LocalProcess) -> io::Result<()> { + match nix::sys::signal::killpg(process::pid(leader.pid)?, None) { + Err(nix::errno::Errno::ESRCH) => {} + _ => return Ok(()), + } + match fs::remove_file(directory.join(GROUPS).join(entry(leader)?)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Kills every group the Sandbox at `directory` named, and the group of +/// `daemon` (its leader), once with KILL, as a container's end kills what +/// ran in it, then waits up to `wait` for their members to be gone. Groups +/// of another boot ended with it. +pub(super) fn end( + directory: &Path, + daemon: Option, + wait: Duration, +) -> io::Result<()> { + let mut leaders: Vec = daemon.into_iter().collect(); + match fs::read_dir(directory.join(GROUPS)) { + Ok(items) => { + for item in items { + let name = item?.file_name(); + let leader = name.to_str().and_then(parse).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "local Sandbox group name") + })?; + leaders.push(leader); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let this_boot = process::boot()?; + leaders.retain(|leader| leader.boot == Some(this_boot)); + let table = processes()?; + let mut killed = Vec::new(); + for leader in leaders { + if !members(&table, leader).is_empty() { + match nix::sys::signal::killpg(process::pid(leader.pid)?, Signal::SIGKILL) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => killed.push(leader), + Err(errno) => return Err(errno.into()), + } + } + } + let deadline = Instant::now() + wait; + loop { + let table = processes()?; + if killed + .iter() + .all(|leader| members(&table, *leader).is_empty()) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::other( + "a process of the local Sandbox still runs after KILL", + )); + } + std::thread::sleep(PAUSE); + } +} + +/// The running members of `leader`'s group: none when the leader's id now +/// names another process (the id was reused, so the group is another's). +fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { + let reused = (table.iter()).any(|(pid, stat)| *pid == leader.pid && stat.start != leader.start); + if reused { + return Vec::new(); + } + (table.iter()) + .filter(|(_, stat)| stat.group == leader.pid && !stat.ended()) + .map(|(pid, _)| *pid) + .collect() +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +fn processes() -> io::Result> { + platform::processes() +} + +/// A group's file name: `--`. +fn entry(leader: LocalProcess) -> io::Result { + let boot = leader + .boot + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "a group leader's boot"))?; + Ok(format!("{boot:032x}-{}-{}", leader.pid, leader.start)) +} + +fn parse(name: &str) -> Option { + let mut parts = name.split('-'); + let boot = parts.next().filter(|b| b.len() == 32)?; + let process = LocalProcess { + boot: Some(u128::from_str_radix(boot, 16).ok()?), + pid: parts.next()?.parse().ok()?, + start: parts.next()?.parse().ok()?, + }; + parts.next().is_none().then_some(process) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/mod.rs b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs new file mode 100644 index 0000000000..1f1fb4ad81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs @@ -0,0 +1,40 @@ +//! Native process runtime: each Sandbox is one daemon process on this kernel, +//! started from a private directory, with no container, image or Engine. +//! +//! It is for a machine that is itself the isolation boundary (a microVM, a +//! CI job, a developer's own Mac). It isolates nothing: the daemon and its +//! commands share this kernel's processes, network, users and filesystem with +//! everything else on it. Commands still run as their own nonroot identity +//! where the platform allows it, and what they leave running ends with the +//! Sandbox. One attempt per operation, as in the Docker runtime; nothing is +//! retried. +//! +//! The lifecycle here is one implementation for both platforms. What differs +//! is read where it lives: [`super::macos`] supplies Darwin process facts, the +//! `mount` table and `umount -f` for the loopback NFS the macOS daemon serves; +//! [`super::linux`] supplies `/proc` facts, the `mountinfo` table and a lazy +//! `MNT_DETACH` for the FUSE mount the root daemon serves. +//! +//! The daemon and its commands start with one search path and nothing else of +//! this process's environment: on Linux a container's, whatever this process +//! has; on macOS this process's own `PATH`, since a native command is this +//! user's own tool (Homebrew's `/opt/homebrew/bin`, `~/.local/bin`, a version +//! manager's `node`) and no image supplies it. +mod create; +mod exec; +mod groups; +pub(crate) mod process; +mod provider; +mod sandbox; +pub(crate) mod search; +#[cfg(target_os = "linux")] +pub(crate) use super::linux as platform; +#[cfg(target_os = "macos")] +pub(crate) use super::macos as platform; +pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; +pub use exec::LocalExit; +pub use process::LocalProcess; +pub use provider::{ + NativeLinuxInstance, NativeLinuxProvider, NativeMacosInstance, NativeMacosProvider, +}; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/native/process.rs b/engine/crates/layerfs-sandbox/src/backend/native/process.rs new file mode 100644 index 0000000000..74d58bf34d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/process.rs @@ -0,0 +1,85 @@ +//! A local process named by its id, kernel start time and boot, so a reused +//! id, or the same id and start time after a reboot or snapshot restore, is +//! never taken for the process that was started. The three facts come from +//! `/proc` on Linux and from `proc_pidinfo`/`sysctl` on macOS (see +//! [`super::macos`]). +pub(crate) use super::platform::{boot, stat}; +use nix::{sys::signal::Signal, unistd::Pid}; +use std::io; + +/// One started process: its id, its start time in clock ticks since boot +/// (`/proc//stat` field 22) and the boot it started in +/// (`/proc/sys/kernel/random/boot_id`). All must match for it to count as +/// alive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalProcess { + pub pid: u32, + pub start: u64, + /// The boot id as a number. `None` only for a record written before boot + /// ids were kept: it is matched by id and start time alone, as it was + /// then, and never signalled. + pub boot: Option, +} + +impl LocalProcess { + /// The process running now under `pid`, read once from this kernel, in + /// this boot. + pub fn observe(pid: u32) -> io::Result { + let stat = stat(pid)?; + Ok(Self { + pid, + start: stat.start, + boot: Some(boot()?), + }) + } + /// True while this exact process exists and has not exited: a zombie + /// (or a dead entry) is not alive, and another process under the same id + /// has another start time or another boot. + pub fn alive(self) -> bool { + let this_boot = self + .boot + .is_none_or(|recorded| boot().ok() == Some(recorded)); + match stat(self.pid) { + Ok(stat) => this_boot && stat.start == self.start && !stat.ended(), + Err(_) => false, + } + } + /// Sends `signal` once, only after this exact process was observed alive. + pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if self.boot.is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "a local process recorded without its boot is never signalled", + )); + } + if !self.alive() { + return Ok(()); + } + match nix::sys::signal::kill(pid(self.pid)?, signal) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), + Err(errno) => Err(errno.into()), + } + } +} + +/// What this kernel says of one process. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Stat { + pub(crate) state: u8, + pub(crate) group: u32, + pub(crate) start: u64, +} + +impl Stat { + /// A zombie or a dead entry: it runs nothing any more. + pub(crate) fn ended(self) -> bool { + matches!(self.state, b'Z' | b'X' | b'x') + } +} + +/// A process id as the kernel's signed type. +pub(super) fn pid(id: u32) -> io::Result { + i32::try_from(id) + .map(Pid::from_raw) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width")) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/provider.rs b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs new file mode 100644 index 0000000000..b5e48d9916 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs @@ -0,0 +1,178 @@ +//! The existing native runtime as a concrete backend; no alternate process engine. +use super::{Local, LocalPhase, LocalProcess, Sandbox}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, LocalRecord, MountDriver, + ProviderCapabilities, ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct NativeProvider { + local: Local, +} +/// Native macOS processes with loopback NFS and the caller's identity. +pub type NativeMacosProvider = NativeProvider; +/// Native Linux processes with FUSE and a separate nonroot command identity. +pub type NativeLinuxProvider = NativeProvider; +/// The existing local owner, using Darwin process facts. +pub type NativeMacosInstance = Sandbox; +/// The existing local owner, using Linux `/proc` process facts. +pub type NativeLinuxInstance = Sandbox; +impl NativeProvider { + /// Refuses other platforms before accessing the root or launching anything. + pub fn new(root: impl AsRef) -> Result { + if (LINUX && !cfg!(target_os = "linux")) || (!LINUX && !cfg!(target_os = "macos")) { + return Err(RuntimeError::Unsupported("native provider platform")); + } + if LINUX && !nix::unistd::Uid::effective().is_root() { + return Err(RuntimeError::Unsupported( + "native Linux provider requires root", + )); + } + Ok(Self { + local: if LINUX { + Local::new(root)? + } else { + Local::native(root)? + }, + }) + } + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: if LINUX { + "native-linux" + } else { + "native-macos" + }, + isolation: IsolationModel::HostProcess, + mount_driver: if LINUX { + MountDriver::Fuse + } else { + MountDriver::LoopbackNfs + }, + requires_image: false, + search_path_policy: if LINUX { + SearchPathPolicy::FixedContainer + } else { + SearchPathPolicy::InheritHost + }, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Local { + command_identity, + mut environment, + } = request.options + else { + return Err(fail(RuntimeError::Unsupported( + "native provider requires local launch options", + ))); + }; + command_identity.check().map_err(fail)?; + if (!LINUX + && (command_identity.uid != nix::unistd::Uid::effective().as_raw() + || command_identity.gid != nix::unistd::Gid::effective().as_raw())) + || environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != if LINUX { "fuse" } else { "nfs" }) + }) + { + return Err(fail(RuntimeError::Protocol( + "native command identity/environment", + ))); + } + environment.retain(|(key, _)| key != "LAYERFS_KERNEL"); + environment.push(( + "LAYERFS_KERNEL".into(), + if LINUX { "fuse" } else { "nfs" }.into(), + )); + let local = self + .local + .clone() + .naming(&request.name) + .map_err(fail)? + .environment(environment); + local + .create( + &request.daemon_executable, + &request.daemon_config, + request.startup_timeout, + request.labels, + ) + .map_err(|failure| { + Box::new(LaunchFailure { + phase: match failure.phase { + LocalPhase::Validate => LaunchPhase::Validate, + LocalPhase::Create => LaunchPhase::Create, + LocalPhase::Start => LaunchPhase::Start, + LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: failure + .sandbox + .map(|sandbox| Box::new(crate::backend::Sandbox::Native(sandbox))), + cause: failure.cause, + }) + }) + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Sandbox::reopen( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + ) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Ok(Sandbox::stopped( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + )) + } +} +fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs new file mode 100644 index 0000000000..8c67717882 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs @@ -0,0 +1,335 @@ +//! One local Sandbox: its directory, its daemon process and its exact +//! attempts. No Drop cleanup or replay, as for a container; a Drop only +//! reaps a daemon this process started that has already exited. A stop +//! returns only once a daemon this process started is reaped, so it never +//! stays a zombie of this process. +use super::{groups, platform, process::LocalProcess}; +use crate::provider::LocalRecord; +use crate::{ContainerId, RuntimeError, WireFailure}; +use nix::sys::signal::Signal; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The daemon's output file in a Sandbox directory, stdout and stderr together. +pub(super) const LOG: &str = "daemon.log"; +/// Pause between two looks at a stopping process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for a killed process to be gone. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged local Sandbox directory and, once started, its daemon. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) directory: PathBuf, + pub(super) labels: Vec<(String, String)>, + pub(super) process: Option, + pub(super) port: u16, + /// The daemon this process started, kept so its exit is reaped here and + /// it does not stay a zombie of this process. `None` after a reopen. + pub(super) child: Mutex>, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + /// Its process groups were killed and found empty (by its stop). + pub(super) ended: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// A directory made by a start that has not spawned its daemon yet. + pub(super) fn unstarted( + id: ContainerId, + directory: PathBuf, + labels: Vec<(String, String)>, + ) -> Self { + Self { + id, + directory, + labels, + process: None, + port: 0, + child: Mutex::new(None), + stop_attempted: false, + stopped: false, + ended: false, + delete_attempted: false, + deleted: false, + } + } + /// Reopens a recorded Sandbox whose daemon still runs: the same process + /// (id and start time) and its loopback port. It starts nothing. + pub fn reopen( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Result> { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + if !owner.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved local Sandbox")); + } + owner.endpoint()?; + Ok(owner) + } + /// A recorded Sandbox that [`Self::reopen`] found stopped + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub fn stopped( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Self { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + /// Its private directory (setup, overlay, Workspace mounts, daemon log). + pub fn directory(&self) -> &Path { + &self.directory + } + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + self.directory.join("tmp") + } + /// Its daemon process, once started. + pub fn process(&self) -> Option { + self.process + } + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let process = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: process.pid, + start_time: process.start, + boot: process.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + /// The daemon's endpoint while that exact process runs. A daemon that + /// exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(process) = self.process else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !process.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then + /// KILL (a daemon this process started has ended once reaped); then + /// KILL to every process group of its commands and its daemon, as a + /// container's end kills what ran in it. A daemon already gone is the + /// same stopped state. Crash teardown, not a graceful filesystem drain. + pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original local Stop admission")); + } + self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = platform::mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } + if let Some(process) = self.process { + self.end( + process, + Signal::SIGTERM, + Duration::from_secs(u64::from(seconds)), + )?; + if !self.gone(process) { + self.end(process, Signal::SIGKILL, KILL_WAIT)?; + } + if process.alive() { + return Err(self.invalid("local daemon still runs after KILL")); + } + } + self.stopped = true; + groups::end(&self.directory, self.process, KILL_WAIT).map_err(|error| self.io(error))?; + self.ended = true; + Ok(()) + } + /// Deletes its directory once, after a known stop: any process left in + /// its groups is killed first (when its stop did not run here, as for a + /// daemon found stopped), and any mount its daemon left is detached, so + /// nothing writes into it or is removed through a mount. The shared + /// Store is outside it and never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.process.is_some() && !self.stopped) { + return Err(self.invalid("original local Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.process.is_some_and(LocalProcess::alive) { + return Err(self.invalid("local daemon runs again; nothing was deleted")); + } + if !self.ended { + groups::end(&self.directory, self.process, KILL_WAIT) + .map_err(|error| self.io(error))?; + self.ended = true; + } + platform::mounts::detach(&self.directory).map_err(|error| self.io(error))?; + match fs::remove_dir_all(&self.directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(self.directory.join(LOG))?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + /// Names the process group `pid` leads or is about to lead: a process + /// started at `start` by a command of this Sandbox in a session of its + /// own (the SDK's `setsid` background process), so it ends with the + /// Sandbox too. One already gone needs nothing. + pub fn track(&self, pid: u32, start: u64) -> Result<(), Box> { + match LocalProcess::observe(pid) { + Ok(process) if process.start == start && process.alive() => { + groups::name(&self.directory, process).map_err(|error| self.io(error)) + } + _ => Ok(()), + } + } + /// Reaps the daemon this process started, if it exited. True when no + /// daemon this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + /// The daemon has ended and, if this process started it, is reaped. Its + /// main thread can end (a zombie id, so not alive) while its other + /// threads still run and it cannot be reaped yet: not gone. + fn gone(&self, process: LocalProcess) -> bool { + self.reap() && !process.alive() + } + /// Sends `signal` once and waits up to `wait` for the process to be gone. + /// A KILL also goes to the daemon this process started and has not + /// reaped: its id cannot be reused until reaped, so it is still the one + /// started, though its main thread may have ended already. + fn end( + &self, + process: LocalProcess, + signal: Signal, + wait: Duration, + ) -> Result<(), Box> { + process.signal(signal).map_err(|error| self.io(error))?; + if signal == Signal::SIGKILL { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + let _ = started.kill(); + } + } + let deadline = Instant::now() + wait; + loop { + if self.gone(process) || Instant::now() >= deadline { + return Ok(()); + } + std::thread::sleep(PAUSE); + } + } + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// A daemon this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running daemon + /// is left alone: stopping it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/search.rs b/engine/crates/layerfs-sandbox/src/backend/native/search.rs new file mode 100644 index 0000000000..3db862f089 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/search.rs @@ -0,0 +1,26 @@ +//! The search path a local daemon and its commands start with: a container's +//! on Linux, this process's own on macOS (the native route). +use std::ffi::OsString; + +/// A container's search path: what a local daemon and its commands start with +/// on Linux, where the local runtime stands in for a container. +const CONTAINER_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on Linux [`CONTAINER_PATH`], whatever this +/// process has. +#[cfg(not(target_os = "macos"))] +pub(crate) fn search_path() -> OsString { + OsString::from(CONTAINER_PATH) +} + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on macOS this process's own `PATH`; without +/// one, or an empty one, [`CONTAINER_PATH`] and Homebrew's `/opt/homebrew/bin`. +#[cfg(target_os = "macos")] +pub(crate) fn search_path() -> OsString { + match std::env::var_os("PATH") { + Some(path) if !path.is_empty() => path, + _ => OsString::from(format!("{CONTAINER_PATH}:/opt/homebrew/bin")), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/runtime.rs b/engine/crates/layerfs-sandbox/src/backend/runtime.rs new file mode 100644 index 0000000000..43ed8208a0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/runtime.rs @@ -0,0 +1,307 @@ +//! The concrete runtimes a Sandbox can run on: one enum names the deployment's +//! choice, and [`Sandbox`] is the launched owner over it. There is no provider +//! trait; each backend is a concrete type and dispatch is a `match` here. +use super::docker::{self, DockerProvider}; +#[cfg(target_os = "linux")] +use super::namespaces::{self, NamespaceProvider}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use super::native::{self, NativeLinuxProvider, NativeMacosProvider}; +use crate::provider::{ + ExecExit, LaunchFailure, LaunchRequest, LocalRecord, ProviderCapabilities, SandboxInspection, + SandboxRecord, +}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +/// What Windows would need and does not have yet: no host runtime exists. +const WINDOWS: &str = "windows native Sandbox runtime"; + +/// One deployment's runtime choice, kept for every Sandbox it starts. +/// +/// A Sandbox's record names the variant that made it; a platform without a +/// runtime is refused before any effect, never replaced by another backend. +#[derive(Clone, Debug)] +pub enum Runtime { + /// A local Docker Engine container; the default. + Docker(DockerProvider), + /// A local daemon process serving loopback NFS as this user (macOS). + #[cfg(any(target_os = "linux", target_os = "macos"))] + MacosNative(NativeMacosProvider), + /// A root local daemon process serving FUSE, commands as a nonroot + /// identity (Linux). + #[cfg(any(target_os = "linux", target_os = "macos"))] + LinuxNative(NativeLinuxProvider), + /// A root daemon in its own mount, PID and IPC namespaces and cgroup, + /// commands entering them as a nonroot identity (Linux). + #[cfg(target_os = "linux")] + Namespaces(NamespaceProvider), + /// Placeholder: selecting Windows refuses before any effect. + Windows, +} + +impl Runtime { + pub fn capabilities(&self) -> Result { + match self { + Self::Docker(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => Ok(provider.capabilities()), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => Ok(provider.capabilities()), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// Local resource ownership, checked by API before encoding daemon topology. + /// Nonlocal providers return None. No resource is created by this observation. + pub fn local_directory(&self, name: &str) -> Result, RuntimeError> { + match self { + Self::Docker(_) => Ok(None), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// The named native Sandbox's paths, for its daemon setup. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn native_paths(&self, name: &str) -> Result { + match self { + Self::MacosNative(provider) => provider.paths(name), + Self::LinuxNative(provider) => provider.paths(name), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name), + Self::Docker(_) | Self::Windows => Err(RuntimeError::Unsupported( + "native Sandbox paths need a native runtime", + )), + } + } + + /// Starts one Sandbox once; a failed launch keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + match self { + Self::Docker(provider) => provider.launch(request).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .launch(request) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(Box::new(LaunchFailure { + phase: crate::provider::LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(WINDOWS), + })), + } + } + + /// Reopen starts nothing and checks the recorded deployment or process identity. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen_stopped(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen_stopped(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } +} + +/// The launched owner of one Sandbox, on the runtime that started it. +#[derive(Debug)] +pub enum Sandbox { + Docker(docker::Sandbox), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Native(native::Sandbox), + #[cfg(target_os = "linux")] + Namespaces(Box), +} + +impl Sandbox { + pub fn identity(&self) -> ContainerId { + match self { + Self::Docker(sandbox) => sandbox.identity(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.identity(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.identity(), + } + } + + pub fn endpoint(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.endpoint(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.endpoint(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.endpoint(), + } + } + + /// One command to its end, its streams copied to the sinks as they arrive. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + match self { + Self::Docker(sandbox) => docker::run_command(sandbox, request, stdout, stderr), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + let exit = sandbox.run(request, stdout, stderr)?; + Ok(ExecExit { + code: exit.code, + pid: Some(exit.pid), + stdout_bytes: exit.stdout, + stderr_bytes: exit.stderr, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.run(request, stdout, stderr), + } + } + + /// Names a background process a command started in a session of its own, + /// so it ends with the Sandbox as a container's descendants do. + pub fn track(&self, pid: u32, start_time: u64) -> Result<(), Box> { + match self { + // Container teardown owns all descendants; no host process is adopted. + Self::Docker(_) => Ok(()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.track(pid, start_time), + // its cgroup owns every process the Sandbox started, whatever + // session it left, so nothing needs naming + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.track(pid, start_time), + } + } + + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + match self { + Self::Docker(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.log_tail(lines, cap), + } + } + + pub fn inspect(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.inspect(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + } + } + + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.stop(timeout_seconds), + } + } + + pub fn delete(&mut self) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.delete(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.delete(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.delete(), + } + } + + /// Where its commands keep scratch files: `/tmp` in a container, the + /// Sandbox's own directory for a native one, deleted with it. + pub fn scratch_directory(&self) -> String { + match self { + Self::Docker(_) => "/tmp".into(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.tmp().display().to_string(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.tmp().display().to_string(), + } + } + + /// Physical facts for the binding's existing record format, when this is native. + pub fn local_record(&self) -> Option { + match self { + Self::Docker(_) => None, + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.local_record(), + #[cfg(target_os = "linux")] + Self::Namespaces(_) => None, + } + } + + /// Persisted deployment facts for reopening this runtime instance. + pub fn record(&self) -> Option { + match self { + Self::Docker(sandbox) => Some(SandboxRecord::Docker(sandbox.request().clone().into())), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(_) => self.local_record().map(SandboxRecord::Local), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.record(), + } + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/streams.rs b/engine/crates/layerfs-sandbox/src/backend/streams.rs new file mode 100644 index 0000000000..6c6e011a17 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/streams.rs @@ -0,0 +1,23 @@ +//! An ordinary command's streams, copied to the caller's sinks as they +//! arrive. Both local backends read the same two pipes and count only what +//! each sink accepted. +use std::io::{self, Read, Write}; + +/// Copies one stream to its sink until end of stream; the bytes delivered. +pub(crate) fn copy(source: Option, sink: &mut (dyn Write + Send)) -> io::Result { + let Some(mut source) = source else { + return Ok(0); + }; + let mut total = 0u64; + let mut window = [0u8; 8192]; + loop { + let n = match source.read(&mut window) { + Ok(0) => return Ok(total), + Ok(n) => n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + sink.write_all(&window[..n])?; + total = total.saturating_add(n as u64); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs b/engine/crates/layerfs-sandbox/src/backend/strings.rs similarity index 86% rename from engine/crates/layerfs-sandbox/src/backend/docker/strings.rs rename to engine/crates/layerfs-sandbox/src/backend/strings.rs index 1d2afc32e8..fb22cf7c79 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs +++ b/engine/crates/layerfs-sandbox/src/backend/strings.rs @@ -41,6 +41,27 @@ impl Json { self.string_each(|c| text.push(c, capture))?; Ok(text) } + /// Reads a string value into `out` as UTF-8, refusing one longer than + /// `max`: the Engine's selected fields are short, and the longest string + /// this crate reads is an image archive's path. Its only caller unpacks + /// images, which this crate does on Linux only. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn text_into(&mut self, out: &mut Vec, max: usize) -> Result<(), RuntimeError> { + let mut too_long = false; + self.string_each(|c| { + let mut encoded = [0; 4]; + let bytes = c.encode_utf8(&mut encoded).as_bytes(); + if out.len() + bytes.len() > max { + too_long = true; + } else { + out.extend_from_slice(bytes); + } + })?; + if too_long { + return Err(RuntimeError::Protocol("JSON string length")); + } + Ok(()) + } pub fn string_equals(&mut self, expected: &str) -> Result { let mut expected = expected.chars(); let mut equal = true; diff --git a/engine/crates/layerfs-sandbox/src/backend/windows/README.md b/engine/crates/layerfs-sandbox/src/backend/windows/README.md new file mode 100644 index 0000000000..b667620c81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/windows/README.md @@ -0,0 +1,49 @@ +# Windows：占位（尚未实现） + +状态：占位。选到 `Runtime::Windows` 时返回既有的“不支持此平台”错误 +（`RuntimeError::Unsupported`），不 panic、不新建公开错误码、不产生任何副作用。 + +## 现在有什么 + +- `layerfs_sandbox::Runtime` 的四个取值：`Docker`、`MacosNative`、 + `LinuxNative`、`Windows`。前三个已有后端，Windows 只有这个变体。 +- `Runtime::{capabilities, local_directory, launch, reopen, reopen_stopped}` + 在 Windows 上全部返回 `Unsupported`；`Sandbox` 句柄没有 Windows 分支， + 因为 Windows 上不可能启动出实例。 +- `layerfs-sandbox` 的 `backend` 只在 unix 下编译，`layerfs-node` 也只用 + unix 系统调用，所以今天在 Windows 主机上选不到这个变体：它存在的意义是 + 把“没有 Windows 宿主运行时”写进类型，而不是让调用方自己判平台。 + +## 为什么不做“Windows 本地进程”后端 + +macOS 原生的前提是：daemon 以本用户身份跑 NFS 回环服务，命令以同一身份 +运行，一切在同一个内核里。Windows 上没有等价物：LayerFS 的 FUSE 服务端是 +Linux 内核特性，Store 的 WAL 与目录锁只在一个内核内协调（见 #352 直接 +Store 访问的安全条件），命令身份、挂载可见性和进程组语义都要另建一套。 +因此 Windows 的路线不是“再写一个本地后端”，而是把 Linux 侧的能力搬到一个 +Linux 宿主里。 + +## 计划方向 + +两条路线，都由 #515 的 placement 设置表达，而不是在 Windows 本机上跑 daemon： + +1. **WSL2 分发（近期）**：Windows 上的集成适配器把 SDK 调用转发给 + 每个 LayerFS home 一个的、仅回环且带密钥的 worker；worker 在一个选定的 + WSL2 Linux 分发里直接用真实 SDK 打开共享 Store（ext4/xfs/btrfs）， + 并用该分发的 Docker Engine 起 Sandbox daemon。Windows 只拥有 UI、绑定和 + trace，Linux 拥有本地句柄和项目发现。这是仓库根 `AGENTS.md` 已记录的 + Windows 例外，规则不变：只连回环、每次调用只尝试一次、丢失的回复不重放、 + 结果不确定的 Init 保留其 Linux home 和卷。 +2. **Hyper-V / VM 方向（后续）**：每个 Store 一个 Linux VM，Sandbox 是 VM 内 + 的命名空间；宿主侧只通过一条私有流连到 VM 内的 worker。见 #515 §6 + （`vm:` placement 与各 provider 的 VM/磁盘/私有流对照表）和 + #512 的实验结论。 + +## 实现时要先定的事 + +- placement 与后端选择的设置名（#515 V9 提议 `LAYERFS_PLACEMENT` 与 + `LAYERFS_SANDBOX_BACKEND`，只读 `LAYERFS_*`；集成自己的前缀由集成映射）。 +- Windows 上 `deploy.sandbox`、`deploy.localDir` 等现有设置的语义：#515 + 要求 product SDK 与 daemon 不为 Windows 改动，Windows 差异留在适配器里。 +- 到那时 `Runtime` 是否新增一个持有 worker 端点的变体（那时才需要数据）， + 以及记录里如何标注该 Sandbox 由哪个运行时创建（无字段即 Docker）。 diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..497fa2024b 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,9 +1,14 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. -#![forbid(unsafe_code)] +// unsafe only in backend/macos/darwin.rs: macOS process facts come from +// proc_pidinfo, a system call (`ps` is setuid, which Seatbelt refuses) +#![deny(unsafe_code)] #[cfg(unix)] pub mod backend; +pub mod provider; mod types; +#[cfg(unix)] +pub use backend::{Runtime, Sandbox}; pub use types::{ - CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, PendingRequest, - RuntimeError, WireFailure, + CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, ExecRequest, Image, + PendingRequest, RuntimeError, WireFailure, }; diff --git a/engine/crates/layerfs-sandbox/src/provider/capabilities.rs b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs new file mode 100644 index 0000000000..a11548c3eb --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs @@ -0,0 +1,30 @@ +//! Explicit runtime capabilities; these describe support, not qualification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProviderCapabilities { + pub name: &'static str, + pub isolation: IsolationModel, + pub mount_driver: MountDriver, + pub requires_image: bool, + pub search_path_policy: SearchPathPolicy, + pub supports_resource_limits: bool, + pub direct_store_access: bool, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IsolationModel { + Container, + HostProcess, + MicroVm, + RemoteCloud, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MountDriver { + Fuse, + LoopbackNfs, + SharedFs, + None, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchPathPolicy { + FixedContainer, + InheritHost, +} diff --git a/engine/crates/layerfs-sandbox/src/provider/launch.rs b/engine/crates/layerfs-sandbox/src/provider/launch.rs new file mode 100644 index 0000000000..0c654ad481 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/launch.rs @@ -0,0 +1,26 @@ +//! One-attempt launch failure: acknowledged partial ownership and the last phase reached. +use crate::backend::Sandbox; +use crate::RuntimeError; +use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LaunchPhase { + Validate, + Create, + Start, + Listener, +} +/// The caller retains the exact resource even when launch fails after creating it. +pub struct LaunchFailure { + pub phase: LaunchPhase, + pub instance: Option>, + pub cause: RuntimeError, +} +impl fmt::Debug for LaunchFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchFailure") + .field("phase", &self.phase) + .field("identity", &self.instance.as_ref().map(|i| i.identity())) + .field("cause", &self.cause) + .finish() + } +} diff --git a/engine/crates/layerfs-sandbox/src/provider/mod.rs b/engine/crates/layerfs-sandbox/src/provider/mod.rs new file mode 100644 index 0000000000..b9eee1e304 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/mod.rs @@ -0,0 +1,12 @@ +//! Runtime inputs and observations; concrete backends own their own lifecycle. +mod capabilities; +mod launch; +mod options; +pub use capabilities::{IsolationModel, MountDriver, ProviderCapabilities, SearchPathPolicy}; +pub use launch::{LaunchFailure, LaunchPhase}; +pub use options::{ + DockerRecord, ExecExit, LaunchRequest, LocalRecord, NamespaceRecord, ProviderLaunchOptions, + SandboxInspection, SandboxRecord, +}; +/// Compatible name for the existing exact 64-hex runtime identity. +pub type SandboxId = crate::ContainerId; diff --git a/engine/crates/layerfs-sandbox/src/provider/options.rs b/engine/crates/layerfs-sandbox/src/provider/options.rs new file mode 100644 index 0000000000..0ea23bd979 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/options.rs @@ -0,0 +1,193 @@ +//! Launch inputs and observations independent of the daemon control protocol. +use crate::{CommandIdentity, ContainerId}; +use std::{fmt, path::PathBuf, time::Duration}; +/// Configuration bytes may contain secrets; Debug only reports their length. +#[derive(Clone)] +pub struct LaunchRequest { + pub name: String, + pub daemon_executable: PathBuf, + pub daemon_config: Vec, + pub labels: Vec<(String, String)>, + pub startup_timeout: Duration, + pub options: ProviderLaunchOptions, +} +impl fmt::Debug for LaunchRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchRequest") + .field("name", &self.name) + .field("daemon_executable", &self.daemon_executable) + .field("config_bytes", &self.daemon_config.len()) + .field("startup_timeout", &self.startup_timeout) + .finish_non_exhaustive() + } +} +#[derive(Clone, Debug)] +pub enum ProviderLaunchOptions { + Local { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + }, + /// The Linux namespace backend: a namespaced daemon plus the Store file it + /// opens. `store` is the exact file; its directory is what the Sandbox's + /// private root binds, so SQLite's WAL and the writers' directory lock see + /// the same inode as every other opener on this kernel. `image` is the + /// digest its root is an overlay of, and `archive` the docker-archive in + /// the bundle that digest names, which is unpacked once per digest. + Namespaces { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + store: PathBuf, + image: crate::Image, + archive: PathBuf, + }, + Docker { + image: String, + store_volume: String, + port: u16, + connect_host: Option, + }, +} +/// Persisted physical facts supplied by the record owner; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalRecord { + pub name: String, + pub pid: u32, + pub start_time: u64, + pub boot: Option, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// A namespaced Linux Sandbox's record: the local facts (its directory name, +/// init process identity and listener port) and the image digest its root is +/// an overlay of, so a reopen finds the same unpacked image. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamespaceRecord { + pub local: LocalRecord, + pub image: crate::Image, +} +impl NamespaceRecord { + /// The physical local facts every record of this backend keeps. + pub fn local(&self) -> &LocalRecord { + &self.local + } + /// The image digest the Sandbox's root is an overlay of. + pub fn image(&self) -> &crate::Image { + &self.image + } +} +/// Persisted deployment facts for a Docker container; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DockerRecord { + pub image: String, + pub store_volume: String, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// Recorded runtime identity and deployment facts for reopen. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SandboxRecord { + Local(LocalRecord), + /// A namespaced Linux Sandbox; the physical facts kept are the same as a + /// local one (its directory name, init process identity and listener port), + /// plus the image digest its root is an overlay of. The variant itself + /// names the backend, so a record is never reopened by another one. + Namespaces(NamespaceRecord), + Docker(DockerRecord), +} +impl SandboxRecord { + pub fn local(&self) -> Option<&LocalRecord> { + match self { + Self::Local(record) => Some(record), + _ => None, + } + } + pub fn namespaces(&self) -> Option<&NamespaceRecord> { + match self { + Self::Namespaces(record) => Some(record), + _ => None, + } + } + pub fn docker(&self) -> Option<&DockerRecord> { + match self { + Self::Docker(record) => Some(record), + _ => None, + } + } +} +impl From for SandboxRecord { + fn from(record: LocalRecord) -> Self { + Self::Local(record) + } +} +impl From<&LocalRecord> for SandboxRecord { + fn from(record: &LocalRecord) -> Self { + Self::Local(record.clone()) + } +} +impl From for SandboxRecord { + fn from(record: DockerRecord) -> Self { + Self::Docker(record) + } +} +impl From<&DockerRecord> for SandboxRecord { + fn from(record: &DockerRecord) -> Self { + Self::Docker(record.clone()) + } +} +impl From for DockerRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image, + store_volume: request.store_volume, + port: request.port, + labels: request.labels, + } + } +} +impl From<&crate::backend::docker::SandboxRequest> for DockerRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image.clone(), + store_volume: request.store_volume.clone(), + port: request.port, + labels: request.labels.clone(), + } + } +} +impl From for crate::backend::docker::SandboxRequest { + fn from(record: DockerRecord) -> Self { + Self { + image: record.image, + store_volume: record.store_volume, + port: record.port, + labels: record.labels, + } + } +} +impl From for SandboxRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +impl From<&crate::backend::docker::SandboxRequest> for SandboxRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecExit { + pub code: i32, + /// Docker may not expose a process id after exit. + pub pid: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, +} +/// Unknown resource/timestamp fields stay None rather than invented zeroes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SandboxInspection { + pub identity: ContainerId, + pub running: bool, + pub exit_code: Option, + pub process_id: Option, + pub scratch_bytes: Option, +} diff --git a/engine/crates/layerfs-sandbox/src/types/failure.rs b/engine/crates/layerfs-sandbox/src/types/failure.rs index de4ddfe27d..26c3b3d915 100644 --- a/engine/crates/layerfs-sandbox/src/types/failure.rs +++ b/engine/crates/layerfs-sandbox/src/types/failure.rs @@ -3,6 +3,9 @@ use std::{fmt, io}; /// Original ordinary-runtime refusal or I/O cause. #[derive(Debug)] pub enum RuntimeError { + /// Original Docker provider stage failure, including retained transfer/stream owners. + #[cfg(unix)] + Docker(Box), Io(io::Error), Protocol(&'static str), /// Original HTTP status, with a separately retained bounded error-body diagnostic. @@ -18,6 +21,8 @@ impl From for RuntimeError { impl fmt::Display for RuntimeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + #[cfg(unix)] + Self::Docker(failure) => write!(f, "Docker provider: {failure:?}"), Self::Io(e) => write!(f, "runtime I/O: {e}"), Self::Protocol(e) => write!(f, "runtime protocol: {e}"), Self::Http(s) => write!(f, "runtime HTTP status {s}"), @@ -53,6 +58,24 @@ pub struct WireFailure { /// The cause [`WireFailure::container_stopped`] reports. pub(crate) const STOPPED: &str = "exact stopped container observation"; impl WireFailure { + /// A non-HTTP cause before an attempt, with no invented transport facts. + pub fn local(cause: RuntimeError) -> Box { + Box::new(Self { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: None, + requested_exec: None, + error_body: None, + }) + } /// The Engine still has the exact requested container, intact, but it is /// not running: its daemon exited, so nothing it held survives. pub fn container_stopped(&self) -> bool { diff --git a/engine/crates/layerfs-sandbox/src/types/image.rs b/engine/crates/layerfs-sandbox/src/types/image.rs new file mode 100644 index 0000000000..107ce35bae --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/image.rs @@ -0,0 +1,43 @@ +//! A Sandbox image's identity: the digest a deployment names, which is also +//! the name its unpacked root is cached under. It is a value type on every +//! platform, so a record written where the image backend runs can still be +//! read where it cannot. +use super::RuntimeError; + +/// The prefix every image digest carries. +const ALGORITHM: &str = "sha256:"; + +/// A Sandbox image's identity: `sha256:<64 lowercase hex>`, the same text the +/// Docker backend's `deploy.image` holds and a docker-archive names its config +/// blob by. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Image(String); + +impl Image { + /// Parses `sha256:<64 lowercase hex>`, the only form an archive's config + /// digest is compared with. + pub fn parse(text: &str) -> Result { + let hex = text.strip_prefix(ALGORITHM).unwrap_or(""); + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(RuntimeError::Protocol( + "Sandbox image must be an exact digest, sha256:<64 lowercase hex>", + )); + } + Ok(Self(text.to_owned())) + } + + /// The whole digest text, `sha256:<64 hex>`. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The 64 hex digits without the algorithm: the name this image's + /// unpacked root is cached under. + pub fn hex(&self) -> &str { + &self.0[ALGORITHM.len()..] + } +} diff --git a/engine/crates/layerfs-sandbox/src/types/mod.rs b/engine/crates/layerfs-sandbox/src/types/mod.rs index 69827343cb..2d8c1768ae 100644 --- a/engine/crates/layerfs-sandbox/src/types/mod.rs +++ b/engine/crates/layerfs-sandbox/src/types/mod.rs @@ -2,9 +2,13 @@ mod execution; mod failure; mod identity; +mod image; +mod request; mod transfer; pub use execution::{CommandIdentity, ExecInspection}; pub(crate) use failure::STOPPED; pub use failure::{ErrorDiagnostic, RuntimeError, WireFailure}; pub use identity::{ContainerId, ExecId}; +pub use image::Image; +pub use request::ExecRequest; pub use transfer::PendingRequest; diff --git a/engine/crates/layerfs-sandbox/src/types/request.rs b/engine/crates/layerfs-sandbox/src/types/request.rs new file mode 100644 index 0000000000..e14e795a9b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/request.rs @@ -0,0 +1,48 @@ +//! Provider-independent ordinary command metadata and validation. +use crate::{CommandIdentity, RuntimeError}; +use std::fmt; +/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. +#[derive(Clone)] +pub struct ExecRequest { + pub identity: CommandIdentity, + pub arguments: Vec, + pub environment: Vec, + pub directory: String, + pub stdin: bool, +} +impl fmt::Debug for ExecRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecRequest") + .field("identity", &self.identity) + .field("arguments_count", &self.arguments.len()) + .field("environment_count", &self.environment.len()) + .field("directory", &self.directory) + .field("stdin", &self.stdin) + .finish() + } +} +impl ExecRequest { + pub(crate) fn check(&self) -> Result<(), RuntimeError> { + self.identity.check()?; + if self.arguments.is_empty() + || self.arguments[0].is_empty() + || !std::path::Path::new(&self.directory).is_absolute() + || self + .arguments + .iter() + .chain(&self.environment) + .chain(std::iter::once(&self.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if self + .environment + .iter() + .any(|v| !v.contains('=') || v.starts_with('=')) + { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + Ok(()) + } +} diff --git a/engine/crates/layerfs-sandbox/tests/docker_provider.rs b/engine/crates/layerfs-sandbox/tests/docker_provider.rs new file mode 100644 index 0000000000..8022f295d6 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/docker_provider.rs @@ -0,0 +1,394 @@ +//! Docker provider contracts exercised through bounded Unix HTTP peers, without an Engine. +#![cfg(unix)] +#[allow(dead_code)] +#[path = "support/engine.rs"] +mod peer; +use layerfs_sandbox::{ + backend::docker::{DockerProvider, ProviderFailure, Sandbox, SandboxRequest}, + provider::{ + DockerRecord, LaunchPhase, LaunchRequest, LocalRecord, ProviderLaunchOptions, SandboxRecord, + }, + CommandIdentity, ContainerId, ExecRequest, Runtime, RuntimeError, +}; +use peer::{fixed, frame, upgrade, Peer}; +use std::{ + fs, io, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; +const CID: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const IMAGE: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +static NEXT: AtomicUsize = AtomicUsize::new(0); +fn deployment() -> SandboxRequest { + SandboxRequest { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + } +} +struct Binary(PathBuf); +impl Binary { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "h22-provider-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&path, b"sealed executable").unwrap(); + Self(path) + } + fn launch(&self) -> LaunchRequest { + LaunchRequest { + name: "h22-sandbox-1".into(), + daemon_executable: self.0.clone(), + daemon_config: b"private config".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: None, + }, + } + } +} +impl Drop for Binary { + fn drop(&mut self) { + fs::remove_file(&self.0).unwrap(); + } +} +fn logs(payload: &[u8]) -> Vec { + let mut bytes = format!("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.multiplexed-stream\r\nContent-Length: {}\r\n\r\n", payload.len()).into_bytes(); + bytes.extend(payload); + bytes +} +fn launch_responses() -> Vec> { + vec![ + fixed( + 200, + br#"{"Name":"provider-store","Driver":"local","Scope":"local","Options":null}"#, + ), + fixed(201, format!(r#"{{"Id":"{CID}"}}"#).as_bytes()), + fixed(200, b""), + fixed(204, b""), + logs(&frame(1, b"LAYERFS_DAEMON_LISTEN 0.0.0.0:30421\n")), + ] +} +fn exec_request() -> ExecRequest { + ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), "printf ordinary".into()], + environment: vec![], + directory: "/workspaces/1".into(), + stdin: false, + } +} +fn exec_inspection(running: bool, exit: &str) -> Vec { + fixed(200, format!(r#"{{"ID":"{EID}","ContainerID":"{CID}","Running":{running},"ExitCode":{exit},"Pid":42}}"#).as_bytes()) +} +#[test] +fn launch_stream_log_stop_delete_and_negotiated_versions_use_the_original_runtime() { + let binary = Binary::new(); + for minor in [48, 54] { + let mut responses = vec![fixed( + 200, + format!(r#"{{"ApiVersion":"1.{minor}"}}"#).as_bytes(), + )]; + responses.extend(launch_responses()); + let mut payload = frame(1, &vec![b'x'; 1024 * 1024 + 3]); + payload.extend(frame(2, b"err")); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&payload), + exec_inspection(false, "7"), + logs(&frame(1, b"daemon log\n")), + fixed(204, b""), + fixed(204, b""), + ]); + let (docker, peer) = Peer::scripted(responses, 8192); + let provider = Runtime::Docker(DockerProvider::new(docker)); + assert_eq!(provider.capabilities().unwrap().name, "docker"); + let mut instance = provider.launch(binary.launch()).unwrap(); + assert_eq!(instance.identity().to_string(), CID); + assert_eq!(instance.scratch_directory(), "/tmp"); + assert!(instance.local_record().is_none()); + assert_eq!( + instance.record(), + Some(SandboxRecord::Docker(DockerRecord { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + })) + ); + instance.track(42, 1).unwrap(); + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let exit = instance + .run(&exec_request(), &mut stdout, &mut stderr) + .unwrap(); + assert_eq!((exit.code, exit.pid), (7, Some(42))); + assert_eq!(exit.stdout_bytes, 1024 * 1024 + 3); + assert_eq!(stdout.len() as u64, exit.stdout_bytes); + assert_eq!(stderr, b"err"); + assert_eq!(instance.log_tail(20, 2048).unwrap(), b"daemon log\n"); + instance.stop(1).unwrap(); + instance.delete().unwrap(); + assert!(instance.stop(1).is_err()); + assert!(instance.delete().is_err()); + let calls = peer.finish(); + assert_eq!(calls.len(), 12); + assert!(calls[0].starts_with(b"GET /version ")); + for call in &calls[1..] { + assert!(String::from_utf8_lossy(call) + .split(' ') + .nth(1) + .unwrap() + .starts_with(&format!("/v1.{minor}/"))); + } + assert!(String::from_utf8_lossy(&calls[2]).contains("create?name=h22-sandbox-1")); + assert!(String::from_utf8_lossy(calls.last().unwrap()).contains("force=false&v=false")); + } +} +#[test] +fn uncertain_create_retains_partial_id_and_never_adopts_or_cleans_it() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.truncate(2); + responses[1] = fixed(201, format!(r#"{{"Id":"{CID}","Id":"{CID}"}}"#).as_bytes()); + let (docker, peer) = Peer::new(responses, 1); + let failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, LaunchPhase::Create); + assert!(failure.instance.is_none()); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed original cause") + }; + let ProviderFailure::Create(cause) = *cause else { + panic!("create custody") + }; + assert_eq!(cause.transfer.observed_container.unwrap().to_string(), CID); + assert!(cause.transfer.attempted); + assert_eq!(peer.finish().len(), 2); +} +#[test] +fn upload_start_and_listener_failures_keep_acknowledged_instance_and_original_receipts() { + let binary = Binary::new(); + for (stage, expected) in [ + (2, LaunchPhase::Create), + (3, LaunchPhase::Start), + (4, LaunchPhase::Listener), + ] { + let mut responses = launch_responses(); + responses.truncate(stage + 1); + responses[stage] = fixed(409, b"refused"); + let (docker, peer) = Peer::new(responses, 8192); + let mut failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, expected); + assert_eq!(failure.instance.take().unwrap().identity().to_string(), CID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed receipt") + }; + match (*cause, stage) { + (ProviderFailure::Upload(f), 2) => { + assert!(f.work.tar_bytes > 0); + assert_eq!(f.transfer.status, Some(409)); + } + (ProviderFailure::Start(f), 3) => assert_eq!(f.status, Some(409)), + (ProviderFailure::Listener(f), 4) => assert_eq!(f.transfer.status, Some(409)), + _ => panic!("original stage lost"), + } + assert_eq!(peer.finish().len(), stage + 1); + } +} +#[test] +fn validation_and_stdin_refuse_without_an_engine_attempt() { + let binary = Binary::new(); + let (docker, peer) = Peer::new(vec![], 1); + let provider = Runtime::Docker(DockerProvider::new(docker.clone())); + let mut request = binary.launch(); + request.name = "../escape".into(); + assert_eq!( + provider.launch(request).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut bad_host = binary.launch(); + bad_host.options = ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: Some("bad host\0".into()), + }; + assert_eq!( + provider.launch(bad_host).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = binary.launch(); + wrong_options.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + let foreign = SandboxRecord::Local(LocalRecord { + name: "test".into(), + pid: 1, + start_time: 1, + boot: None, + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(CID).unwrap(); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + assert!( + !provider + .reopen_stopped(id, &foreign) + .err() + .unwrap() + .attempted + ); + let stopped = Sandbox::stopped(docker, ContainerId::parse(CID).unwrap(), deployment()); + let mut request = exec_request(); + request.stdin = true; + assert!( + !layerfs_sandbox::Sandbox::Docker(stopped) + .run(&request, &mut Vec::new(), &mut Vec::new()) + .unwrap_err() + .attempted + ); + assert!(peer.finish().is_empty()); +} +#[test] +fn eof_with_unknown_exit_preserves_the_handle_without_polling_or_replay() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"ok")), + exec_inspection(true, "null"), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecExitUnknown { + handle, + inspection, + progress, + } = *cause + else { + panic!("EOF must not invent exit") + }; + assert_eq!(progress.stdout, 2); + assert_eq!(progress.stderr, 0); + assert!(inspection.running); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(peer.finish().len(), 8); +} +#[test] +fn exec_stage_failures_keep_deciding_wire_facts_and_delivered_progress() { + let binary = Binary::new(); + for stage in 0..3 { + let mut responses = launch_responses(); + let mut exec = vec![ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"delivered")), + exec_inspection(false, "0"), + ]; + exec.truncate(stage + 1); + exec[stage] = fixed(409, b"original refusal"); + responses.extend(exec); + let (docker, peer) = Peer::new(responses, 8192); + let owner = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = owner + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.status, Some(409)); + assert!(failure.sent_bytes > 0); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("original custody") + }; + match (*cause, stage) { + (ProviderFailure::ExecCreate(f), 0) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes) + } + (ProviderFailure::ExecStart(f), 1) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes); + assert_eq!(f.created.identity().1.to_string(), EID); + } + ( + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + 2, + ) => { + assert_eq!(failure.sent_bytes, f.sent_bytes); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(progress.stdout, 9); + assert_eq!(progress.stderr, 0); + } + _ => panic!("wrong stage"), + } + assert_eq!(peer.finish().len(), 6 + stage); + } +} +struct BrokenSink; +impl io::Write for BrokenSink { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn failed_sink_retains_pending_output_and_exec_identity_without_inspection_or_cleanup() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"undelivered")), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut BrokenSink, &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecOutput { handle, failure } = *cause else { + panic!("stream owner retained") + }; + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(failure.output.progress().stdout, 0); + assert_eq!(failure.output.progress().wire_payload, 11); + assert_eq!(peer.finish().len(), 7); +} diff --git a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs index 9c59dc4c25..e3d18209d7 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs @@ -405,6 +405,9 @@ fn reopened_from(body: &str) -> Box { fn only_the_exact_intact_container_not_running_is_a_stopped_container() { let stopped = reopened_from(&exited()); assert!(stopped.container_stopped()); + assert!(stopped.attempted); + assert_eq!(stopped.status, Some(200)); + assert!(stopped.sent_bytes >= stopped.request_header_bytes && stopped.request_header_bytes > 0); assert!(!stopped.container_gone()); let other = "c".repeat(64); assert!(!reopened_from(&exited().replace(CID, &other)).container_stopped()); @@ -414,6 +417,47 @@ fn only_the_exact_intact_container_not_running_is_a_stopped_container() { assert!(!reopened_from(&image).container_stopped()); } +#[test] +fn provider_inspects_and_reopens_exact_running_and_stopped_deployments() { + use layerfs_sandbox::backend::docker::DockerProvider; + use layerfs_sandbox::provider::SandboxRecord; + let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); + let running = inspect().replace( + r#""Running":true"#, + r#""Running":true,"Pid":42,"ExitCode":0"#, + ); + let stopped = exited().replace( + r#""Running":false"#, + r#""Running":false,"Pid":0,"ExitCode":7"#, + ); + let (docker, peer) = Peer::new( + vec![ + fixed(200, running.as_bytes()), + fixed(200, running.as_bytes()), + fixed(200, stopped.as_bytes()), + fixed(204, b""), + ], + 64, + ); + let provider = DockerProvider::new(docker); + let record = SandboxRecord::from(request()); + let mut owner = provider.reopen(id, &record).unwrap(); + let facts = owner.inspect().unwrap(); + assert!(facts.running); + assert_eq!(facts.process_id, Some(42)); + assert_eq!(facts.exit_code, None); + assert_eq!(facts.scratch_bytes, None); + let facts = owner.inspect().unwrap(); + assert!(!facts.running); + assert_eq!(facts.process_id, None); + assert_eq!(facts.exit_code, Some(7)); + // A read-only observation does not authorize Delete on the running owner. + assert!(!owner.delete().unwrap_err().attempted); + let mut stopped = provider.reopen_stopped(id, &record).unwrap(); + stopped.delete().unwrap(); + assert_eq!(peer.finish().len(), 4); +} + #[test] fn a_stopped_container_is_deleted_once_and_never_forced() { let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); diff --git a/engine/crates/layerfs-sandbox/tests/local_search_path.rs b/engine/crates/layerfs-sandbox/tests/local_search_path.rs new file mode 100644 index 0000000000..a6ff147a11 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/local_search_path.rs @@ -0,0 +1,113 @@ +//! A native (macOS) local Sandbox's daemon and commands start with this +//! process's own PATH, so this user's tools (`node` in `~/.local/bin`, +//! Homebrew's `/opt/homebrew/bin`) are found as in the caller's shell; with +//! no PATH, or an empty one, a container's search path and Homebrew's. +//! A small shell script stands in for the daemon. One test: it changes this +//! process's PATH, which no other test in this binary reads. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::{ + docker::ExecRequest, + native::{Local, Sandbox}, + }, + CommandIdentity, +}; +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const WAIT: Duration = Duration::from_secs(10); +const FALLBACK: &str = + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/homebrew/bin"; + +/// A fresh 0755 runtime root and a stand-in daemon that prints its +/// environment, then its listener line, then waits. +fn fixture() -> (PathBuf, PathBuf) { + let base = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("layerfs-search-path-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("root")).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + let daemon = base.join("daemon.sh"); + fs::write( + &daemon, + "#!/bin/sh\n[ \"$1\" = --config ] && [ -f \"$2\" ] || exit 3\n/usr/bin/env >&2\necho \"LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\"\nexec /bin/sleep 60\n", + ) + .unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + (base.join("root"), daemon) +} + +fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { + Local::native(root) + .unwrap() + .naming(name) + .unwrap() + .create(daemon, b"setup bytes", WAIT, Vec::new()) + .unwrap() +} + +/// The daemon's PATH line, from its log. +fn daemon_path(sandbox: &Sandbox) -> String { + let log = String::from_utf8(sandbox.log_tail(200, 16384).unwrap()).unwrap(); + log.lines() + .find_map(|line| line.strip_prefix("PATH=")) + .unwrap_or_else(|| panic!("no PATH line: {log}")) + .to_owned() +} + +/// A command's PATH: `printf` by its absolute path, so a lookup is not needed. +fn command_path(sandbox: &Sandbox, environment: &[&str]) -> String { + let request = ExecRequest { + identity: CommandIdentity { + uid: nix::unistd::Uid::effective().as_raw(), + gid: nix::unistd::Gid::effective().as_raw(), + }, + arguments: vec!["/bin/sh".into(), "-c".into(), "printf %s \"$PATH\"".into()], + environment: environment.iter().map(|e| e.to_string()).collect(), + directory: sandbox.directory().display().to_string(), + stdin: false, + }; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + assert_eq!(exit.code, 0, "{}", String::from_utf8_lossy(&err)); + String::from_utf8(out).unwrap() +} + +#[test] +fn a_native_daemon_and_its_commands_start_with_this_processs_path() { + let (root, daemon) = fixture(); + let own = "/Users/someone/.local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; + let saved = std::env::var_os("PATH"); + std::env::set_var("PATH", own); + let mut sandbox = started(&root, &daemon, "layerfs-path-1"); + assert_eq!(daemon_path(&sandbox), own); + assert_eq!(command_path(&sandbox, &[]), own); + // a command's own PATH still wins + assert_eq!( + command_path(&sandbox, &["PATH=/opt/x/bin:/bin"]), + "/opt/x/bin:/bin" + ); + // no PATH, then an empty one: a container's search path and Homebrew's + std::env::remove_var("PATH"); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + std::env::set_var("PATH", ""); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + let mut second = started(&root, &daemon, "layerfs-path-2"); + assert_eq!(daemon_path(&second), FALLBACK); + match saved { + Some(path) => std::env::set_var("PATH", path), + None => std::env::remove_var("PATH"), + } + for sandbox in [&mut sandbox, &mut second] { + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + } + let _ = fs::remove_dir_all(root.parent().unwrap()); +} diff --git a/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs b/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs new file mode 100644 index 0000000000..a1cb95089b --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/local_stop_mount.rs @@ -0,0 +1,48 @@ +//! Opt-in host proof against one caller-owned local Sandbox fixture. +//! Supply LAYERFS_TEST_STOP_DIRECTORY, _PID and _PORT. The fixture must be +//! disposable: this test stops its daemon and force-detaches its mounts. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::native::{LocalProcess, Sandbox}, + ContainerId, +}; +use std::{path::PathBuf, process::Command}; + +#[test] +#[ignore = "requires an explicitly supplied disposable mounted Sandbox"] +fn stop_detaches_mounts_before_returning_without_delete() { + let directory = PathBuf::from(std::env::var("LAYERFS_TEST_STOP_DIRECTORY").unwrap()); + let pid = std::env::var("LAYERFS_TEST_STOP_PID") + .unwrap() + .parse() + .unwrap(); + let port = std::env::var("LAYERFS_TEST_STOP_PORT") + .unwrap() + .parse() + .unwrap(); + assert!(directory.is_absolute()); + let mounts = || { + let out = Command::new("/sbin/mount").output().unwrap(); + assert!(out.status.success()); + String::from_utf8(out.stdout) + .unwrap() + .lines() + .filter(|line| line.contains(&format!(" on {}/", directory.display()))) + .map(str::to_owned) + .collect::>() + }; + assert!(!mounts().is_empty(), "fixture must start with a real mount"); + let process = LocalProcess::observe(pid).unwrap(); + let mut sandbox = Sandbox::reopen( + directory.clone(), + ContainerId::parse(&"1".repeat(64)).unwrap(), + process, + port, + Vec::new(), + ) + .unwrap(); + sandbox.stop(1).unwrap(); + assert!(!process.alive(), "fixture daemon survived Stop"); + assert!(directory.exists(), "Stop must not delete its directory"); + assert!(mounts().is_empty(), "Stop left mounts until Delete"); +} diff --git a/engine/crates/layerfs-sandbox/tests/native_platform.rs b/engine/crates/layerfs-sandbox/tests/native_platform.rs new file mode 100644 index 0000000000..46ffae1258 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_platform.rs @@ -0,0 +1,71 @@ +//! Platform refusal precedes filesystem access; a runtime never selects another backend. +#![cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ContainerId, Runtime, RuntimeError, +}; +use std::time::Duration; + +#[test] +fn wrong_platform_is_refused_before_root_access() { + #[cfg(target_os = "macos")] + let result = layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider", + ); + #[cfg(target_os = "linux")] + let result = layerfs_sandbox::backend::native::NativeMacosProvider::new( + "/does-not-exist/native-provider", + ); + assert!(matches!(result, Err(RuntimeError::Unsupported(_)))); +} + +#[test] +fn native_linux_requires_root_before_root_access() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(matches!( + layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider" + ), + Err(RuntimeError::Unsupported(_)) + )); + } +} + +/// Windows has no host runtime: the placeholder variant refuses every +/// operation before any effect, and never falls back to Docker or native. +#[test] +fn windows_is_refused_before_any_effect() { + let runtime = Runtime::Windows; + assert!(matches!( + runtime.capabilities(), + Err(RuntimeError::Unsupported(_)) + )); + assert!(matches!( + runtime.local_directory("one"), + Err(RuntimeError::Unsupported(_)) + )); + let request = LaunchRequest { + name: "one".into(), + daemon_executable: "/does-not-exist/daemon".into(), + daemon_config: b"setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }, + }; + let failure = runtime.launch(request).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Validate); + assert!(failure.instance.is_none()); + assert!(matches!(failure.cause, RuntimeError::Unsupported(_))); + let record = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-store".into(), + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(&"b".repeat(64)).unwrap(); + assert!(runtime.reopen(id, &record).is_err()); + assert!(runtime.reopen_stopped(id, &record).is_err()); +} diff --git a/engine/crates/layerfs-sandbox/tests/native_provider.rs b/engine/crates/layerfs-sandbox/tests/native_provider.rs new file mode 100644 index 0000000000..29cf512469 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_provider.rs @@ -0,0 +1,124 @@ +//! Real native runtime lifecycle and partial launch custody through the Runtime enum. +#![cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg(target_os = "linux")] +use layerfs_sandbox::backend::native::NativeLinuxProvider as NativeProvider; +#[cfg(target_os = "macos")] +use layerfs_sandbox::backend::native::NativeMacosProvider as NativeProvider; +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, Runtime, +}; +use std::{fs, os::unix::fs::PermissionsExt, time::Duration}; + +/// The native runtime this host offers, over `root`. +#[cfg(target_os = "linux")] +fn native(root: &std::path::Path) -> Runtime { + Runtime::LinuxNative(NativeProvider::new(root).unwrap()) +} + +#[cfg(target_os = "macos")] +fn native(root: &std::path::Path) -> Runtime { + Runtime::MacosNative(NativeProvider::new(root).unwrap()) +} + +#[test] +fn runtime_preserves_partial_launch_owner_and_reopens_the_exact_process() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(NativeProvider::new("/no-access-required").is_err()); + return; + } + let base = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("layerfs-provider-{}", std::process::id())); + fs::create_dir(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let binary = base.join("daemon"); + fs::write( + &binary, + "#!/bin/sh\necho LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\nexec /bin/sleep 30\n", + ) + .unwrap(); + fs::set_permissions(&binary, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = native(&base); + assert_eq!( + provider.capabilities().unwrap().name, + if cfg!(target_os = "linux") { + "native-linux" + } else { + "native-macos" + } + ); + let request = |name: &str| LaunchRequest { + name: name.into(), + daemon_executable: binary.clone(), + daemon_config: b"private setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { + uid: if cfg!(target_os = "linux") { + 501 + } else { + nix::unistd::Uid::effective().as_raw() + }, + gid: if cfg!(target_os = "linux") { + 20 + } else { + nix::unistd::Gid::effective().as_raw() + }, + }, + environment: vec![], + }, + }; + let mut sandbox = provider.launch(request("success")).unwrap(); + let id = sandbox.identity(); + let local_record = sandbox.local_record().unwrap(); + let record = sandbox.record().unwrap(); + assert_eq!(record, SandboxRecord::Local(local_record)); + assert!(sandbox.inspect().unwrap().running); + let reopened = provider.reopen(id, &record).unwrap(); + assert_eq!(reopened.identity(), id); + assert_eq!(reopened.endpoint().unwrap(), sandbox.endpoint().unwrap()); + let foreign = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-foreign".into(), + port: 30421, + labels: vec![], + }); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + drop(reopened); + sandbox.stop(1).unwrap(); + assert!(!sandbox.inspect().unwrap().running); + assert!(provider + .reopen(id, &record) + .err() + .unwrap() + .container_stopped()); + sandbox.delete().unwrap(); + fs::write(&binary, "#!/bin/sh\nexit 3\n").unwrap(); + let mut failure = provider.launch(request("failure")).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Listener); + let mut partial = failure.instance.take().expect("created resource retained"); + partial.stop(1).unwrap(); + partial.delete().unwrap(); + let mut invalid = request("../escape"); + invalid.daemon_config = b"NEVER_PRINT_THIS_SECRET".to_vec(); + assert!(!format!("{invalid:?}").contains("NEVER_PRINT_THIS_SECRET")); + assert_eq!( + provider.launch(invalid).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = request("wrong-options"); + wrong_options.options = ProviderLaunchOptions::Docker { + image: format!("sha256:{}", "b".repeat(64)), + store_volume: "lfs-foreign".into(), + port: 30421, + connect_host: None, + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + fs::remove_dir_all(base).unwrap(); +} diff --git a/engine/crates/layerfs-sandbox/tests/runtime_enum.rs b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs new file mode 100644 index 0000000000..1f7c1ba78f --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs @@ -0,0 +1,15 @@ +//! The Runtime enum's Docker selection: no local directory, container capabilities. +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::Runtime; +use std::time::Duration; + +/// Selecting Docker names an Engine endpoint only; nothing is dialed here. +#[test] +fn docker_reports_container_capabilities_and_owns_no_local_directory() { + let docker = Docker::new("/does-not-exist/docker.sock", Duration::from_secs(1)).unwrap(); + let runtime = Runtime::Docker(DockerProvider::new(docker)); + let capabilities = runtime.capabilities().unwrap(); + assert_eq!(capabilities.name, "docker"); + assert!(capabilities.requires_image); + assert!(runtime.local_directory("anything").unwrap().is_none()); +} diff --git a/engine/crates/layerfs-store-service/src/install.rs b/engine/crates/layerfs-store-service/src/install.rs index 850570a334..f150c5685c 100644 --- a/engine/crates/layerfs-store-service/src/install.rs +++ b/engine/crates/layerfs-store-service/src/install.rs @@ -21,6 +21,7 @@ use std::sync::Arc; /// The daemon's user inside its Sandbox (the runtime's `User` is `0:0`). A /// daemon opens only a Store file and directory it owns, so an installer /// must run as this user too. +#[cfg(not(target_os = "macos"))] const DAEMON_UID: u32 = 0; /// The destination and its exclusive temporary name. @@ -202,7 +203,15 @@ pub fn install_local( result.map_err(|cause: String| LocalInstallFailure { cause, ..left }) } +/// macOS has only the native route, whose daemon runs as the user who +/// installs its Store (there is no root daemon to match); the daemon's own +/// check that it owns the Store file still holds. +#[cfg(target_os = "macos")] +fn installer_user() -> Result<(), String> { + Ok(()) +} /// Refuses an installer that is not the daemons' user. +#[cfg(not(target_os = "macos"))] fn installer_user() -> Result<(), String> { use std::os::unix::fs::MetadataExt; // the owner of /proc/self is this process's effective user diff --git a/engine/crates/layerfs-store-service/src/kernel_guard.rs b/engine/crates/layerfs-store-service/src/kernel_guard.rs index 49489cac1e..e0e3945972 100644 --- a/engine/crates/layerfs-store-service/src/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/src/kernel_guard.rs @@ -56,7 +56,32 @@ pub fn check(store: &Path) -> Result<&'static str, Unsupported> { )) }) } - #[cfg(not(target_os = "linux"))] + // macOS native (LAYERFS_SANDBOX=native): the daemon, the SDK and every + // command share this one macOS kernel, so the Store's locks coordinate + // exactly as on Linux; only a local filesystem may hold the Store. + #[cfg(target_os = "macos")] + { + let directory = store + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let stat = nix::sys::statfs::statfs(directory).map_err(|error| { + Unsupported(format!( + "the Store's directory {} cannot be examined: {error}", + directory.display() + )) + })?; + match stat.filesystem_type_name() { + "apfs" => Ok("apfs"), + "hfs" => Ok("hfs"), + other => Err(Unsupported(format!( + "filesystem {other} at {} does not carry SQLite locks between kernels; \ + direct Store access needs apfs or hfs", + directory.display() + ))), + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { let _ = store; Err(Unsupported( diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index d9050f370d..e99d46ea85 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -1,11 +1,13 @@ //! Direct Store access is refused `Unsupported` where the Store's locks do -//! not reach every opener (#352 F-020): on macOS, and on Linux for a -//! filesystem outside the allowlist (here tmpfs). The refusal comes before any -//! file is opened: the Store file's bytes are unchanged and no overlay -//! template is created. +//! not reach every opener (#352 F-020): off Linux and macOS, and on Linux for +//! a filesystem outside the allowlist (here tmpfs). The refusal comes before +//! any file is opened: the Store file's bytes are unchanged and no overlay +//! template is created. On macOS a local apfs or hfs directory is allowed. +#[cfg(not(target_os = "macos"))] use layerfs_store_service::{OpenRefusal, StoreService}; use std::path::{Path, PathBuf}; +#[cfg(not(target_os = "macos"))] fn refused(directory: &Path) -> (String, Vec, Vec) { let store = directory.join("store.sqlite"); std::fs::write(&store, b"not opened").unwrap(); @@ -30,7 +32,16 @@ fn scratch(parent: &Path, label: &str) -> PathBuf { directory } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] +#[test] +fn direct_access_is_allowed_on_a_local_macos_filesystem() { + let directory = scratch(&std::env::temp_dir(), "apfs"); + let checked = layerfs_store_service::kernel_guard::check(&directory.join("store.sqlite")); + assert!(matches!(checked, Ok("apfs") | Ok("hfs")), "{checked:?}"); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] fn direct_access_is_unsupported_off_linux() { let directory = scratch(&std::env::temp_dir(), "host"); diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index c4159c3c12..97201c5102 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -17,12 +17,14 @@ # Crates whose `unsafe` surface is bounded. layerfs-storage keeps one audited # FFI module (encoding/codec.rs); Persistence has the owner-authorized macOS -# seal file-control module. Their siblings stay unsafe-free. The +# seal file-control module; Sandbox reads macOS process facts through +# proc_pidinfo in backend/macos/darwin.rs. Their siblings stay unsafe-free. The # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-sandbox": "src/backend/macos/darwin.rs", } UNSAFE_FREE_CRATES = ( "layerfs-content", @@ -32,10 +34,10 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", - "layerfs-sandbox", ) UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", @@ -47,10 +49,11 @@ "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", + "layerfs-nfs": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", - "layerfs-sandbox": "#![forbid(unsafe_code)]", + "layerfs-sandbox": "#![deny(unsafe_code)]", } @@ -84,6 +87,8 @@ "layerfs-overlay": set(), "layerfs-workspace": {"layerfs-content", "layerfs-overlay", "layerfs-telemetry"}, "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, + # The NFS export reuses the kernel ports and operations; never daemon. + "layerfs-nfs": {"layerfs-content", "layerfs-overlay", "layerfs-workspace", "layerfs-fuse"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. "layerfs-store-service": { @@ -103,6 +108,7 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-bridge", "layerfs-content", "layerfs-storage", @@ -135,6 +141,7 @@ "layerfs-bridge": ENGINE, "layerfs-daemon": ENGINE, "layerfs-fuse": ENGINE, + "layerfs-nfs": ENGINE, "layerfs-overlay": ENGINE, "layerfs-persistence": ENGINE, "layerfs-project": ENGINE, @@ -196,6 +203,7 @@ def test_path_violations(path, source, packages): "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", diff --git a/tools/ci_tests.py b/tools/ci_tests.py index ac284bf549..b07ff4b0d5 100644 --- a/tools/ci_tests.py +++ b/tools/ci_tests.py @@ -40,6 +40,7 @@ # (package, test target): run seconds on the macOS host, 2026-10-23. SLOW = { + ("layerfs-nfs", "helper_deadline"): 60.1, # Six Linux FUSE teardown cases, including two bounded Join observations. ("layerfs-daemon", "stalled_unmount"): 21.28, # Linux native binding proof, owner-excluded from CI; run explicitly. diff --git a/tools/test_check_product_boundary.py b/tools/test_check_product_boundary.py index ec4196fbff..55f600b5f4 100644 --- a/tools/test_check_product_boundary.py +++ b/tools/test_check_product_boundary.py @@ -126,7 +126,15 @@ def test_sdk_source_edges_and_unsafe_boundary(self): def test_sandbox_active_edges_and_unsafe_boundary(self): path = Path("engine/crates/layerfs-sandbox/src/backend/docker/http.rs") self.assertTrue(unsafe_violations(path, "unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![deny(unsafe_code)]")) + self.assertTrue(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + darwin = Path("engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs") + self.assertFalse(unsafe_violations(darwin, "fn f() { unsafe { libc::proc_listallpids(); } }")) + self.assertTrue( + unsafe_violations( + Path("engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs"), "fn f() { unsafe { call(); } }" + ) + ) prefix = '[package]\nname="layerfs-sandbox"\n[dependencies]\n' for dependency in ( "layerfs-api-core",