From 125723a3ab943e7d341b0e657d4158a6641b87b5 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:16 +0800 Subject: [PATCH 01/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/Cargo.toml (not production (test, manifest or docs)). No tests changed or deleted. Production LOC: 108,451 -> 108,451 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- core/crates/layerfs-sandbox/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/crates/layerfs-sandbox/Cargo.toml b/core/crates/layerfs-sandbox/Cargo.toml index e75b3fd4ce..05b1387cbe 100644 --- a/core/crates/layerfs-sandbox/Cargo.toml +++ b/core/crates/layerfs-sandbox/Cargo.toml @@ -7,4 +7,4 @@ rust-version.workspace = true license.workspace = true [dependencies] -nix = { version = "=0.31.3", features = ["poll", "socket", "fs"] } +nix = { version = "=0.31.3", features = ["poll", "socket", "fs", "signal", "mount", "user"] } From e4cbcad34c0e18c211364e5840e65561e8503959 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:17 +0800 Subject: [PATCH 02/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/mod.rs (production). No tests changed or deleted. Production LOC: 108,451 -> 108,460 (delta +9) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../layerfs-sandbox/src/backend/local/mod.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/mod.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/mod.rs b/core/crates/layerfs-sandbox/src/backend/local/mod.rs new file mode 100644 index 0000000000..8de8a6f18c --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/mod.rs @@ -0,0 +1,17 @@ +//! Local process runtime: each Sandbox is one daemon process on this kernel, +//! started from a private directory, with no container, image or Engine. +//! +//! It is for a machine that is itself the isolation boundary (a microVM, a +//! CI job). It isolates nothing: the daemon and its commands share this +//! kernel's processes, network, users and filesystem with everything else on +//! it. Commands still run as their own nonroot identity. One attempt per +//! operation, as in the Docker runtime; nothing is retried. +mod create; +mod exec; +mod mounts; +mod process; +mod sandbox; +pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; +pub use exec::LocalExit; +pub use process::LocalProcess; +pub use sandbox::Sandbox; From c43060fef206602fddb55a3478e13cf1b62c1089 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:19 +0800 Subject: [PATCH 03/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/process.rs (production). No tests changed or deleted. Production LOC: 108,460 -> 108,505 (delta +45) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../src/backend/local/process.rs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/process.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/process.rs b/core/crates/layerfs-sandbox/src/backend/local/process.rs new file mode 100644 index 0000000000..67eb36d4c1 --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/process.rs @@ -0,0 +1,60 @@ +//! A local process named by its id and kernel start time, so a reused id is +//! never taken for the process that was started. +use nix::{sys::signal::Signal, unistd::Pid}; +use std::io; + +/// One started process: its id and its start time in clock ticks since boot +/// (`/proc//stat` field 22). Both must match for it to count as alive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalProcess { + pub pid: u32, + pub start: u64, +} + +impl LocalProcess { + /// The process running now under `pid`, read once from `/proc`. + pub fn observe(pid: u32) -> io::Result { + let (_, start) = stat(pid)?; + Ok(Self { pid, start }) + } + /// True while this exact process exists and has not exited: a zombie + /// (or a dead entry) is not alive, and another process under the same id + /// has another start time. + pub fn alive(self) -> bool { + match stat(self.pid) { + Ok((state, start)) => start == self.start && !matches!(state, b'Z' | b'X' | b'x'), + Err(_) => false, + } + } + /// Sends `signal` once, only after this exact process was observed alive. + pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if !self.alive() { + return Ok(()); + } + let pid = i32::try_from(self.pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width"))?; + match nix::sys::signal::kill(Pid::from_raw(pid), signal) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), + Err(errno) => Err(errno.into()), + } + } +} + +/// State and start time of `pid`. The command name may hold spaces and +/// parentheses, so fields are counted after its last `)`. +fn stat(pid: u32) -> io::Result<(u8, u64)> { + let text = std::fs::read_to_string(format!("/proc/{pid}/stat"))?; + let invalid = || io::Error::new(io::ErrorKind::InvalidData, "process stat fields"); + let rest = text.rsplit_once(')').ok_or_else(invalid)?.1; + let mut fields = rest.split_ascii_whitespace(); + let state = fields + .next() + .and_then(|s| s.bytes().next()) + .ok_or_else(invalid)?; + // field 3 was the state; field 22 is 19 fields further on + let start = fields + .nth(18) + .and_then(|s| s.parse::().ok()) + .ok_or_else(invalid)?; + Ok((state, start)) +} From 0604b72c8950834793c41e4c171f2595a092f533 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:21 +0800 Subject: [PATCH 04/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/mounts.rs (production). No tests changed or deleted. Production LOC: 108,505 -> 108,576 (delta +71) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../src/backend/local/mounts.rs | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/mounts.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/mounts.rs b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs new file mode 100644 index 0000000000..e7b84b8ded --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs @@ -0,0 +1,85 @@ +//! Kernel mounts left under a local Sandbox's directory once its daemon is +//! gone. A container's mounts end with its namespace; a local daemon's stay +//! in this one ("transport endpoint is not connected") until detached. +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// Mount points at or under `directory`, deepest first, from this +/// process's mount table. +pub(super) fn under(directory: &Path) -> io::Result> { + let table = std::fs::read_to_string("/proc/self/mountinfo")?; + let mut found = Vec::new(); + for line in table.lines() { + // field 5 is the mount point, with space, tab, newline and backslash octal-escaped + let Some(point) = line.split(' ').nth(4) else { + continue; + }; + let point = PathBuf::from(unescape(point)); + if point.starts_with(directory) { + found.push(point); + } + } + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + +fn unescape(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let octal = bytes + .get(i + 1..i + 4) + // one byte is at most octal 377, so its first digit is 0 to 3 + .filter(|digits| { + bytes[i] == b'\\' + && digits[0] <= b'3' + && digits.iter().all(|d| (b'0'..=b'7').contains(d)) + }); + match octal { + Some(digits) => { + out.push(digits.iter().fold(0u8, |n, d| n * 8 + (d - b'0'))); + i += 4; + } + None => { + out.push(bytes[i]); + i += 1; + } + } + } + String::from_utf8_lossy(&out).into_owned() +} + +/// Detaches every mount under `directory` once (lazy unmount: a process +/// still inside keeps its view; new lookups no longer reach it), then checks +/// that none is left. +#[cfg(target_os = "linux")] +pub(super) fn detach(directory: &Path) -> io::Result<()> { + use nix::mount::{umount2, MntFlags}; + for point in under(directory)? { + match umount2(&point, MntFlags::MNT_DETACH) { + Ok(()) | Err(nix::errno::Errno::EINVAL) | Err(nix::errno::Errno::ENOENT) => {} + Err(errno) => return Err(errno.into()), + } + } + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::other("a mount under the local Sandbox remains")) + } +} + +#[cfg(not(target_os = "linux"))] +pub(super) fn detach(directory: &Path) -> io::Result<()> { + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "detaching a local Sandbox mount needs Linux", + )) + } +} From 509987190f06caccfc3775fa6d60ad13aa9314fa Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:23 +0800 Subject: [PATCH 05/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/create.rs (production). No tests changed or deleted. Production LOC: 108,576 -> 108,834 (delta +258) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../src/backend/local/create.rs | 311 ++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/create.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/create.rs b/core/crates/layerfs-sandbox/src/backend/local/create.rs new file mode 100644 index 0000000000..eadff04d72 --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/create.rs @@ -0,0 +1,311 @@ +//! One local daemon start: the Sandbox's private directory, its 0600 setup, +//! one spawn and its listener line. Nothing is retried. +use super::{ + process::LocalProcess, + sandbox::{Sandbox, LOG}, +}; +use crate::{ContainerId, RuntimeError}; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Read, Seek, SeekFrom, Write}, + net::SocketAddr, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; + +/// The daemon's startup line; the address follows it. +const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; +/// Pause between two reads of a starting daemon's output. +const PAUSE: Duration = Duration::from_millis(10); +/// The daemon's environment: a search path and nothing of this process's. +const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// A local runtime root: one directory per Sandbox under it. +#[derive(Clone, Debug)] +pub struct Local { + root: PathBuf, + name: Option, +} + +/// Where one Sandbox keeps its parts, all under its directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalPaths { + /// The Sandbox directory, `/`, 0755. + pub directory: PathBuf, + /// Its daemon setup, `config/daemon.setup`, 0600 in a 0700 directory. + pub config: PathBuf, + /// Its daemon overlay template, `overlay/overlay.sqlite`, in a 0700 directory. + pub overlay: PathBuf, + /// Its Workspace mount home, `workspaces`, made 0755 by the daemon. + pub mounts: PathBuf, + /// Its daemon's output, `daemon.log`, 0600. + pub log: PathBuf, + /// Its commands' scratch directory, `tmp`, 1777 like a container's + /// `/tmp`, so what they leave there goes with the Sandbox. + pub tmp: PathBuf, +} + +/// The boundary one failed create reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LocalPhase { + /// Local checks; nothing made. + Validate, + /// The directory, setup and log file. + Create, + /// The daemon spawn. + Start, + /// The wait for the daemon's listener line. + Listener, +} + +/// A failed create. `sandbox` holds what it made (its directory, and its +/// daemon once spawned), for the caller to stop and delete once. +#[derive(Debug)] +pub struct LocalCreateFailure { + pub phase: LocalPhase, + pub sandbox: Option, + pub cause: RuntimeError, +} + +impl Local { + /// Selects `root`: an existing absolute directory owned by this user, + /// writable by it alone and traversable by others (commands run as + /// another identity and must reach their Workspace below it). + pub fn new(root: impl AsRef) -> Result { + let root = root.as_ref(); + if !root.is_absolute() + || root + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return Err(RuntimeError::Protocol("absolute local runtime root")); + } + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() + || metadata.uid() != nix::unistd::Uid::effective().as_raw() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & 0o001 == 0 + { + return Err(RuntimeError::Protocol( + "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", + )); + } + Ok(Self { + root: root.to_owned(), + name: None, + }) + } + /// Names the next Sandbox's directory `name`, under the Docker name rule + /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at + /// most 128), so it is one plain path component. + pub fn naming(mut self, name: &str) -> Result { + let bytes = name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 128 + || !bytes[0].is_ascii_alphanumeric() + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) + { + return Err(RuntimeError::Protocol("local Sandbox name")); + } + self.name = Some(name.to_owned()); + Ok(self) + } + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self) -> Result { + let name = self + .name + .as_deref() + .ok_or(RuntimeError::Protocol("local Sandbox name"))?; + let directory = self.root.join(name); + Ok(LocalPaths { + config: directory.join("config").join("daemon.setup"), + overlay: directory.join("overlay").join("overlay.sqlite"), + mounts: directory.join("workspaces"), + log: directory.join(LOG), + tmp: directory.join("tmp"), + directory, + }) + } + /// Makes the named Sandbox's directory, writes `config` (the encoded + /// daemon setup naming [`Self::paths`]) 0600, spawns `executable + /// --config ` in its own process group with no inherited + /// environment, and waits up to `wait` for its listener line. Once + /// anything is made, a failure returns it. + pub fn create( + &self, + executable: &Path, + config: &[u8], + wait: Duration, + labels: Vec<(String, String)>, + ) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(LocalCreateFailure { + phase, + sandbox, + cause, + }) + }; + let paths = self + .paths() + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let checked = (|| { + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("local daemon executable")); + } + if config.is_empty() || config.len() > 8192 || wait.is_zero() { + return Err(RuntimeError::Protocol("local daemon setup/startup wait")); + } + match fs::symlink_metadata(&paths.directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(_) => Err(RuntimeError::Protocol("local Sandbox directory exists")), + } + })(); + checked.map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted(id, paths.directory.clone(), labels); + let log = match prepare(&paths, config) { + Ok(log) => log, + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + }; + let spawned = log.try_clone().and_then(|output| { + Command::new(executable) + .arg("--config") + .arg(&paths.config) + .env_clear() + .env("PATH", PATH) + .current_dir(&paths.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + let process = match LocalProcess::observe(child.id()) { + Ok(process) => process, + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + }; + sandbox.process = Some(process); + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener(&sandbox, &paths.log, wait) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } + } +} + +/// A fresh random identity in the Engine's form (64 lowercase hex, not all +/// zero), so records and names treat a local Sandbox like a container. +fn identity() -> Result { + let mut bytes = [0u8; 32]; + File::open("/dev/urandom")?.read_exact(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let text: String = (bytes.iter()) + .flat_map(|b| [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]]) + .map(char::from) + .collect(); + ContainerId::parse(&text) +} + +/// The setup and overlay directories (0700), the scratch directory (1777), +/// the setup file (0600, written once) and the empty log file (0600) it +/// returns. +fn prepare(paths: &LocalPaths, config: &[u8]) -> io::Result { + // the process umask may narrow the create mode + fs::set_permissions(&paths.directory, Permissions::from_mode(0o755))?; + for file in [&paths.config, &paths.overlay] { + let parent = file + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "local path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.tmp)?; + fs::set_permissions(&paths.tmp, Permissions::from_mode(0o1777))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + private(&paths.config)?.write_all(config)?; + private(&paths.log) +} + +/// Waits until the daemon's output names its loopback address, the daemon +/// exits, or `wait` ends. Each look reads only what was appended since. +fn listener(sandbox: &Sandbox, log: &Path, wait: Duration) -> Result { + let deadline = Instant::now() + wait; + let mut file = File::open(log)?; + let mut offset = 0u64; + let mut line = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + file.seek(SeekFrom::Start(offset))?; + loop { + let n = file.read(&mut chunk)?; + if n == 0 { + break; + } + offset += n as u64; + for &byte in &chunk[..n] { + if byte != b'\n' { + // a startup line is short; a longer one is not the marker + if line.len() < 256 { + line.push(byte); + } + continue; + } + if let Some(address) = line.strip_prefix(MARKER) { + let address = std::str::from_utf8(address) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|a| a.ip().is_loopback() && a.port() != 0) + .ok_or(RuntimeError::Protocol("local daemon loopback listener"))?; + return Ok(address.port()); + } + line.clear(); + } + } + sandbox.reap(); + if !sandbox.process.is_some_and(LocalProcess::alive) { + return Err(RuntimeError::Protocol( + "local daemon exited before its listener line", + )); + } + if Instant::now() >= deadline { + return Err(io::Error::from(io::ErrorKind::TimedOut).into()); + } + std::thread::sleep(PAUSE); + } +} From 12a734abedb30abbe6f8eec79e19cadf502f3add Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:25 +0800 Subject: [PATCH 06/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/sandbox.rs (production). No tests changed or deleted. Production LOC: 108,834 -> 109,057 (delta +223) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../src/backend/local/sandbox.rs | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/sandbox.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs new file mode 100644 index 0000000000..51c1bd26a7 --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs @@ -0,0 +1,259 @@ +//! One local Sandbox: its directory, its daemon process and its exact +//! attempts. No Drop cleanup or replay, as for a container. +use super::{mounts, process::LocalProcess}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use nix::sys::signal::Signal; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The daemon's output file in a Sandbox directory, stdout and stderr together. +pub(super) const LOG: &str = "daemon.log"; +/// Pause between two looks at a stopping process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for a killed process to be gone. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged local Sandbox directory and, once started, its daemon. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) directory: PathBuf, + pub(super) labels: Vec<(String, String)>, + pub(super) process: Option, + pub(super) port: u16, + /// The daemon this process started, kept so its exit is reaped here and + /// it does not stay a zombie of this process. `None` after a reopen. + pub(super) child: Mutex>, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// A directory made by a start that has not spawned its daemon yet. + pub(super) fn unstarted( + id: ContainerId, + directory: PathBuf, + labels: Vec<(String, String)>, + ) -> Self { + Self { + id, + directory, + labels, + process: None, + port: 0, + child: Mutex::new(None), + stop_attempted: false, + stopped: false, + delete_attempted: false, + deleted: false, + } + } + /// Reopens a recorded Sandbox whose daemon still runs: the same process + /// (id and start time) and its loopback port. It starts nothing. + pub fn reopen( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Result> { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + if !owner.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved local Sandbox")); + } + owner.endpoint()?; + Ok(owner) + } + /// A recorded Sandbox that [`Self::reopen`] found stopped + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub fn stopped( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Self { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + /// Its private directory (setup, overlay, Workspace mounts, daemon log). + pub fn directory(&self) -> &Path { + &self.directory + } + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + self.directory.join("tmp") + } + /// Its daemon process, once started. + pub fn process(&self) -> Option { + self.process + } + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + /// The daemon's endpoint while that exact process runs. A daemon that + /// exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(process) = self.process else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !process.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + /// Stops the daemon once: TERM, up to `seconds` for it to end, then + /// KILL. A daemon already gone is the same stopped state. Crash + /// teardown, not a graceful filesystem drain. + pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original local Stop admission")); + } + self.stop_attempted = true; + if let Some(process) = self.process { + self.end( + process, + Signal::SIGTERM, + Duration::from_secs(u64::from(seconds)), + )?; + if process.alive() { + self.end(process, Signal::SIGKILL, KILL_WAIT)?; + } + if process.alive() { + return Err(self.invalid("local daemon still runs after KILL")); + } + } + self.stopped = true; + Ok(()) + } + /// Deletes its directory once, after a known stop: any mount its daemon + /// left is detached first, so nothing is removed through a mount. The + /// shared Store is outside it and never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.process.is_some() && !self.stopped) { + return Err(self.invalid("original local Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.process.is_some_and(LocalProcess::alive) { + return Err(self.invalid("local daemon runs again; nothing was deleted")); + } + mounts::detach(&self.directory).map_err(|error| self.io(error))?; + match fs::remove_dir_all(&self.directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(self.directory.join(LOG))?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + /// Reaps the daemon this process started, if it exited. + pub(super) fn reap(&self) { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + if matches!(started.try_wait(), Ok(Some(_))) { + *child = None; + } + } + } + /// Sends `signal` once and waits up to `wait` for the process to end. + fn end( + &self, + process: LocalProcess, + signal: Signal, + wait: Duration, + ) -> Result<(), Box> { + process.signal(signal).map_err(|error| self.io(error))?; + let deadline = Instant::now() + wait; + loop { + self.reap(); + if !process.alive() || Instant::now() >= deadline { + return Ok(()); + } + std::thread::sleep(PAUSE); + } + } + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} From e20bb9348c8799a5491cc3a10176ce36251baa31 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:27 +0800 Subject: [PATCH 07/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/local/exec.rs (production). No tests changed or deleted. Production LOC: 109,057 -> 109,166 (delta +109) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../layerfs-sandbox/src/backend/local/exec.rs | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/exec.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/exec.rs b/core/crates/layerfs-sandbox/src/backend/local/exec.rs new file mode 100644 index 0000000000..0673564715 --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/exec.rs @@ -0,0 +1,130 @@ +//! One ordinary command to its end in a local Sandbox: the command identity, +//! the Workspace directory and a search path; nothing of this process's +//! environment, no shell of its own and no implicit timeout. +use super::sandbox::Sandbox; +use crate::{backend::docker::ExecRequest, RuntimeError, WireFailure}; +use std::{ + io::{self, Read, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +/// The search path a command starts with, before its own environment. +const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// One command's end: its exit code (128 + the signal for one a signal +/// ended, as the Engine reports it), its process id (diagnostic only) and the +/// bytes each stream delivered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalExit { + pub code: i32, + pub pid: u32, + pub stdout: u64, + pub stderr: u64, +} + +impl Sandbox { + /// Runs `request` once while this Sandbox's daemon runs: as its uid/gid + /// with no supplementary groups, in its directory, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. Standard input is not offered. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + self.endpoint()?; + let mut command = Command::new(&request.arguments[0]); + command + .args(&request.arguments[1..]) + .env_clear() + .env("PATH", PATH) + .current_dir(Path::new(&request.directory)) + // std drops supplementary groups (setgroups 0) when it sets a uid as root + .uid(request.identity.uid) + .gid(request.identity.gid) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || copy(err, stderr)); + let outs = copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("local command exit status"))?; + Ok(LocalExit { + code, + pid, + stdout: out_bytes, + stderr: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} + +/// Copies one stream to its sink until end of stream; the bytes delivered. +fn copy(source: Option, sink: &mut (dyn Write + Send)) -> io::Result { + let Some(mut source) = source else { + return Ok(0); + }; + let mut total = 0u64; + let mut window = [0u8; 8192]; + loop { + let n = match source.read(&mut window) { + Ok(0) => return Ok(total), + Ok(n) => n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + sink.write_all(&window[..n])?; + total = total.saturating_add(n as u64); + } +} From c5e150253ff6e25f058479f929709e1087a25a9b Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:29 +0800 Subject: [PATCH 08/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/src/backend/mod.rs (production). No tests changed or deleted. Production LOC: 109,166 -> 109,167 (delta +1) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- core/crates/layerfs-sandbox/src/backend/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/crates/layerfs-sandbox/src/backend/mod.rs b/core/crates/layerfs-sandbox/src/backend/mod.rs index 509d7e29fa..7fe356b929 100644 --- a/core/crates/layerfs-sandbox/src/backend/mod.rs +++ b/core/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,2 +1,3 @@ //! Concrete ordinary runtime providers. pub mod docker; +pub mod local; From f8324908284c6ab200cdfb902156af90e03abf86 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:31 +0800 Subject: [PATCH 09/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: core/crates/layerfs-sandbox/tests/local_runtime.rs (not production (test, manifest or docs)). No tests changed or deleted. Production LOC: 109,167 -> 109,167 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- .../layerfs-sandbox/tests/local_runtime.rs | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 core/crates/layerfs-sandbox/tests/local_runtime.rs diff --git a/core/crates/layerfs-sandbox/tests/local_runtime.rs b/core/crates/layerfs-sandbox/tests/local_runtime.rs new file mode 100644 index 0000000000..f90e682433 --- /dev/null +++ b/core/crates/layerfs-sandbox/tests/local_runtime.rs @@ -0,0 +1,413 @@ +//! The local process runtime (`backend::local`): one daemon process per +//! Sandbox directory, observed by process id and start time, its commands run +//! as their own identity, and its stop and delete each attempted once. A +//! small shell script stands in for the daemon; nothing here needs root. +use layerfs_sandbox::{ + backend::{ + docker::ExecRequest, + local::{Local, LocalPhase, LocalProcess, Sandbox}, + }, + CommandIdentity, RuntimeError, +}; +use std::{ + fs, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; + +static NEXT: AtomicUsize = AtomicUsize::new(0); +const WAIT: Duration = Duration::from_secs(10); + +/// A fresh 0755 runtime root with a stand-in daemon `daemon.sh` beside it. +fn fixture(script: &str) -> (PathBuf, PathBuf) { + let base = std::env::temp_dir().join(format!( + "layerfs-local-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("root")).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + let daemon = base.join("daemon.sh"); + fs::write(&daemon, format!("#!/bin/sh\n{script}\n")).unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + (base.join("root"), daemon) +} + +/// Prints the listener line after checking its arguments, then waits. +const LISTENS: &str = r#"[ "$1" = --config ] && [ -f "$2" ] || exit 3 +env >&2 +echo "starting" +echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" +exec sleep 60"#; + +/// The command identity: this user, or nobody when the tests run as root. +fn me() -> CommandIdentity { + let meta = fs::metadata("/proc/self").unwrap(); + if meta.uid() == 0 { + CommandIdentity { + uid: 65534, + gid: 65534, + } + } else { + CommandIdentity { + uid: meta.uid(), + gid: meta.gid(), + } + } +} + +fn mode(path: &Path) -> u32 { + fs::symlink_metadata(path).unwrap().mode() & 0o7777 +} + +fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { + Local::new(root) + .unwrap() + .naming(name) + .unwrap() + .create( + daemon, + b"setup bytes", + WAIT, + vec![("layerfs.role".into(), "sandbox".into())], + ) + .unwrap() +} + +fn run(sandbox: &Sandbox, arguments: &[&str], environment: &[&str]) -> (i32, String, String) { + let request = ExecRequest { + identity: me(), + arguments: arguments.iter().map(|a| a.to_string()).collect(), + environment: environment.iter().map(|a| a.to_string()).collect(), + directory: sandbox.directory().display().to_string(), + stdin: false, + }; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + assert_eq!(exit.stdout, out.len() as u64); + assert_eq!(exit.stderr, err.len() as u64); + ( + exit.code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +#[test] +fn a_root_must_be_absolute_owned_private_to_write_and_traversable() { + let (root, _) = fixture("exit 0"); + assert!(matches!( + Local::new("relative/root"), + Err(RuntimeError::Protocol(_)) + )); + assert!(matches!( + Local::new(root.join("..").join("root")), + Err(RuntimeError::Protocol(_)) + )); + assert!(matches!( + Local::new(root.join("missing")), + Err(RuntimeError::Io(_)) + )); + fs::set_permissions(&root, fs::Permissions::from_mode(0o775)).unwrap(); + assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let local = Local::new(&root).unwrap(); + for bad in ["", ".hidden", "-x", "a/b", "..", "a b", &"x".repeat(129)] { + assert!(local.clone().naming(bad).is_err(), "{bad:?} admitted"); + } + assert!(local.paths().is_err(), "paths before a name"); + let paths = local.naming("layerfs-shop-1").unwrap().paths().unwrap(); + assert_eq!(paths.directory, root.join("layerfs-shop-1")); + assert_eq!(paths.config, paths.directory.join("config/daemon.setup")); + assert_eq!( + paths.overlay, + paths.directory.join("overlay/overlay.sqlite") + ); + assert_eq!(paths.mounts, paths.directory.join("workspaces")); + assert_eq!(paths.log, paths.directory.join("daemon.log")); + assert_eq!(paths.tmp, paths.directory.join("tmp")); +} + +#[test] +fn a_start_writes_a_private_setup_spawns_once_and_reads_the_listener() { + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-1"); + let directory = root.join("layerfs-shop-1"); + assert_eq!(sandbox.directory(), directory); + assert_eq!(sandbox.port(), 40404); + assert_eq!( + sandbox.endpoint().unwrap(), + "127.0.0.1:40404".parse().unwrap() + ); + assert_eq!( + sandbox.labels(), + [("layerfs.role".to_owned(), "sandbox".to_owned())] + ); + let process = sandbox.process().unwrap(); + assert!(process.alive()); + assert_eq!(LocalProcess::observe(process.pid).unwrap(), process); + assert!(!LocalProcess { + start: process.start + 1, + ..process + } + .alive()); + // private setup, private overlay home, traversable Sandbox directory + assert_eq!(mode(&directory), 0o755); + assert_eq!(mode(&directory.join("config")), 0o700); + assert_eq!(mode(&directory.join("config/daemon.setup")), 0o600); + assert_eq!( + fs::read(directory.join("config/daemon.setup")).unwrap(), + b"setup bytes" + ); + assert_eq!(mode(&directory.join("overlay")), 0o700); + assert_eq!(mode(&directory.join("daemon.log")), 0o600); + assert_eq!(mode(&directory.join("tmp")), 0o1777); + assert_eq!(sandbox.tmp(), directory.join("tmp")); + // the daemon inherits nothing of this process's environment but PATH + let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); + assert!(log.contains("PATH=/usr/local/sbin:"), "{log}"); + assert!(!log.contains("CARGO"), "{log}"); + assert_eq!( + String::from_utf8(sandbox.log_tail(1, 4096).unwrap()).unwrap(), + "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\n" + ); + assert_eq!(sandbox.log_tail(1, 5).unwrap(), b"0404\n"); + // the same name again is refused before anything is made + let again = Local::new(&root) + .unwrap() + .naming("layerfs-shop-1") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(again.phase, LocalPhase::Validate); + assert!(again.sandbox.is_none()); + // delete needs a known stop; stop ends the process; each is attempted once + assert!(sandbox.delete().is_err()); + let mut sandbox = Sandbox::reopen( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ) + .unwrap(); + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + assert!(sandbox.stop(1).is_err()); + assert!(sandbox.endpoint().is_err()); + sandbox.delete().unwrap(); + assert!(!directory.exists()); + assert!(sandbox.delete().is_err()); +} + +#[test] +fn a_command_runs_as_its_identity_in_its_directory_with_only_its_environment() { + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-2"); + let identity = me(); + let (code, out, err) = run( + &sandbox, + &[ + "bash", + "-c", + "id -u; id -g; pwd; echo \"$HOME|${CARGO:-none}\"; id -G >&2; echo oops >&2; exit 7", + ], + &["HOME=/tmp"], + ); + assert_eq!(code, 7); + assert_eq!( + out, + format!( + "{}\n{}\n{}\n/tmp|none\n", + identity.uid, + identity.gid, + sandbox.directory().display() + ) + ); + let (groups, err) = err.split_once('\n').unwrap(); + assert_eq!(err, "oops\n"); + // as root, the command keeps no supplementary group of this process + if fs::metadata("/proc/self").unwrap().uid() == 0 { + assert_eq!(groups, identity.gid.to_string()); + } + // a command a signal ended reports 128 + the signal, as the Engine does + let (code, _, _) = run(&sandbox, &["bash", "-c", "kill -9 $$"], &[]); + assert_eq!(code, 137); + // more output than one pipe holds, on both streams at once + let (code, out, err) = run( + &sandbox, + &[ + "bash", + "-c", + "head -c 300000 /dev/zero | tr '\\0' a; head -c 200000 /dev/zero | tr '\\0' b >&2", + ], + &[], + ); + assert_eq!((code, out.len(), err.len()), (0, 300000, 200000)); + // refused before any effect: root, stdin, a relative directory, a bad variable + let mut request = ExecRequest { + identity, + arguments: vec!["true".into()], + environment: Vec::new(), + directory: "/".into(), + stdin: false, + }; + for change in [ + (|r: &mut ExecRequest| r.identity = CommandIdentity { uid: 0, gid: 0 }) + as fn(&mut ExecRequest), + |r| r.stdin = true, + |r| r.directory = "relative".into(), + |r| r.environment = vec!["NOEQUALS".into()], + |r| r.arguments = Vec::new(), + ] { + let mut changed = request.clone(); + change(&mut changed); + let failure = sandbox + .run(&changed, &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(!failure.attempted, "{failure:?}"); + } + sandbox.stop(1).unwrap(); + // no command once the daemon is stopped + request.directory = "/".into(); + assert!(sandbox + .run(&request, &mut Vec::new(), &mut Vec::new()) + .is_err()); + sandbox.delete().unwrap(); +} + +#[test] +fn a_daemon_that_ends_is_the_stopped_observation_and_only_its_delete_remains() { + let (root, daemon) = fixture(LISTENS); + let sandbox = started(&root, &daemon, "layerfs-shop-3"); + let process = sandbox.process().unwrap(); + let directory = sandbox.directory().to_owned(); + // killed from outside, as a crash or the OOM killer would + assert!(Command::new("kill") + .args(["-9", &process.pid.to_string()]) + .status() + .unwrap() + .success()); + let deadline = Instant::now() + WAIT; + let failure = loop { + match sandbox.endpoint() { + Err(failure) => break failure, + Ok(_) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(20)), + Ok(_) => panic!("killed daemon still observed running"), + } + }; + assert!(failure.container_stopped(), "{failure:?}"); + assert!(!failure.container_gone()); + // this process reaped its child: no zombie remains + let stat = fs::read_to_string(format!("/proc/{}/stat", process.pid)); + assert!(stat.is_err() || !stat.unwrap().contains(") Z ")); + // a reopen from the record observes the same + let reopened = Sandbox::reopen( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ) + .unwrap_err(); + assert!(reopened.container_stopped()); + let mut stopped = Sandbox::stopped( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ); + assert!(stopped.endpoint().is_err()); + stopped.delete().unwrap(); + assert!(!directory.exists()); +} + +#[test] +fn a_failed_start_returns_what_it_made() { + // exits before its listener line + let (root, daemon) = fixture("echo broken >&2; exit 9"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-4") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + let mut sandbox = failure.sandbox.unwrap(); + assert_eq!(sandbox.log_tail(5, 100).unwrap(), b"broken\n"); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-4").exists()); + // never prints it: the wait ends + let (root, daemon) = fixture("exec sleep 60"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-5") + .unwrap() + .create(&daemon, b"x", Duration::from_millis(300), Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + assert!( + matches!(&failure.cause, RuntimeError::Io(e) if e.kind() == std::io::ErrorKind::TimedOut) + ); + let mut sandbox = failure.sandbox.unwrap(); + let process = sandbox.process().unwrap(); + assert!(process.alive()); + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + sandbox.delete().unwrap(); + // a listener that is not loopback is refused + let (root, daemon) = fixture("echo 'LAYERFS_DAEMON_LISTEN 0.0.0.0:5'; exec sleep 60"); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-6") + .unwrap() + .create(&daemon, b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!(failure.phase, LocalPhase::Listener); + let mut sandbox = failure.sandbox.unwrap(); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + // a missing executable or an empty setup makes nothing + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-7") + .unwrap() + .create(&root.join("missing"), b"x", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!( + (failure.phase, failure.sandbox.is_none()), + (LocalPhase::Validate, true) + ); + let failure = Local::new(&root) + .unwrap() + .naming("layerfs-shop-7") + .unwrap() + .create(&daemon, b"", WAIT, Vec::new()) + .unwrap_err(); + assert_eq!( + (failure.phase, failure.sandbox.is_none()), + (LocalPhase::Validate, true) + ); + assert!(!root.join("layerfs-shop-7").exists()); +} + +#[test] +fn a_zombie_is_not_alive() { + let mut child = Command::new("true").spawn().unwrap(); + let process = LocalProcess::observe(child.id()).unwrap(); + let deadline = Instant::now() + WAIT; + while process.alive() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process.alive(), "an exited, unreaped child counted alive"); + child.wait().unwrap(); +} From 726869a14b779e393309a4eaf814675cad3fb731 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:33 +0800 Subject: [PATCH 10/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/runtime.rs (production). No tests changed or deleted. Production LOC: 109,167 -> 109,274 (delta +107) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/runtime.rs | 140 +++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 sdk/js/native/src/runtime.rs diff --git a/sdk/js/native/src/runtime.rs b/sdk/js/native/src/runtime.rs new file mode 100644 index 0000000000..ef9737c76e --- /dev/null +++ b/sdk/js/native/src/runtime.rs @@ -0,0 +1,140 @@ +//! A Sandbox's runtime: a Docker container (the default), or with +//! `deploy.sandbox` `local` (LAYERFS_SANDBOX=local) a daemon process on this +//! kernel, for a VM or CI box that is itself the isolation boundary. Both run +//! the same daemon and answer the same calls; only how it is started, how +//! commands are launched and how it is observed differ. +use crate::job::{failed, refused}; +use crate::output::Spool; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::backend::{docker, local}; +use layerfs_sandbox::{CommandIdentity, ContainerId, WireFailure}; +use std::net::SocketAddr; + +/// The image a local Sandbox records: placement keeps local and Docker +/// Sandboxes of one project apart by it. +pub(crate) const LOCAL: &str = "local"; + +/// One Sandbox's runtime owner. +pub(crate) enum Runtime { + Docker(docker::Sandbox), + Local(local::Sandbox), +} + +/// Whether the deployment selects the local runtime: `deploy.sandbox` unset +/// or `docker` is Docker, `local` is local; anything else is refused before +/// any effect. +pub(crate) fn local(deploy: &SdkValue) -> Result { + match deploy.get_text("sandbox") { + None | Some("docker") => Ok(false), + Some("local") => Ok(true), + Some(other) => Err(refused( + "InvalidInput", + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or local, not {other}"), + )), + } +} + +/// The image a new Workspace of this deployment is placed by: `local` for +/// the local runtime, else `deploy.image`. +pub(crate) fn image(args: &SdkValue) -> Result, SdkRefusal> { + let Some(deploy) = args.get("deploy") else { + return Ok(None); + }; + Ok(if local(deploy)? { + Some(LOCAL.to_owned()) + } else { + deploy.get_text("image").map(str::to_owned) + }) +} + +impl Runtime { + /// The recorded identity: the container's, or a local Sandbox's own. + pub(crate) fn identity(&self) -> ContainerId { + match self { + Self::Docker(sandbox) => sandbox.identity(), + Self::Local(sandbox) => sandbox.identity(), + } + } + /// The image it runs; `local` for a local Sandbox. + pub(crate) fn image(&self) -> &str { + match self { + Self::Docker(sandbox) => &sandbox.request().image, + Self::Local(_) => LOCAL, + } + } + /// Its daemon's endpoint while it runs, observed once; a container or + /// daemon found gone or stopped is [`WireFailure::container_gone`] or + /// [`WireFailure::container_stopped`]. + pub(crate) fn endpoint(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.endpoint(), + Self::Local(sandbox) => sandbox.endpoint(), + } + } + /// Where its commands keep scratch files: `/tmp` in a container, the + /// Sandbox's own `tmp` directory for a local one (deleted with it), so + /// a local Sandbox leaves nothing in this machine's `/tmp`. + pub(crate) fn scratch(&self) -> String { + match self { + Self::Docker(_) => "/tmp".to_owned(), + Self::Local(sandbox) => sandbox.tmp().display().to_string(), + } + } + /// Deletes a Sandbox found stopped, once. + pub(crate) fn delete(&mut self) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.delete(), + Self::Local(sandbox) => sandbox.delete(), + } + } + /// One command to its end: exit code, stdout, stderr. `spool` sends + /// every non-empty stream to its file instead of keeping it inline. + pub(crate) fn run( + &self, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, + spool: bool, + ) -> Result<(i32, Spool, Spool), SdkRefusal> { + let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); + let code = match self { + Self::Docker(sandbox) => { + let created = layerfs_api::internal::exec( + sandbox, + identity, + arguments, + environment, + directory.to_owned(), + false, + ) + .map_err(failed)?; + let attached = created.start().map_err(failed)?; + let (_input, output, handle) = attached.into_parts().map_err(failed)?; + output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; + let inspected = handle.inspect().map_err(failed)?; + inspected + .known_root_exit() + .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))? + } + Self::Local(sandbox) => { + let request = docker::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + sandbox + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)? + .code + } + }; + Ok((code, stdout, stderr)) + } +} + +#[cfg(test)] +#[path = "../tests/unit/runtime.rs"] +mod runtime_tests; From 13d863124653b53a86e118abb8da72828f121afe Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:35 +0800 Subject: [PATCH 11/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/local.rs (production). No tests changed or deleted. Production LOC: 109,274 -> 109,527 (delta +253) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/local.rs | 288 +++++++++++++++++++++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 sdk/js/native/src/local.rs diff --git a/sdk/js/native/src/local.rs b/sdk/js/native/src/local.rs new file mode 100644 index 0000000000..4ca4201219 --- /dev/null +++ b/sdk/js/native/src/local.rs @@ -0,0 +1,288 @@ +//! Local Sandboxes (`deploy.sandbox` `local`, LAYERFS_SANDBOX=local): the +//! project's daemon runs as a process of this kernel instead of in a +//! container, for a VM or CI box that is itself the isolation boundary. It +//! isolates nothing from the rest of that machine. +//! +//! Each Sandbox gets a directory `/-` +//! with what a container held at fixed paths: its setup +//! (`/layerfs-local/config`), overlay template (`/layerfs-local/overlay`) and +//! Workspace mounts (`/workspaces`). The daemon opens the installed Store +//! where this process opens it (`storePath`), so the daemon setup and the +//! Store's locator name that path instead of `/layerfs-store`. The daemon +//! listens on 127.0.0.1 only. Commands run as the same nonroot identity as +//! in a container. Start, failure and reclaim follow the Docker route: one +//! attempt, a creation intent while the outcome is open, and a failed start's +//! daemon stopped and its directory deleted, each once. +use crate::job::{failed, refused}; +use crate::registry::{lock, text, SandboxState, SANDBOXES}; +use crate::runtime::Runtime; +use layerfs_api::internal::Control; +use layerfs_bridge::control::{DaemonPhase, HelloRequest, Reply, Request, SdkRefusal, SdkValue}; +use layerfs_bridge::duplex::initiate_observed; +use layerfs_bridge::provision::StoreManifest; +use layerfs_sandbox::backend::local::{Local, LocalCreateFailure, Sandbox}; +use layerfs_sandbox::CommandIdentity; +use std::net::{SocketAddr, TcpStream}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// Starts the project's next Sandbox as a local daemon. Called by +/// [`crate::sandbox::start`] under the project's guard, with the project +/// and its installed manifest already read. +pub(crate) fn start( + args: &SdkValue, + directory: &Path, + project: (&crate::registry::Project, StoreManifest), +) -> Result, SdkRefusal> { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + // Inputs are read first, so a refused request leaves no creation intent. + if !crate::discovery::root() { + return Err(refused( + "Unsupported", + "LAYERFS_SANDBOX=local needs root: its daemon mounts FUSE and its commands run as \ + uid 501, gid 20. Run it as root inside the VM or CI box that isolates it", + )); + } + let root = deploy.get_text("localDir").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a local Sandbox needs it", + ) + })?; + let local = Local::new(root) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let executable = PathBuf::from(text(deploy, "daemon")?); + let store = match args.get_text("storePath") { + Some(path) if Path::new(path).is_absolute() => path.to_owned(), + _ => { + return Err(refused( + "InvalidInput", + "a local Sandbox's daemon opens the client's storePath (LAYERFS_STORE_PATH), an absolute path", + )) + } + }; + crate::sandbox::unresolved( + directory, + &format!("a directory under {root} named for it, and its daemon process"), + )?; + let (id, saved_directory) = crate::discovery::claim_next(args)?; + let made = start_claimed( + args, + directory, + &saved_directory, + &id, + project, + (local, executable, store), + ); + if made.is_err() { + let _ = std::fs::remove_dir(&saved_directory); + } + made +} + +/// Starts the claimed Sandbox `id`: directory and daemon on the installed +/// Store, its first control connection, then its records. +fn start_claimed( + args: &SdkValue, + directory: &Path, + saved_directory: &Path, + id: &str, + (project, installed): (&crate::registry::Project, StoreManifest), + (local, executable, store): (Local, PathBuf, String), +) -> Result, SdkRefusal> { + let name = text(args, "project")?; + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + let sandbox_name = format!("{prefix}{name}-{id}"); + let local = local + .naming(&sandbox_name) + .map_err(|_| refused("InvalidInput", format!("local Sandbox name {sandbox_name}")))?; + let paths = local.paths().map_err(failed)?; + let path_text = |path: &Path| { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_LOCAL_DIR must be UTF-8")) + }; + let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let mut setup = crate::sandbox::setup(daemon.private, controller.public); + setup.listen = "127.0.0.1:0".into(); + setup.store = store.clone(); + setup.overlay = path_text(&paths.overlay)?; + setup.mounts = path_text(&paths.mounts)?; + let mut existing = project.manifest.clone(); + existing.daemon_sqlite = installed.daemon_sqlite; + // the Store's path in the daemon's filesystem, which is this one + existing.locator = store; + setup.existing_store = Some(existing); + let identity = CommandIdentity { + uid: setup.command_uid, + gid: setup.command_gid, + }; + let config = setup.encode().map_err(failed)?; + // Read only if this process ends before the intent is resolved. + crate::discovery::save( + &directory.join("create.intent"), + SdkValue::text(format!("creation of Sandbox {id} began and never finished")), + )?; + let labels = crate::sandbox::labels(name, id); + let sandbox = local + .create(&executable, &config, crate::sandbox::STARTUP, labels) + .map_err(|failure| { + let LocalCreateFailure { + phase, + sandbox, + cause, + } = *failure; + abandon( + directory, + id, + format!("local sandbox create stopped at {phase:?}: {cause:?}"), + sandbox, + ) + })?; + let first = sandbox.endpoint().map_err(failed).and_then(|endpoint| { + first(endpoint, controller.private, daemon.public).map(|c| (endpoint, c)) + }); + let (endpoint, (control, instance)) = match first { + Ok(opened) => opened, + Err(refusal) => { + return Err(abandon( + directory, + id, + format!("local sandbox first connection: {}", refusal.message), + Some(sandbox), + )) + } + }; + let dial = crate::pool::Dial { + endpoint, + host: None, + secret: controller.private, + peer: daemon.public, + instance, + }; + let state = Arc::new(SandboxState { + id: id.to_owned(), + project: name.to_owned(), + name: sandbox_name, + directory: saved_directory.to_owned(), + runtime: Runtime::Local(sandbox), + identity, + controls: crate::pool::Controls::new(control, dial), + mounts: Default::default(), + processes: Default::default(), + }); + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed)?; + lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); + Ok(state) +} + +/// The first control connection to a new daemon: one connect, one +/// authentication, one Hello that waits for the Store and must find the +/// daemon ControlReady; its instance identifies the daemon from then on. +fn first( + endpoint: SocketAddr, + secret: [u8; 32], + peer: [u8; 32], +) -> Result<(Control, [u8; 32]), SdkRefusal> { + let wait = crate::sandbox::STARTUP; + let stream = TcpStream::connect_timeout(&endpoint, wait).map_err(failed)?; + stream.set_read_timeout(Some(wait)).map_err(failed)?; + stream.set_write_timeout(Some(wait)).map_err(failed)?; + let connection = initiate_observed(stream, &secret, peer) + .result + .map_err(failed)?; + connection.clear_io_waits().map_err(failed)?; + let mut control = Control::new(connection); + match control + .call(Request::Hello(HelloRequest { + expected_instance: None, + wait_for_store: true, + })) + .map_err(failed)? + { + Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => { + Ok((control, status.instance)) + } + reply => Err(refused( + "Failed", + format!("the new daemon is not ControlReady: {reply:?}"), + )), + } +} + +/// The refusal for a failed start. Everything a local start makes is its +/// own (a directory and a child process), so each effect is known: the +/// daemon's last log lines are read once, then it is stopped and its +/// directory deleted, each once, and the intent removed. If that cleanup +/// fails, the intent stays and says what is left. +fn abandon(directory: &Path, id: &str, mut cause: String, sandbox: Option) -> SdkRefusal { + let container = sandbox.as_ref().map(|s| s.identity().to_string()); + let mut log = None; + let mut left = None; + if let Some(mut sandbox) = sandbox { + match sandbox.log_tail(crate::sandbox::LOG_LINES, crate::sandbox::LOG_BYTES) { + Ok(bytes) => { + let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); + cause = format!("{cause}; its daemon's last log lines:\n{text}"); + log = Some(text); + } + Err(error) => cause = format!("{cause}; its daemon log was unreadable: {error:?}"), + } + let removed = match sandbox.process() { + Some(_) => sandbox.stop(1).and_then(|()| sandbox.delete()), + None => sandbox.delete(), + }; + left = removed.err().map(|error| { + format!( + "its local Sandbox {} could not be removed: {error:?}", + sandbox.directory().display() + ) + }); + } + let intent = directory.join("create.intent"); + let (message, removed) = match left { + None => match std::fs::remove_file(&intent) { + Ok(()) => (cause, container.is_some()), + Err(error) => ( + format!("{cause}; its creation intent stays: {error}"), + container.is_some(), + ), + }, + Some(left) => { + let state = format!("creation of Sandbox {id} failed ({cause}); {left}"); + let kept = crate::discovery::save(&intent, SdkValue::text(&state)) + .err() + .map(|error| format!(" (recording it failed: {})", error.message)) + .unwrap_or_default(); + (format!("{state}; its creation intent stays{kept}"), false) + } + }; + SdkRefusal { + code: "Failed".to_owned(), + message, + detail: SdkValue::map([ + ("container", SdkValue::opt_text(container)), + ("removed", SdkValue::Bool(removed)), + ("log", SdkValue::opt_text(log)), + ]), + } +} From eda15378d750f8ae1f80efb4f2edebf80335828e Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:37 +0800 Subject: [PATCH 12/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/registry.rs (production). No tests changed or deleted. Production LOC: 109,527 -> 109,527 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/registry.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/js/native/src/registry.rs b/sdk/js/native/src/registry.rs index aeeb25bc47..5f1e902287 100644 --- a/sdk/js/native/src/registry.rs +++ b/sdk/js/native/src/registry.rs @@ -2,7 +2,8 @@ //! project itself is only its home entry (`//`), so a project //! removed there can be initialized again without a restart. //! -//! A Sandbox is one container and its daemon. Its runtime runs commands +//! A Sandbox is one container and its daemon (or, with the local runtime, +//! one daemon process: `runtime`). Its runtime runs commands //! directly; its control connections (`pool`) each carry one exchange at a //! time, and concurrent calls take separate connections. The Sandbox runtime owner (this //! binding) also keeps the background processes it started; the daemon holds @@ -11,7 +12,7 @@ use crate::job::refused; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; +use layerfs_sandbox::CommandIdentity; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -42,10 +43,11 @@ pub(crate) struct Process { pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, - /// Its container's name, `-`. + /// Its container's name, `-` (a local Sandbox's + /// directory name). pub(crate) name: String, pub(crate) directory: std::path::PathBuf, - pub(crate) runtime: Sandbox, + pub(crate) runtime: crate::runtime::Runtime, pub(crate) identity: CommandIdentity, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, From 4f0ce6dddb7ffc5a149aedba76d0e05fc484db96 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:39 +0800 Subject: [PATCH 13/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/discovery.rs (production). No tests changed or deleted. Production LOC: 109,527 -> 109,644 (delta +117) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/discovery.rs | 227 +++++++++++++++++++++++++-------- 1 file changed, 177 insertions(+), 50 deletions(-) diff --git a/sdk/js/native/src/discovery.rs b/sdk/js/native/src/discovery.rs index ff9c77f1c8..7862715723 100644 --- a/sdk/js/native/src/discovery.rs +++ b/sdk/js/native/src/discovery.rs @@ -1,4 +1,5 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +use crate::runtime::Runtime; use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, @@ -11,7 +12,8 @@ use layerfs_bridge::{ use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; use layerfs_sandbox::{ backend::docker::{Docker, Sandbox, SandboxRequest}, - CommandIdentity, ContainerId, + backend::local::{self, LocalProcess}, + CommandIdentity, ContainerId, WireFailure, }; use std::{ fs::{self, File, OpenOptions}, @@ -68,6 +70,11 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } +/// Whether this process runs as root, which the local runtime needs: its +/// daemon mounts FUSE and its commands run as another uid/gid. +pub(crate) fn root() -> bool { + unsafe { geteuid() == 0 } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -339,39 +346,64 @@ pub(crate) fn save_sandbox( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "deploy"))?; - save( - &directory.join("sandbox.record"), - SdkValue::map([ - ("id", SdkValue::text(&state.id)), - ("project", SdkValue::text(&state.project)), - ("name", SdkValue::text(&state.name)), - ( - "container", - SdkValue::text(state.runtime.identity().to_string()), - ), + let labels = |labels: &[(String, String)]| { + SdkValue::List( + (labels.iter()) + .map(|(key, value)| { + SdkValue::map([ + ("key", SdkValue::text(key)), + ("value", SdkValue::text(value)), + ]) + }) + .collect(), + ) + }; + let mut fields = vec![ + ("id", SdkValue::text(&state.id)), + ("project", SdkValue::text(&state.project)), + ("name", SdkValue::text(&state.name)), + ( + "container", + SdkValue::text(state.runtime.identity().to_string()), + ), + ]; + match &state.runtime { + Runtime::Docker(runtime) => fields.extend([ ("docker", SdkValue::text(registry::text(deploy, "docker")?)), - ("image", SdkValue::text(&state.runtime.request().image)), - ( - "volume", - SdkValue::text(&state.runtime.request().store_volume), - ), - ( - "labels", - SdkValue::List( - (state.runtime.request().labels.iter()) - .map(|(key, value)| { - SdkValue::map([ - ("key", SdkValue::text(key)), - ("value", SdkValue::text(value)), - ]) - }) - .collect(), - ), - ), - ("secret", SdkValue::Bytes(secret.to_vec())), - ("peer", SdkValue::Bytes(peer.to_vec())), - ("instance", SdkValue::Bytes(instance.to_vec())), + ("image", SdkValue::text(&runtime.request().image)), + ("volume", SdkValue::text(&runtime.request().store_volume)), + ("labels", labels(&runtime.request().labels)), ]), + Runtime::Local(runtime) => { + let process = runtime + .process() + .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + fields.extend([ + ("runtime", SdkValue::text(crate::runtime::LOCAL)), + ( + "directory", + SdkValue::text(runtime.directory().display().to_string()), + ), + ("pid", SdkValue::Int(i64::from(process.pid))), + ("start", SdkValue::text(process.start.to_string())), + ("port", SdkValue::Int(i64::from(runtime.port()))), + ("image", SdkValue::text(crate::runtime::LOCAL)), + ("labels", labels(runtime.labels())), + ]); + } + } + fields.extend([ + ("secret", SdkValue::Bytes(secret.to_vec())), + ("peer", SdkValue::Bytes(peer.to_vec())), + ("instance", SdkValue::Bytes(instance.to_vec())), + ]); + save( + &directory.join("sandbox.record"), + SdkValue::Map( + (fields.into_iter()) + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ), ) } /// A Sandbox whose container no longer runs it: the Engine no longer has the @@ -406,31 +438,121 @@ pub(crate) fn gone_container(refusal: &SdkRefusal) -> Option<(&str, bool)> { _ => None, } } -/// The Engine, container and deployment a Sandbox record names. -pub(crate) fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { +/// The labels a Sandbox record keeps; a record written before #373 has none. +fn labels(value: &SdkValue) -> Result> { + match value.get("labels") { + Some(SdkValue::List(items)) => items + .iter() + .map(|item| { + Ok(( + registry::text(item, "key")?.to_owned(), + registry::text(item, "value")?.to_owned(), + )) + }) + .collect(), + _ => Ok(Vec::new()), + } +} +/// The Engine, container and deployment a Docker Sandbox record names. +fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { let docker = Docker::new(registry::text(value, "docker")?, Duration::from_secs(30)).map_err(failed)?; let request = SandboxRequest { image: registry::text(value, "image")?.to_owned(), store_volume: registry::text(value, "volume")?.to_owned(), port: 30421, - // a record written before #373 has no labels, nor has its container - labels: match value.get("labels") { - Some(SdkValue::List(items)) => items - .iter() - .map(|item| { - Ok(( - registry::text(item, "key")?.to_owned(), - registry::text(item, "value")?.to_owned(), - )) - }) - .collect::>()?, - _ => Vec::new(), - }, + labels: labels(value)?, }; let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; Ok((docker, container, request)) } +/// What a Sandbox record names: a Docker container, or (`runtime` `local`) +/// a local daemon process, its directory and port. +pub(crate) enum Recorded { + Docker(Docker, ContainerId, SandboxRequest), + Local { + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + }, +} +/// Reads a Sandbox record's runtime. A local record's directory must be +/// absolute, plain and named for its Sandbox, so a damaged record never +/// points a delete elsewhere. +pub(crate) fn recorded(value: &SdkValue) -> Result { + if value.get_text("runtime") != Some(crate::runtime::LOCAL) { + let (docker, container, request) = runtime(value)?; + return Ok(Recorded::Docker(docker, container, request)); + } + let directory = PathBuf::from(registry::text(value, "directory")?); + let name = registry::text(value, "name")?; + if name.is_empty() + || !directory.is_absolute() + || directory.file_name().and_then(|n| n.to_str()) != Some(name) + || directory.components().any(|c| { + matches!( + c, + std::path::Component::ParentDir | std::path::Component::CurDir + ) + }) + { + return Err(refused("InvalidInput", "local Sandbox record directory")); + } + let number = |key: &str| { + value + .get_int(key) + .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) + }; + let process = LocalProcess { + pid: u32::try_from(number("pid")?).map_err(failed)?, + start: registry::text(value, "start")? + .parse::() + .map_err(failed)?, + }; + Ok(Recorded::Local { + directory, + id: ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + process, + port: u16::try_from(number("port")?).map_err(failed)?, + labels: labels(value)?, + }) +} +impl Recorded { + /// The runtime, observed running once (it starts nothing). + pub(crate) fn reopen(self) -> std::result::Result> { + match self { + Self::Docker(docker, id, request) => { + Sandbox::reopen(docker, id, request).map(Runtime::Docker) + } + Self::Local { + directory, + id, + process, + port, + labels, + } => local::Sandbox::reopen(directory, id, process, port, labels).map(Runtime::Local), + } + } + /// The runtime found stopped: only its one Delete is admitted. + pub(crate) fn stopped(self) -> Runtime { + match self { + Self::Docker(docker, id, request) => { + Runtime::Docker(Sandbox::stopped(docker, id, request)) + } + Self::Local { + directory, + id, + process, + port, + labels, + } => Runtime::Local(local::Sandbox::stopped( + directory, id, process, port, labels, + )), + } + } +} pub(crate) fn reopen(args: &SdkValue) -> Result> { let id = registry::text(args, "sandbox")?; let directory = sandbox_dir(args, id)?; @@ -447,8 +569,13 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let (docker, selector, request) = runtime(&value)?; - let runtime = Sandbox::reopen(docker, selector, request).map_err(|failure| { + let recorded = recorded(&value)?; + // a local daemon listens on this kernel's loopback; no other host reaches it + let host = match recorded { + Recorded::Docker(..) => crate::pool::daemon_host(args), + Recorded::Local { .. } => None, + }; + let runtime = recorded.reopen().map_err(|failure| { if failure.container_gone() || failure.container_stopped() { gone(id, container, failure.container_stopped()) } else { @@ -457,7 +584,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { })?; let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, - host: crate::pool::daemon_host(args), + host, secret: bytes(&value, "secret")?, peer: bytes(&value, "peer")?, instance: bytes(&value, "instance")?, From 8d2473c19a35464be1aef4febade2d5c1e94a061 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:41 +0800 Subject: [PATCH 14/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/exec.rs (production). No tests changed or deleted. Production LOC: 109,644 -> 109,633 (delta -11) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/exec.rs | 44 +++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/sdk/js/native/src/exec.rs b/sdk/js/native/src/exec.rs index a0803ca58e..1ee27b45dd 100644 --- a/sdk/js/native/src/exec.rs +++ b/sdk/js/native/src/exec.rs @@ -5,46 +5,37 @@ //! daemon is not involved. A background command is started in its own session //! with its output in a log file, and this binding keeps its process id. The //! Engine offers no per-exec cancel, so `cancel` signals the process group -//! from a second command and waits (bounded) until it is gone. -use crate::job::{failed, refused, Outcome}; +//! from a second command and waits (bounded) until it is gone. A local +//! Sandbox (`runtime::Runtime::Local`) runs the same commands as processes of +//! this kernel, so the same probes and signals apply. +use crate::job::{refused, Outcome}; use crate::output::Spool; use crate::registry::{lock, naming, sandbox, sandbox_live, text, Process, SandboxState}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::time::Duration; -/// Home of the command identity, which has no account in the image. -const HOME: &str = "HOME=/tmp"; /// Checks that a cancelled process group is gone. const GONE_CHECKS: u32 = 50; /// Pause between those checks. const GONE_PAUSE: Duration = Duration::from_millis(100); /// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. +/// non-empty stream to its file instead of keeping it inline. Its home is +/// the Sandbox's scratch directory (`/tmp` in a container), since the +/// command identity has no account there. fn run( state: &SandboxState, directory: &str, arguments: Vec, spool: bool, ) -> Result<(i32, Spool, Spool), SdkRefusal> { - let created = layerfs_api::internal::exec( - &state.runtime, + state.runtime.run( state.identity, arguments, - vec![HOME.to_owned()], - directory.to_owned(), - false, + vec![format!("HOME={}", state.runtime.scratch())], + directory, + spool, ) - .map_err(failed)?; - let attached = created.start().map_err(failed)?; - let (_input, output, handle) = attached.into_parts().map_err(failed)?; - let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); - output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; - let inspected = handle.inspect().map_err(failed)?; - let code = inspected - .known_root_exit() - .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))?; - Ok((code, stdout, stderr)) } /// `workspace.exec(ws, command, { background?, spool? })`; with @@ -91,11 +82,14 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { state.id, layerfs_bridge::sdk_values::hex(&crate::registry::random()?[..12]) ); - let script = format!("setsid bash -c \"$0\" >/tmp/layerfs-{id}.log 2>&1 \"$1\" 2>&1 Outcome { /// The container's first process is the daemon, which does not reap orphans, /// so a finished member can stay a zombie that `kill -0` still finds. fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { - let (code, out, _) = run(state, "/tmp", vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; + let (code, out, _) = run(state, &state.runtime.scratch(), vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; if code == 0 { Ok(Some(String::from_utf8_lossy(out.kept()).into_owned())) } else { @@ -151,7 +145,7 @@ fn alive(state: &SandboxState, pid: u32) -> Result { ); let (code, _, _) = run( state, - "/tmp", + &state.runtime.scratch(), vec!["bash".into(), "-c".into(), probe], false, )?; @@ -283,7 +277,7 @@ pub(crate) fn cancel(args: &SdkValue) -> Outcome { } run( &state, - "/tmp", + &state.runtime.scratch(), vec![ "bash".into(), "-c".into(), From 055655ded77b83f4f49ebe6df6e048a13189d572 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:43 +0800 Subject: [PATCH 15/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/sandbox.rs (production). No tests changed or deleted. Production LOC: 109,633 -> 109,641 (delta +8) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/sandbox.rs | 65 +++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/sdk/js/native/src/sandbox.rs b/sdk/js/native/src/sandbox.rs index 5f74266e1f..90d665e27f 100644 --- a/sdk/js/native/src/sandbox.rs +++ b/sdk/js/native/src/sandbox.rs @@ -28,12 +28,12 @@ use std::time::Duration; /// Port the daemon listens on inside its container. const PORT: u16 = 30421; /// Wait for the daemon's listener after Start. -const STARTUP: Duration = Duration::from_secs(30); +pub(crate) const STARTUP: Duration = Duration::from_secs(30); /// Wait for one Engine control response. const ENGINE: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. -const LOG_LINES: u16 = 20; -const LOG_BYTES: usize = 2048; +pub(crate) const LOG_LINES: u16 = 20; +pub(crate) const LOG_BYTES: usize = 2048; pub(crate) fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -52,7 +52,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +pub(crate) fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -82,7 +82,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { } /// A container name prefix when the deployment sets none. -const PREFIX: &str = "layerfs-"; +pub(crate) const PREFIX: &str = "layerfs-"; /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. @@ -113,6 +113,9 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result state, - _ => "its record is unreadable".to_owned(), - }; - return Err(refused( - "Unknown", - format!( - "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ - is unknown, so it is neither retried nor taken as done. Remove any container it \ - left (docker ps -a --filter volume={store_volume}), then delete {}", - intent.display() - ), - )); - } + unresolved( + directory, + &format!("docker ps -a --filter volume={store_volume}"), + )?; // The number is claimed before anything is made; a failed start gives it // back (its directory holds no record yet), so the number is used again. let (id, saved_directory) = crate::discovery::claim_next(args)?; @@ -172,6 +162,29 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result<(), SdkRefusal> { + let intent = directory.join("create.intent"); + if intent.exists() { + let state = match crate::discovery::load(&intent) { + Ok(SdkValue::Text(state)) => state, + _ => "its record is unreadable".to_owned(), + }; + return Err(refused( + "Unknown", + format!( + "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ + is unknown, so it is neither retried nor taken as done. Remove any container it \ + left ({left}), then delete {}", + intent.display() + ), + )); + } + Ok(()) +} + /// Starts the claimed Sandbox `id`: container, daemon on the installed /// Store, then its records. fn start_claimed( @@ -241,7 +254,7 @@ fn start_claimed( project: name.to_owned(), name: container_name, directory: saved_directory.to_owned(), - runtime, + runtime: crate::runtime::Runtime::Docker(runtime), identity, controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), @@ -339,7 +352,7 @@ fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkR } } -/// Reclaims Sandbox `id`, whose container a reopen found gone or stopped +/// Reclaims Sandbox `id`, whose container (or local daemon) a reopen found gone or stopped /// (`container`, `stopped`): one unforced Delete of a stopped container, then /// its records, so its number is free again. Called under the project's /// guard; the Sandbox's own guard is taken too. The record is read again @@ -364,9 +377,7 @@ pub(crate) fn reclaim(args: &SdkValue, id: &str, container: &str, stopped: bool) )); } if stopped { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - match layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request).delete() - { + match crate::discovery::recorded(&value)?.stopped().delete() { Err(failure) if !failure.container_gone() => return Err(failed(failure)), _ => {} } From 4906f639f233e5c1658240cddb96429562f6983f Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:45 +0800 Subject: [PATCH 16/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/workspace.rs (production). No tests changed or deleted. Production LOC: 109,641 -> 109,636 (delta -5) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/workspace.rs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/sdk/js/native/src/workspace.rs b/sdk/js/native/src/workspace.rs index 83e5b31b5d..696e7de15f 100644 --- a/sdk/js/native/src/workspace.rs +++ b/sdk/js/native/src/workspace.rs @@ -35,9 +35,9 @@ fn recorded(directory: &Path) -> Result { /// lowest-numbered running one of the call's image with a free slot, else a /// new one. fn place(args: &SdkValue, project: &Path) -> Result, SdkRefusal> { - let image = args - .get("deploy") - .and_then(|deploy| deploy.get_text("image")); + // `local` for the local runtime: local and Docker Sandboxes never share + let image = crate::runtime::image(args)?; + let image = image.as_deref(); for id in crate::discovery::sandbox_ids(args)? { let directory = crate::discovery::sandbox_dir(args, &id)?; let record = directory.join("sandbox.record"); @@ -81,7 +81,7 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { if let SdkValue::Map(fields) = &mut value { fields.push(("sandbox".to_owned(), SdkValue::text(state.id.as_str()))); fields.push(("container".to_owned(), SdkValue::text(state.name.as_str()))); - let image = state.runtime.request().image.as_str(); + let image = state.runtime.image(); fields.push(("image".to_owned(), SdkValue::text(image))); } crate::discovery::save_workspace(&state, &mounted, branch)?; @@ -96,7 +96,8 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { } /// `operator.sandbox.list()`: the project's running Sandboxes, lowest -/// number first: `{ id, name, image }`. Each recorded container is inspected once +/// number first: `{ id, name, image }`. Each recorded container (or local +/// daemon) is inspected once /// (no daemon is dialed); one found gone or stopped, an idle-closed /// Sandbox's, is reclaimed under the project's guard and left out. It starts /// nothing. @@ -123,11 +124,7 @@ pub(crate) fn list(args: &SdkValue) -> Outcome { let cached = lock(&crate::registry::SANDBOXES)?.get(&directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request) - .map(drop) - } + None => crate::discovery::recorded(&value)?.reopen().map(drop), }; match observed { Ok(()) => items.push(item), From de16c451c09b8a3ba0850a175684292b86f44cef Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:47 +0800 Subject: [PATCH 17/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/src/lib.rs (production). No tests changed or deleted. Production LOC: 109,636 -> 109,638 (delta +2) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/js/native/src/lib.rs b/sdk/js/native/src/lib.rs index 0c8c45c5c9..5f9d2570ec 100644 --- a/sdk/js/native/src/lib.rs +++ b/sdk/js/native/src/lib.rs @@ -5,10 +5,12 @@ mod discovery; mod dispatch; mod exec; mod job; +mod local; mod output; mod pool; mod project; mod registry; +mod runtime; mod sandbox; mod trace; mod workspace; From 3466de35718b6d44138c6ab53e9fde7a7b153e71 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:49 +0800 Subject: [PATCH 18/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/native/tests/unit/runtime.rs (not production (test, manifest or docs)). No tests changed or deleted. Production LOC: 109,638 -> 109,638 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/native/tests/unit/runtime.rs | 95 +++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 sdk/js/native/tests/unit/runtime.rs diff --git a/sdk/js/native/tests/unit/runtime.rs b/sdk/js/native/tests/unit/runtime.rs new file mode 100644 index 0000000000..c2dc1518a7 --- /dev/null +++ b/sdk/js/native/tests/unit/runtime.rs @@ -0,0 +1,95 @@ +//! Which runtime a deployment selects, and what a local Sandbox record may +//! name. Included by runtime.rs. +use super::{image, local, LOCAL}; +use layerfs_bridge::control::SdkValue; + +fn deploy(fields: &[(&str, &str)]) -> SdkValue { + SdkValue::Map( + (fields.iter()) + .map(|(k, v)| ((*k).to_owned(), SdkValue::text(*v))) + .collect(), + ) +} + +#[test] +fn docker_stays_the_default_and_only_local_selects_local() { + let image_digest = format!("sha256:{}", "a".repeat(64)); + // unset and `docker` are Docker, placed by the deployment's image as before + for fields in [vec![], vec![("sandbox", "docker")]] { + let mut fields = fields; + fields.push(("image", image_digest.as_str())); + let deploy = deploy(&fields); + assert!(!local(&deploy).unwrap()); + let args = SdkValue::map([("deploy", deploy)]); + assert_eq!( + image(&args).unwrap().as_deref(), + Some(image_digest.as_str()) + ); + } + // no deployment: no image filter, as before + assert_eq!( + image(&SdkValue::map([("project", SdkValue::text("p"))])).unwrap(), + None + ); + // local ignores any image: its Sandboxes are placed as `local` + let deploy_local = deploy(&[("sandbox", "local"), ("image", image_digest.as_str())]); + assert!(local(&deploy_local).unwrap()); + let args = SdkValue::map([("deploy", deploy_local)]); + assert_eq!(image(&args).unwrap().as_deref(), Some(LOCAL)); + // anything else is refused, never taken as Docker or local + for typo in ["Local", "lokal", "LOCAL", " local", "docker "] { + let refusal = local(&deploy(&[("sandbox", typo)])).unwrap_err(); + assert_eq!(refusal.code, "InvalidInput"); + assert!( + refusal.message.contains("docker or local"), + "{}", + refusal.message + ); + } +} + +fn record(directory: &str, name: &str) -> SdkValue { + SdkValue::map([ + ("id", SdkValue::text("1")), + ("name", SdkValue::text(name)), + ("container", SdkValue::text("c".repeat(64))), + ("runtime", SdkValue::text("local")), + ("directory", SdkValue::text(directory)), + ("pid", SdkValue::Int(4242)), + ("start", SdkValue::text("123456")), + ("port", SdkValue::Int(40404)), + ]) +} + +#[test] +fn a_local_record_names_its_own_directory_only() { + match crate::discovery::recorded(&record("/lfs/local/layerfs-p-1", "layerfs-p-1")) { + Ok(crate::discovery::Recorded::Local { + directory, + process, + port, + .. + }) => { + assert_eq!(directory, std::path::Path::new("/lfs/local/layerfs-p-1")); + assert_eq!((process.pid, process.start, port), (4242, 123456, 40404)); + } + _ => panic!("a valid local record was refused"), + } + for (directory, name) in [ + ("relative/layerfs-p-1", "layerfs-p-1"), + ("/lfs/local/other", "layerfs-p-1"), + ("/lfs/../etc/layerfs-p-1", "layerfs-p-1"), + ("/", ""), + ] { + assert!( + crate::discovery::recorded(&record(directory, name)).is_err(), + "{directory} admitted for {name}" + ); + } + let mut bad_port = record("/lfs/local/layerfs-p-1", "layerfs-p-1"); + if let SdkValue::Map(fields) = &mut bad_port { + fields.retain(|(k, _)| k != "port"); + fields.push(("port".into(), SdkValue::Int(70000))); + } + assert!(crate::discovery::recorded(&bad_port).is_err()); +} From 41a6c201adf8fd21bca012739bcc87ca06130c5f Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:51 +0800 Subject: [PATCH 19/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/src/client.js (production). No tests changed or deleted. Production LOC: 109,638 -> 109,656 (delta +18) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/src/client.js | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/sdk/js/src/client.js b/sdk/js/src/client.js index 4aa6ab3a24..266dbaeebb 100644 --- a/sdk/js/src/client.js +++ b/sdk/js/src/client.js @@ -40,9 +40,18 @@ export function workspaceArgs(ws) { /** Deployment settings from the client options, else the environment * (LAYERFS_, else DSH_LAYERFS_). A Workspace's Sandbox needs - * every one but the container prefix and daemon host. */ + * every one but the container prefix and daemon host. With `sandbox` + * (LAYERFS_SANDBOX) `local`, a Sandbox is a daemon process on this machine + * instead of a container: it needs the daemon and `localDir` + * (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is Docker; + * any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || process.env[`DSH_LAYERFS_${name}`] || undefined; + const sandbox = options.sandbox ?? env("SANDBOX"); + if (sandbox === "local") return localDeployment(options, env); + if (sandbox !== undefined && sandbox !== "docker") { + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or local, not ${sandbox}`, null); + } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", image: options.image ?? env("IMAGE"), @@ -60,6 +69,22 @@ export function deployment(options = {}) { return deploy; } +/** A local Sandbox's settings: the daemon, the directory its Sandboxes + * live in, and the optional name prefix. */ +function localDeployment(options, env) { + const deploy = { + sandbox: "local", + daemon: options.daemon ?? env("DAEMON"), + localDir: options.localDir ?? env("LOCAL_DIR"), + containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), + }; + for (const [name, setting] of [["daemon", "DAEMON"], ["localDir", "LOCAL_DIR"]]) { + if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); + } + if (!deploy.containerPrefix) delete deploy.containerPrefix; + return deploy; +} + /** One client's deployment selection, including cross-process discovery * home and the Store this process opens. */ export function invokeClient(client, op, args) { From 6d9dd417babf33836a14788f09ef6adff272ed88 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:53 +0800 Subject: [PATCH 20/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/src/index.js (production). No tests changed or deleted. Production LOC: 109,656 -> 109,658 (delta +2) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/src/index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/js/src/index.js b/sdk/js/src/index.js index f9098d5053..a1b66c645b 100644 --- a/sdk/js/src/index.js +++ b/sdk/js/src/index.js @@ -35,7 +35,10 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON, each * else its DSH_LAYERFS_ twin; a Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. + * container it starts `-`. LAYERFS_SANDBOX=local + * starts each Sandbox as a daemon process on this machine instead of a + * container, in a directory under LAYERFS_LOCAL_DIR (no Docker setting is + * read; it isolates nothing, so use it only inside a VM or CI box). * `options.daemonHost` (else LAYERFS_DAEMON_HOST) is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on @@ -64,6 +67,8 @@ export class LayerFS { volume: env("VOLUME"), daemon: env("DAEMON"), ...(env("CONTAINER_PREFIX") ? { containerPrefix: env("CONTAINER_PREFIX") } : {}), + ...(env("SANDBOX") ? { sandbox: env("SANDBOX") } : {}), + ...(env("LOCAL_DIR") ? { localDir: env("LOCAL_DIR") } : {}), ...((options.daemonHost ?? env("DAEMON_HOST")) ? { daemonHost: options.daemonHost ?? env("DAEMON_HOST") } : {}), }, storePolicy: options.storePolicy, From 4a5caa7939904d23927b38be7ce7546ced563e8a Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:55 +0800 Subject: [PATCH 21/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/test/local_deployment.mjs (not production (test, manifest or docs)). No tests changed or deleted. Production LOC: 109,658 -> 109,658 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/test/local_deployment.mjs | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 sdk/js/test/local_deployment.mjs diff --git a/sdk/js/test/local_deployment.mjs b/sdk/js/test/local_deployment.mjs new file mode 100644 index 0000000000..17b332fb72 --- /dev/null +++ b/sdk/js/test/local_deployment.mjs @@ -0,0 +1,71 @@ +// Which runtime a Workspace's Sandbox uses. No addon, Store or Docker: +// node sdk/js/test/local_deployment.mjs +// LAYERFS_SANDBOX unset or `docker` gives exactly the Docker deployment it +// always gave; `local` needs only the daemon and LAYERFS_LOCAL_DIR (no Docker +// setting); any other value is refused rather than taken as either. +import assert from "node:assert/strict"; +import { LayerFS } from "@layerfs/sdk"; +import { deployment } from "../src/client.js"; + +const clear = () => { + for (const name of Object.keys(process.env)) { + if (name.startsWith("LAYERFS_") || name.startsWith("DSH_LAYERFS_")) delete process.env[name]; + } +}; +const docker = { docker: "/run/docker.sock", image: `sha256:${"a".repeat(64)}`, volume: "lfs-v", daemon: "/opt/layerfs-daemon" }; +const setDocker = () => { + process.env.LAYERFS_DOCKER = docker.docker; + process.env.LAYERFS_IMAGE = docker.image; + process.env.LAYERFS_VOLUME = docker.volume; + process.env.LAYERFS_DAEMON = docker.daemon; +}; + +// unset: the Docker deployment, field for field, and the client options carry no new key +clear(); +setDocker(); +assert.deepEqual(deployment(), docker); +assert.deepEqual(new LayerFS("website").options.deploy, docker); +// `docker` is the same deployment +process.env.LAYERFS_SANDBOX = "docker"; +assert.deepEqual(deployment(), docker); +// the Docker settings are still required there +delete process.env.LAYERFS_IMAGE; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_IMAGE/.test(e.message)); + +// local: daemon + localDir, no Docker setting read or required +clear(); +process.env.LAYERFS_SANDBOX = "local"; +process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; +process.env.LAYERFS_LOCAL_DIR = "/store/local"; +assert.deepEqual(deployment(), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); +const client = new LayerFS("website"); +assert.equal(client.options.deploy.sandbox, "local"); +assert.equal(client.options.deploy.localDir, "/store/local"); +assert.deepEqual(deployment(client.options.deploy), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); +// an image (workspace.mount's option) does not change a local deployment +assert.deepEqual(deployment({ ...client.options.deploy, image: docker.image }), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); +// the container prefix names local Sandbox directories too +process.env.LAYERFS_CONTAINER_PREFIX = "ci-"; +assert.equal(deployment().containerPrefix, "ci-"); +delete process.env.LAYERFS_CONTAINER_PREFIX; +// each required setting is named when missing +delete process.env.LAYERFS_LOCAL_DIR; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_LOCAL_DIR/.test(e.message)); +process.env.LAYERFS_LOCAL_DIR = "/store/local"; +delete process.env.LAYERFS_DAEMON; +assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_DAEMON/.test(e.message)); +// the DSH_ twins work as for every other setting +clear(); +process.env.DSH_LAYERFS_SANDBOX = "local"; +process.env.DSH_LAYERFS_DAEMON = "/opt/d"; +process.env.DSH_LAYERFS_LOCAL_DIR = "/l"; +assert.deepEqual(deployment(), { sandbox: "local", daemon: "/opt/d", localDir: "/l" }); + +// anything else is refused, never silently Docker +for (const typo of ["Local", "lokal", "LOCAL", "podman"]) { + clear(); + setDocker(); + process.env.LAYERFS_SANDBOX = typo; + assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && e.message.includes(`docker or local, not ${typo}`)); +} +console.log("local_deployment: ok"); From 819bd2c9cd347496d61a5dba708c608356463d59 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:31:57 +0800 Subject: [PATCH 22/64] feat(sandbox): a local process runtime for Sandboxes, LAYERFS_SANDBOX=local, for a VM or CI box without Docker Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole branch (or the single-commit patch). This commit: sdk/js/README.md (not production (test, manifest or docs)). No tests changed or deleted. Production LOC: 109,658 -> 109,658 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, base 5fb68ddc1. --- sdk/js/README.md | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/sdk/js/README.md b/sdk/js/README.md index a0c6f290c4..0daaec449e 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -68,10 +68,12 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open in this process; the client option `storePath` wins. It is `global/store.sqlite` in the project's volume, `/layerfs-store/global/store.sqlite` where that volume is mounted at `/layerfs-store`. Linux only, on a local filesystem the daemons share; elsewhere Store calls fail `Unsupported`. Unset, they fail `InvalidInput`. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional; the client option `daemonHost` wins. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | +| `LAYERFS_SANDBOX` | Optional; the client option `sandbox` wins. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `local`: a Sandbox is a daemon process on this machine, with no container and no Docker setting read (see [Local Sandboxes](#local-sandboxes-no-docker)). Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=local`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each local Sandbox gets its own directory under it. | | `LAYERFS_TRACE` | Optional per-call trace file (see [Call trace](#call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | -| `DSH_LAYERFS_HOME`, `DSH_LAYERFS_DOCKER`, `DSH_LAYERFS_IMAGE`, `DSH_LAYERFS_VOLUME`, `DSH_LAYERFS_DAEMON`, `DSH_LAYERFS_STORE_PATH`, `DSH_LAYERFS_TRACE`, `DSH_LAYERFS_TRACE_STEP`, `DSH_LAYERFS_TRACE_START` | Each is used when its `LAYERFS_` name is unset or empty. A host that may only set `DSH_*` names in a command's environment (deepseek-harness) configures and traces the SDK through these. | +| `DSH_LAYERFS_HOME`, `DSH_LAYERFS_DOCKER`, `DSH_LAYERFS_IMAGE`, `DSH_LAYERFS_VOLUME`, `DSH_LAYERFS_DAEMON`, `DSH_LAYERFS_STORE_PATH`, `DSH_LAYERFS_SANDBOX`, `DSH_LAYERFS_LOCAL_DIR`, `DSH_LAYERFS_TRACE`, `DSH_LAYERFS_TRACE_STEP`, `DSH_LAYERFS_TRACE_START` | Each is used when its `LAYERFS_` name is unset or empty. A host that may only set `DSH_*` names in a command's environment (deepseek-harness) configures and traces the SDK through these. | A Sandbox is a container running one daemon on the project's Store. The project has numbered Sandboxes, `1`, `2`, …, each on one image: @@ -230,6 +232,42 @@ keeps its markers in that Workspace; inspect them through ordinary bounded `exec` commands and Commit each command. An omitted diagnostic is not a clean merge, and a failed/unknown operation must be inspected before a new operation. +## Local Sandboxes (no Docker) + +`LAYERFS_SANDBOX=local` runs each Sandbox's daemon as a process of this +machine instead of in a container, for a Linux VM or CI job that has no Docker +and is itself the isolation boundary (a microVM, for instance). **It isolates +nothing**: the daemon runs as root in this machine's namespaces, and commands +share its processes, network and filesystem; they only run as their own uid +501 and gid 20, with no supplementary groups. Use it only where everything on +the machine belongs to the same trust domain. Docker stays the default. + +It needs root (the daemon mounts FUSE and commands run as another identity; +otherwise `workspace.mount` is refused `Unsupported`), `/dev/fuse` (the +daemon narrows it to 0600, as it does in a container), the daemon executable +(`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` and the client's `storePath`, which +the daemon opens directly: the Store calls and the daemons use the same file, +on ext4, xfs or btrfs. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` and +`_DAEMON_HOST` are not read. Everything else above holds: numbered +Sandboxes of up to 16 Workspaces, placement, idle close after 2 minutes, +reclaim, records under `LAYERFS_HOME//sandboxes//`, background +processes, start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME` and the +background processes' logs) and `daemon.log` (the daemon's output, what +`docker logs` shows for a container; a failed start's report carries its last +20 lines). The daemon listens on 127.0.0.1 only. Its records name the daemon +by process id and start time, so a reused id is never taken for it. A mount +result's `image` is `local`; local and Docker Sandboxes of one project are +never mixed. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next mount or +`operator.sandbox.list()` reclaims it, detaching any mount the dead daemon +left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. + ## Internal operator ```js From 13a791abff5f6cbeb8560cddf2515a2a2d515a84 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:07 +0800 Subject: [PATCH 23/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/process.rs (production). No tests changed or deleted except additions. Production LOC: 109,658 -> 109,696 (delta +38) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../src/backend/local/process.rs | 106 +++++++++++++----- 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/process.rs b/core/crates/layerfs-sandbox/src/backend/local/process.rs index 67eb36d4c1..d56fee6882 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/process.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/process.rs @@ -1,60 +1,114 @@ -//! A local process named by its id and kernel start time, so a reused id is +//! A local process named by its id, kernel start time and boot, so a reused +//! id, or the same id and start time after a reboot or snapshot restore, is //! never taken for the process that was started. use nix::{sys::signal::Signal, unistd::Pid}; -use std::io; +use std::{io, sync::OnceLock}; -/// One started process: its id and its start time in clock ticks since boot -/// (`/proc//stat` field 22). Both must match for it to count as alive. +/// One started process: its id, its start time in clock ticks since boot +/// (`/proc//stat` field 22) and the boot it started in +/// (`/proc/sys/kernel/random/boot_id`). All must match for it to count as +/// alive. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct LocalProcess { pub pid: u32, pub start: u64, + /// The boot id as a number. `None` only for a record written before boot + /// ids were kept: it is matched by id and start time alone, as it was + /// then, and never signalled. + pub boot: Option, } impl LocalProcess { - /// The process running now under `pid`, read once from `/proc`. + /// The process running now under `pid`, read once from `/proc`, in this + /// boot. pub fn observe(pid: u32) -> io::Result { - let (_, start) = stat(pid)?; - Ok(Self { pid, start }) + let stat = stat(pid)?; + Ok(Self { + pid, + start: stat.start, + boot: Some(boot()?), + }) } /// True while this exact process exists and has not exited: a zombie /// (or a dead entry) is not alive, and another process under the same id - /// has another start time. + /// has another start time or another boot. pub fn alive(self) -> bool { + let this_boot = self + .boot + .is_none_or(|recorded| boot().ok() == Some(recorded)); match stat(self.pid) { - Ok((state, start)) => start == self.start && !matches!(state, b'Z' | b'X' | b'x'), + Ok(stat) => this_boot && stat.start == self.start && !stat.ended(), Err(_) => false, } } /// Sends `signal` once, only after this exact process was observed alive. pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if self.boot.is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "a local process recorded without its boot is never signalled", + )); + } if !self.alive() { return Ok(()); } - let pid = i32::try_from(self.pid) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width"))?; - match nix::sys::signal::kill(Pid::from_raw(pid), signal) { + match nix::sys::signal::kill(pid(self.pid)?, signal) { Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), Err(errno) => Err(errno.into()), } } } -/// State and start time of `pid`. The command name may hold spaces and -/// parentheses, so fields are counted after its last `)`. -fn stat(pid: u32) -> io::Result<(u8, u64)> { +/// This boot's id (`/proc/sys/kernel/random/boot_id`, a UUID) as a number, +/// read once per process. +pub(super) fn boot() -> io::Result { + static BOOT: OnceLock = OnceLock::new(); + if let Some(boot) = BOOT.get() { + return Ok(*boot); + } + let text = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?; + let hex: String = text.trim().chars().filter(|c| *c != '-').collect(); + let boot = (hex.len() == 32) + .then(|| u128::from_str_radix(&hex, 16).ok()) + .flatten() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "boot id"))?; + Ok(*BOOT.get_or_init(|| boot)) +} + +/// What `/proc//stat` says of one process. +#[derive(Clone, Copy, Debug)] +pub(super) struct Stat { + pub(super) state: u8, + pub(super) group: u32, + pub(super) start: u64, +} + +impl Stat { + /// A zombie or a dead entry: it runs nothing any more. + pub(super) fn ended(self) -> bool { + matches!(self.state, b'Z' | b'X' | b'x') + } +} + +/// State, process group and start time of `pid`. The command name may hold +/// spaces and parentheses, so fields are counted after its last `)`. +pub(super) fn stat(pid: u32) -> io::Result { let text = std::fs::read_to_string(format!("/proc/{pid}/stat"))?; let invalid = || io::Error::new(io::ErrorKind::InvalidData, "process stat fields"); let rest = text.rsplit_once(')').ok_or_else(invalid)?.1; - let mut fields = rest.split_ascii_whitespace(); - let state = fields - .next() - .and_then(|s| s.bytes().next()) - .ok_or_else(invalid)?; - // field 3 was the state; field 22 is 19 fields further on - let start = fields - .nth(18) - .and_then(|s| s.parse::().ok()) - .ok_or_else(invalid)?; - Ok((state, start)) + // field 3 (the state) is the first after the name, so field n is at n - 3 + let fields: Vec<&str> = rest.split_ascii_whitespace().collect(); + let field = |n: usize| fields.get(n - 3).copied().ok_or_else(invalid); + Ok(Stat { + state: field(3)?.bytes().next().ok_or_else(invalid)?, + group: field(5)?.parse().map_err(|_| invalid())?, + start: field(22)?.parse().map_err(|_| invalid())?, + }) +} + +/// A process id as the kernel's signed type. +pub(super) fn pid(id: u32) -> io::Result { + i32::try_from(id) + .map(Pid::from_raw) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width")) } From 13361ba5abc990f21d5a7e3d169ca05244f27f2c Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:10 +0800 Subject: [PATCH 24/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/groups.rs (production). No tests changed or deleted except additions. Production LOC: 109,696 -> 109,818 (delta +122) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../src/backend/local/groups.rs | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/groups.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/groups.rs b/core/crates/layerfs-sandbox/src/backend/local/groups.rs new file mode 100644 index 0000000000..e307f7e763 --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/groups.rs @@ -0,0 +1,157 @@ +//! The process groups a local Sandbox's processes run in. A container's +//! processes end with it; a local Sandbox's would outlive it on this kernel, +//! so each command starts its own process group, named in the Sandbox's +//! `groups` directory while it may have members, and every named group (and +//! the daemon's own) is killed when the Sandbox stops or is deleted. +//! +//! A group is named by its leader: id, start time and boot. Its id stays the +//! group's while any member lives, so a group whose leader is gone is still +//! known by it; a leader found with another start time means the id was +//! reused and the group is not this Sandbox's. A process that leaves its +//! group (`setsid`) is not followed unless it is named here too +//! ([`super::Sandbox::track`]). +use super::process::{self, LocalProcess}; +use nix::{fcntl::OFlag, sys::signal::Signal}; +use std::{ + fs::{self, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::Path, + time::{Duration, Instant}, +}; + +/// The directory of a Sandbox that names its groups, one empty file each. +pub(super) const GROUPS: &str = "groups"; +/// Pause between two looks at a killed group. +const PAUSE: Duration = Duration::from_millis(20); + +/// Names the group `leader` leads (its id is the group's), once. +pub(super) fn name(directory: &Path, leader: LocalProcess) -> io::Result<()> { + let groups = directory.join(GROUPS); + // a Sandbox started before groups were named has no directory yet + match fs::DirBuilder::new().mode(0o700).create(&groups) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(groups.join(entry(leader)?)) + .map(drop) +} + +/// Forgets the group `leader` led once no member of it is left. Called after +/// the leader was reaped; a group that still has members stays named. +pub(super) fn forget_if_empty(directory: &Path, leader: LocalProcess) -> io::Result<()> { + match nix::sys::signal::killpg(process::pid(leader.pid)?, None) { + Err(nix::errno::Errno::ESRCH) => {} + _ => return Ok(()), + } + match fs::remove_file(directory.join(GROUPS).join(entry(leader)?)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Kills every group the Sandbox at `directory` named, and the group of +/// `daemon` (its leader), once with KILL, as a container's end kills what +/// ran in it, then waits up to `wait` for their members to be gone. Groups +/// of another boot ended with it. +pub(super) fn end( + directory: &Path, + daemon: Option, + wait: Duration, +) -> io::Result<()> { + let mut leaders: Vec = daemon.into_iter().collect(); + match fs::read_dir(directory.join(GROUPS)) { + Ok(items) => { + for item in items { + let name = item?.file_name(); + let leader = name.to_str().and_then(parse).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "local Sandbox group name") + })?; + leaders.push(leader); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let this_boot = process::boot()?; + leaders.retain(|leader| leader.boot == Some(this_boot)); + let table = processes()?; + let mut killed = Vec::new(); + for leader in leaders { + if !members(&table, leader).is_empty() { + match nix::sys::signal::killpg(process::pid(leader.pid)?, Signal::SIGKILL) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => killed.push(leader), + Err(errno) => return Err(errno.into()), + } + } + } + let deadline = Instant::now() + wait; + loop { + let table = processes()?; + if killed + .iter() + .all(|leader| members(&table, *leader).is_empty()) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::other( + "a process of the local Sandbox still runs after KILL", + )); + } + std::thread::sleep(PAUSE); + } +} + +/// The running members of `leader`'s group: none when the leader's id now +/// names another process (the id was reused, so the group is another's). +fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { + let reused = (table.iter()).any(|(pid, stat)| *pid == leader.pid && stat.start != leader.start); + if reused { + return Vec::new(); + } + (table.iter()) + .filter(|(_, stat)| stat.group == leader.pid && !stat.ended()) + .map(|(pid, _)| *pid) + .collect() +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +fn processes() -> io::Result> { + let mut table = Vec::new(); + for item in fs::read_dir("/proc")? { + let Some(pid) = item?.file_name().to_str().and_then(|n| n.parse().ok()) else { + continue; + }; + if let Ok(stat) = process::stat(pid) { + table.push((pid, stat)); + } + } + Ok(table) +} + +/// A group's file name: `--`. +fn entry(leader: LocalProcess) -> io::Result { + let boot = leader + .boot + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "a group leader's boot"))?; + Ok(format!("{boot:032x}-{}-{}", leader.pid, leader.start)) +} + +fn parse(name: &str) -> Option { + let mut parts = name.split('-'); + let boot = parts.next().filter(|b| b.len() == 32)?; + let process = LocalProcess { + boot: Some(u128::from_str_radix(boot, 16).ok()?), + pid: parts.next()?.parse().ok()?, + start: parts.next()?.parse().ok()?, + }; + parts.next().is_none().then_some(process) +} From fed02493f2e5cec5f3fb2110f44a8daab3cdde01 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:12 +0800 Subject: [PATCH 25/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/exec.rs (production). No tests changed or deleted except additions. Production LOC: 109,818 -> 109,835 (delta +17) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../layerfs-sandbox/src/backend/local/exec.rs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/exec.rs b/core/crates/layerfs-sandbox/src/backend/local/exec.rs index 0673564715..ec099ddb89 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/exec.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/exec.rs @@ -1,8 +1,11 @@ //! One ordinary command to its end in a local Sandbox: the command identity, //! the Workspace directory and a search path; nothing of this process's -//! environment, no shell of its own and no implicit timeout. -use super::sandbox::Sandbox; +//! environment, no shell of its own and no implicit timeout. Each command +//! leads a process group of its own, named in the Sandbox until it is empty, +//! so what it leaves running ends with the Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox}; use crate::{backend::docker::ExecRequest, RuntimeError, WireFailure}; +use nix::sys::signal::{killpg, Signal}; use std::{ io::{self, Read, Write}, os::unix::process::{CommandExt, ExitStatusExt}, @@ -47,6 +50,7 @@ impl Sandbox { // std drops supplementary groups (setgroups 0) when it sets a uid as root .uid(request.identity.uid) .gid(request.identity.gid) + .process_group(0) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -61,6 +65,23 @@ impl Sandbox { failure })?; let pid = child.id(); + // named before anything is waited for: the leader is not reaped + // before the wait below, so it is observable even once it exited + let named = LocalProcess::observe(pid) + .and_then(|leader| groups::name(self.directory(), leader).map(|()| leader)); + let leader = match named { + Ok(leader) => leader, + Err(error) => { + // an unnamed group would outlive the Sandbox: it ends here, once + if let Ok(group) = process::pid(pid) { + let _ = killpg(group, Signal::SIGKILL); + } + let _ = child.wait(); + let mut failure = self.io(error); + failure.attempted = true; + return Err(failure); + } + }; let out = child.stdout.take(); let err = child.stderr.take(); let copied = std::thread::scope(|scope| { @@ -73,6 +94,8 @@ impl Sandbox { }); // the pipes are closed here either way; the exit is still waited for let status = child.wait(); + // a name left behind is harmless: its group is checked again when ended + let _ = groups::forget_if_empty(self.directory(), leader); let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; let status = status.map_err(|error| self.io(error))?; let code = status From 603b561f13b3b758b0c1d4320c40ac7fad9b9c38 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:15 +0800 Subject: [PATCH 26/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/sandbox.rs (production). No tests changed or deleted except additions. Production LOC: 109,835 -> 109,868 (delta +33) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../src/backend/local/sandbox.rs | 90 +++++++++++++++---- 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs index 51c1bd26a7..2f85dc9a13 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs @@ -1,6 +1,9 @@ //! One local Sandbox: its directory, its daemon process and its exact -//! attempts. No Drop cleanup or replay, as for a container. -use super::{mounts, process::LocalProcess}; +//! attempts. No Drop cleanup or replay, as for a container; a Drop only +//! reaps a daemon this process started that has already exited. A stop +//! returns only once a daemon this process started is reaped, so it never +//! stays a zombie of this process. +use super::{groups, mounts, process::LocalProcess}; use crate::{ContainerId, RuntimeError, WireFailure}; use nix::sys::signal::Signal; use std::{ @@ -34,6 +37,8 @@ pub struct Sandbox { pub(super) child: Mutex>, pub(super) stop_attempted: bool, pub(super) stopped: bool, + /// Its process groups were killed and found empty (by its stop). + pub(super) ended: bool, pub(super) delete_attempted: bool, pub(super) deleted: bool, } @@ -54,6 +59,7 @@ impl Sandbox { child: Mutex::new(None), stop_attempted: false, stopped: false, + ended: false, delete_attempted: false, deleted: false, } @@ -135,8 +141,10 @@ impl Sandbox { Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) } /// Stops the daemon once: TERM, up to `seconds` for it to end, then - /// KILL. A daemon already gone is the same stopped state. Crash - /// teardown, not a graceful filesystem drain. + /// KILL (a daemon this process started has ended once reaped); then + /// KILL to every process group of its commands and its daemon, as a + /// container's end kills what ran in it. A daemon already gone is the + /// same stopped state. Crash teardown, not a graceful filesystem drain. pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { if self.stop_attempted || self.delete_attempted { return Err(self.invalid("original local Stop admission")); @@ -148,7 +156,7 @@ impl Sandbox { Signal::SIGTERM, Duration::from_secs(u64::from(seconds)), )?; - if process.alive() { + if !self.gone(process) { self.end(process, Signal::SIGKILL, KILL_WAIT)?; } if process.alive() { @@ -156,11 +164,15 @@ impl Sandbox { } } self.stopped = true; + groups::end(&self.directory, self.process, KILL_WAIT).map_err(|error| self.io(error))?; + self.ended = true; Ok(()) } - /// Deletes its directory once, after a known stop: any mount its daemon - /// left is detached first, so nothing is removed through a mount. The - /// shared Store is outside it and never touched. + /// Deletes its directory once, after a known stop: any process left in + /// its groups is killed first (when its stop did not run here, as for a + /// daemon found stopped), and any mount its daemon left is detached, so + /// nothing writes into it or is removed through a mount. The shared + /// Store is outside it and never touched. pub fn delete(&mut self) -> Result<(), Box> { if self.delete_attempted || (self.process.is_some() && !self.stopped) { return Err(self.invalid("original local Delete admission; known stop required")); @@ -170,6 +182,11 @@ impl Sandbox { if self.process.is_some_and(LocalProcess::alive) { return Err(self.invalid("local daemon runs again; nothing was deleted")); } + if !self.ended { + groups::end(&self.directory, self.process, KILL_WAIT) + .map_err(|error| self.io(error))?; + self.ended = true; + } mounts::detach(&self.directory).map_err(|error| self.io(error))?; match fs::remove_dir_all(&self.directory) { Ok(()) => {} @@ -207,16 +224,41 @@ impl Sandbox { let tail = &bytes[start..]; Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) } - /// Reaps the daemon this process started, if it exited. - pub(super) fn reap(&self) { - let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); - if let Some(started) = child.as_mut() { - if matches!(started.try_wait(), Ok(Some(_))) { - *child = None; + /// Names the process group `pid` leads or is about to lead: a process + /// started at `start` by a command of this Sandbox in a session of its + /// own (the SDK's `setsid` background process), so it ends with the + /// Sandbox too. One already gone needs nothing. + pub fn track(&self, pid: u32, start: u64) -> Result<(), Box> { + match LocalProcess::observe(pid) { + Ok(process) if process.start == start && process.alive() => { + groups::name(&self.directory, process).map_err(|error| self.io(error)) } + _ => Ok(()), + } + } + /// Reaps the daemon this process started, if it exited. True when no + /// daemon this process started is left unreaped. + pub(super) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; } + reaped } - /// Sends `signal` once and waits up to `wait` for the process to end. + /// The daemon has ended and, if this process started it, is reaped. Its + /// main thread can end (a zombie id, so not alive) while its other + /// threads still run and it cannot be reaped yet: not gone. + fn gone(&self, process: LocalProcess) -> bool { + self.reap() && !process.alive() + } + /// Sends `signal` once and waits up to `wait` for the process to be gone. + /// A KILL also goes to the daemon this process started and has not + /// reaped: its id cannot be reused until reaped, so it is still the one + /// started, though its main thread may have ended already. fn end( &self, process: LocalProcess, @@ -224,10 +266,15 @@ impl Sandbox { wait: Duration, ) -> Result<(), Box> { process.signal(signal).map_err(|error| self.io(error))?; + if signal == Signal::SIGKILL { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + let _ = started.kill(); + } + } let deadline = Instant::now() + wait; loop { - self.reap(); - if !process.alive() || Instant::now() >= deadline { + if self.gone(process) || Instant::now() >= deadline { return Ok(()); } std::thread::sleep(PAUSE); @@ -257,3 +304,12 @@ impl Sandbox { }) } } + +impl Drop for Sandbox { + /// A daemon this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running daemon + /// is left alone: stopping it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} From 1f2307e83d194a99e679dd745b575d3fb64035f4 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:17 +0800 Subject: [PATCH 27/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/create.rs (production). No tests changed or deleted except additions. Production LOC: 109,868 -> 109,882 (delta +14) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../src/backend/local/create.rs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/create.rs b/core/crates/layerfs-sandbox/src/backend/local/create.rs index eadff04d72..b352dadcdc 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/create.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/create.rs @@ -75,8 +75,10 @@ pub struct LocalCreateFailure { impl Local { /// Selects `root`: an existing absolute directory owned by this user, - /// writable by it alone and traversable by others (commands run as - /// another identity and must reach their Workspace below it). + /// writable by it alone and traversable by others, as is every directory + /// above it (commands enter their Workspace below it only after taking + /// another identity, so one closed directory on the way, such as a 0700 + /// `/root`, would fail every command at spawn). pub fn new(root: impl AsRef) -> Result { let root = root.as_ref(); if !root.is_absolute() @@ -96,6 +98,21 @@ impl Local { "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", )); } + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } Ok(Self { root: root.to_owned(), name: None, From 4a66f35cd502fdeff64925ecf9aad7e8b1e35d5c Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:19 +0800 Subject: [PATCH 28/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/mounts.rs (production). No tests changed or deleted except additions. Production LOC: 109,882 -> 109,870 (delta -12) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../layerfs-sandbox/src/backend/local/mounts.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/mounts.rs b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs index e7b84b8ded..290d94beaf 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/mounts.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs @@ -56,7 +56,6 @@ fn unescape(text: &str) -> String { /// Detaches every mount under `directory` once (lazy unmount: a process /// still inside keeps its view; new lookups no longer reach it), then checks /// that none is left. -#[cfg(target_os = "linux")] pub(super) fn detach(directory: &Path) -> io::Result<()> { use nix::mount::{umount2, MntFlags}; for point in under(directory)? { @@ -71,15 +70,3 @@ pub(super) fn detach(directory: &Path) -> io::Result<()> { Err(io::Error::other("a mount under the local Sandbox remains")) } } - -#[cfg(not(target_os = "linux"))] -pub(super) fn detach(directory: &Path) -> io::Result<()> { - if under(directory)?.is_empty() { - Ok(()) - } else { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "detaching a local Sandbox mount needs Linux", - )) - } -} From cba6af04ab11edbdec1aea7e68147ac115240d70 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:21 +0800 Subject: [PATCH 29/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/local/mod.rs (production). No tests changed or deleted except additions. Production LOC: 109,870 -> 109,871 (delta +1) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- core/crates/layerfs-sandbox/src/backend/local/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/mod.rs b/core/crates/layerfs-sandbox/src/backend/local/mod.rs index 8de8a6f18c..870b54c0fd 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/mod.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/mod.rs @@ -4,10 +4,13 @@ //! It is for a machine that is itself the isolation boundary (a microVM, a //! CI job). It isolates nothing: the daemon and its commands share this //! kernel's processes, network, users and filesystem with everything else on -//! it. Commands still run as their own nonroot identity. One attempt per -//! operation, as in the Docker runtime; nothing is retried. +//! it. Commands still run as their own nonroot identity, and what they leave +//! running ends with the Sandbox. One attempt per operation, as in the Docker +//! runtime; nothing is retried. Linux only: it reads `/proc` and detaches +//! mounts. mod create; mod exec; +mod groups; mod mounts; mod process; mod sandbox; From f71df4a084a0fc0e647e04865ee402d8810f6962 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:23 +0800 Subject: [PATCH 30/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/src/backend/mod.rs (production). No tests changed or deleted except additions. Production LOC: 109,871 -> 109,872 (delta +1) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- core/crates/layerfs-sandbox/src/backend/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/crates/layerfs-sandbox/src/backend/mod.rs b/core/crates/layerfs-sandbox/src/backend/mod.rs index 7fe356b929..1512ff5338 100644 --- a/core/crates/layerfs-sandbox/src/backend/mod.rs +++ b/core/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,3 +1,4 @@ //! Concrete ordinary runtime providers. pub mod docker; +#[cfg(target_os = "linux")] pub mod local; From 0466af2cbb2dfc74280ee0e2906c55e9994154c2 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:25 +0800 Subject: [PATCH 31/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: core/crates/layerfs-sandbox/tests/local_runtime.rs (not production (test)). No tests changed or deleted except additions. Production LOC: 109,872 -> 109,872 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- .../layerfs-sandbox/tests/local_runtime.rs | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) diff --git a/core/crates/layerfs-sandbox/tests/local_runtime.rs b/core/crates/layerfs-sandbox/tests/local_runtime.rs index f90e682433..c948c7ff5e 100644 --- a/core/crates/layerfs-sandbox/tests/local_runtime.rs +++ b/core/crates/layerfs-sandbox/tests/local_runtime.rs @@ -2,6 +2,8 @@ //! Sandbox directory, observed by process id and start time, its commands run //! as their own identity, and its stop and delete each attempted once. A //! small shell script stands in for the daemon; nothing here needs root. +//! The runtime is Linux only, and so are these tests. +#![cfg(target_os = "linux")] use layerfs_sandbox::{ backend::{ docker::ExecRequest, @@ -411,3 +413,224 @@ fn a_zombie_is_not_alive() { assert!(!process.alive(), "an exited, unreaped child counted alive"); child.wait().unwrap(); } + +#[test] +fn every_directory_above_a_root_must_be_traversable_by_others() { + let (root, _) = fixture("exit 0"); + let base = root.parent().unwrap().to_owned(); + // a root inside a directory others cannot pass, as `/root` (0700) is + let closed = base.join("closed"); + fs::create_dir(&closed).unwrap(); + fs::create_dir(closed.join("root")).unwrap(); + fs::set_permissions(closed.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(&closed, fs::Permissions::from_mode(0o700)).unwrap(); + let refused = |path: &Path| match Local::new(path) { + Err(RuntimeError::Io(error)) => { + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + error.to_string() + } + other => panic!("{} admitted: {other:?}", path.display()), + }; + let message = refused(&closed.join("root")); + assert!( + message.contains(&format!("{} is not traversable", closed.display())), + "{message}" + ); + // the same through a symlinked parent: the directory it resolves to counts + std::os::unix::fs::symlink(&closed, base.join("link")).unwrap(); + let message = refused(&base.join("link").join("root")); + assert!(message.contains("is not traversable"), "{message}"); + // traversable without being listable is enough + fs::set_permissions(&closed, fs::Permissions::from_mode(0o711)).unwrap(); + Local::new(closed.join("root")).unwrap(); + Local::new(base.join("link").join("root")).unwrap(); +} + +/// The process ids a command printed, one per line. +fn pids(out: &str) -> Vec { + out.lines() + .map(|line| line.trim().parse().unwrap()) + .collect() +} + +/// True once none of `pids` runs any more (a zombie has ended). +fn gone(pids: &[u32]) -> bool { + pids.iter().all( + |pid| match fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(stat) => matches!( + stat.rsplit_once(") ").map(|(_, rest)| &rest[..1]), + Some("Z" | "X") + ), + Err(_) => true, + }, + ) +} + +/// Leaves behind what an exec can: a `nohup` server writing into the +/// Sandbox's scratch directory, and a `setsid` one named with +/// [`Sandbox::track`] as the SDK names its background processes. +fn leave_running(sandbox: &Sandbox) -> Vec { + let home = format!("HOME={}", sandbox.tmp().display()); + let (code, out, err) = run( + sandbox, + &[ + "bash", + "-c", + "cd \"$HOME\"; nohup sh -c 'while :; do echo x >> \"$HOME/server.out\"; sleep 0.01; done' >/dev/null 2>&1 & echo $!", + ], + &[&home], + ); + assert_eq!(code, 0, "{err}"); + let mut left = pids(&out); + let (code, out, err) = run( + sandbox, + &[ + "bash", + "-c", + "setsid sleep 300 >/dev/null 2>&1 = 2, "{named}"); + let (code, _, _) = run(sandbox, &["true"], &[]); + assert_eq!(code, 0); + assert_eq!(groups(), named); + let deadline = Instant::now() + WAIT; + while !sandbox.tmp().join("server.out").exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!gone(&left)); + left +} + +#[test] +fn what_commands_leave_running_ends_with_the_sandbox() { + // stopped and deleted by its owner + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-8"); + let left = leave_running(&sandbox); + sandbox.stop(1).unwrap(); + assert!(gone(&left), "{left:?} outlived the stop"); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-8").exists()); + // its daemon killed from outside, then found stopped from its record + // and deleted, as a reclaim does + let sandbox = started(&root, &daemon, "layerfs-shop-9"); + let left = leave_running(&sandbox); + let process = sandbox.process().unwrap(); + assert!(Command::new("kill") + .args(["-9", &process.pid.to_string()]) + .status() + .unwrap() + .success()); + let deadline = Instant::now() + WAIT; + while sandbox.endpoint().is_ok() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert!(!gone(&left), "a command ended with its daemon"); + let directory = sandbox.directory().to_owned(); + let mut stopped = Sandbox::stopped( + directory.clone(), + sandbox.identity(), + process, + 40404, + Vec::new(), + ); + stopped.delete().unwrap(); + assert!(gone(&left), "{left:?} outlived the delete"); + assert!(!directory.exists()); +} + +#[test] +fn a_process_of_another_boot_is_not_alive_and_one_without_a_boot_is_never_signalled() { + let (root, daemon) = fixture(LISTENS); + let sandbox = started(&root, &daemon, "layerfs-shop-10"); + let process = sandbox.process().unwrap(); + let boot = fs::read_to_string("/proc/sys/kernel/random/boot_id").unwrap(); + let boot = u128::from_str_radix(&boot.trim().replace('-', ""), 16).unwrap(); + assert_eq!(process.boot, Some(boot)); + // the same id and start time in another boot (a reboot or a restored + // snapshot) is another process + let other = LocalProcess { + boot: Some(boot ^ 1), + ..process + }; + assert!(!other.alive()); + assert!(Sandbox::reopen( + root.join("layerfs-shop-10"), + sandbox.identity(), + other, + 40404, + Vec::new() + ) + .unwrap_err() + .container_stopped()); + // a record from before boot ids is matched by id and start as then, and + // a stop through it signals nothing + let legacy = LocalProcess { + boot: None, + ..process + }; + assert!(legacy.alive()); + let mut reopened = Sandbox::reopen( + root.join("layerfs-shop-10"), + sandbox.identity(), + legacy, + 40404, + Vec::new(), + ) + .unwrap(); + assert!(reopened.stop(1).is_err()); + assert!(process.alive()); + let mut sandbox = sandbox; + sandbox.stop(1).unwrap(); + assert!(!process.alive()); + sandbox.delete().unwrap(); +} + +/// A daemon whose main thread ends while another thread still runs: its +/// id is then a zombie that cannot be reaped yet, so it is not alive, but +/// its exit has not happened. A stop ends and reaps it before returning, +/// so it never stays a zombie of this process (the stand-in needs python3). +#[test] +fn a_stop_reaps_a_daemon_whose_main_thread_ended_first() { + let python = Command::new("python3").args(["-c", ""]).status(); + if !python.is_ok_and(|status| status.success()) { + eprintln!("skipped: the stand-in daemon needs python3"); + return; + } + let (root, daemon) = fixture( + r#"echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" +exec python3 -c 'import ctypes, threading, time; threading.Thread(target=time.sleep, args=(60,)).start(); ctypes.CDLL(None).pthread_exit(None)'"#, + ); + let mut sandbox = started(&root, &daemon, "layerfs-shop-11"); + let process = sandbox.process().unwrap(); + let deadline = Instant::now() + WAIT; + while process.alive() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process.alive(), "its main thread did not end"); + let tasks = fs::read_dir(format!("/proc/{}/task", process.pid)) + .unwrap() + .count(); + assert!(tasks >= 2, "its other thread ended already ({tasks})"); + sandbox.stop(1).unwrap(); + assert!( + fs::metadata(format!("/proc/{}", process.pid)).is_err(), + "the stopped daemon is still a process of this kernel (unreaped)" + ); + sandbox.delete().unwrap(); + assert!(!root.join("layerfs-shop-11").exists()); +} From 9e0b6a022de683e28be6aa976d78a27da4460217 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:27 +0800 Subject: [PATCH 32/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/runtime.rs (production). No tests changed or deleted except additions. Production LOC: 109,872 -> 109,906 (delta +34) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/runtime.rs | 69 +++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/sdk/js/native/src/runtime.rs b/sdk/js/native/src/runtime.rs index ef9737c76e..e3d460592a 100644 --- a/sdk/js/native/src/runtime.rs +++ b/sdk/js/native/src/runtime.rs @@ -2,21 +2,25 @@ //! `deploy.sandbox` `local` (LAYERFS_SANDBOX=local) a daemon process on this //! kernel, for a VM or CI box that is itself the isolation boundary. Both run //! the same daemon and answer the same calls; only how it is started, how -//! commands are launched and how it is observed differ. +//! commands are launched and how it is observed differ. The local runtime +//! is Linux only; elsewhere selecting it is refused as Unsupported. use crate::job::{failed, refused}; use crate::output::Spool; use layerfs_bridge::control::{SdkRefusal, SdkValue}; -use layerfs_sandbox::backend::{docker, local}; +use layerfs_sandbox::backend::docker; +#[cfg(target_os = "linux")] +use layerfs_sandbox::backend::local; use layerfs_sandbox::{CommandIdentity, ContainerId, WireFailure}; use std::net::SocketAddr; -/// The image a local Sandbox records: placement keeps local and Docker -/// Sandboxes of one project apart by it. +/// A local Sandbox record's `runtime`, and the image it records (a Docker +/// record has no `runtime`). pub(crate) const LOCAL: &str = "local"; /// One Sandbox's runtime owner. pub(crate) enum Runtime { Docker(docker::Sandbox), + #[cfg(target_os = "linux")] Local(local::Sandbox), } @@ -34,17 +38,38 @@ pub(crate) fn local(deploy: &SdkValue) -> Result { } } -/// The image a new Workspace of this deployment is placed by: `local` for -/// the local runtime, else `deploy.image`. -pub(crate) fn image(args: &SdkValue) -> Result, SdkRefusal> { +/// What a new Workspace of a deployment is placed by: its runtime and, for +/// Docker, its image (`deploy.image`). +#[derive(Debug)] +pub(crate) struct Placement { + local: bool, + image: Option, +} + +/// The call's placement; none without a deployment (no filter, as before). +pub(crate) fn placement(args: &SdkValue) -> Result, SdkRefusal> { let Some(deploy) = args.get("deploy") else { return Ok(None); }; - Ok(if local(deploy)? { - Some(LOCAL.to_owned()) - } else { - deploy.get_text("image").map(str::to_owned) - }) + let local = local(deploy)?; + Ok(Some(Placement { + local, + image: (!local) + .then(|| deploy.get_text("image").map(str::to_owned)) + .flatten(), + })) +} + +impl Placement { + /// Whether a Sandbox record may take this Workspace: the same runtime + /// (the record's `runtime`, which a Docker record does not have) and, for + /// Docker, the same image. The image string alone does not tell them + /// apart: `local` is also a valid Docker image name. + pub(crate) fn admits(&self, record: &SdkValue) -> bool { + let local = record.get_text("runtime") == Some(LOCAL); + local == self.local + && (local || self.image.is_none() || record.get_text("image") == self.image.as_deref()) + } } impl Runtime { @@ -52,6 +77,7 @@ impl Runtime { pub(crate) fn identity(&self) -> ContainerId { match self { Self::Docker(sandbox) => sandbox.identity(), + #[cfg(target_os = "linux")] Self::Local(sandbox) => sandbox.identity(), } } @@ -59,6 +85,7 @@ impl Runtime { pub(crate) fn image(&self) -> &str { match self { Self::Docker(sandbox) => &sandbox.request().image, + #[cfg(target_os = "linux")] Self::Local(_) => LOCAL, } } @@ -68,6 +95,7 @@ impl Runtime { pub(crate) fn endpoint(&self) -> Result> { match self { Self::Docker(sandbox) => sandbox.endpoint(), + #[cfg(target_os = "linux")] Self::Local(sandbox) => sandbox.endpoint(), } } @@ -77,6 +105,7 @@ impl Runtime { pub(crate) fn scratch(&self) -> String { match self { Self::Docker(_) => "/tmp".to_owned(), + #[cfg(target_os = "linux")] Self::Local(sandbox) => sandbox.tmp().display().to_string(), } } @@ -84,9 +113,24 @@ impl Runtime { pub(crate) fn delete(&mut self) -> Result<(), Box> { match self { Self::Docker(sandbox) => sandbox.delete(), + #[cfg(target_os = "linux")] Self::Local(sandbox) => sandbox.delete(), } } + /// Names a background process a command started in a session of its own + /// (`pid`, started at `start`), so it ends with a local Sandbox as it + /// ends with a container. + #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] + pub(crate) fn track(&self, pid: u32, start: &str) -> Result<(), SdkRefusal> { + match self { + Self::Docker(_) => Ok(()), + #[cfg(target_os = "linux")] + Self::Local(sandbox) => { + let start = start.parse::().map_err(failed)?; + sandbox.track(pid, start).map_err(failed) + } + } + } /// One command to its end: exit code, stdout, stderr. `spool` sends /// every non-empty stream to its file instead of keeping it inline. pub(crate) fn run( @@ -117,6 +161,7 @@ impl Runtime { .known_root_exit() .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))? } + #[cfg(target_os = "linux")] Self::Local(sandbox) => { let request = docker::ExecRequest { identity, From fd351807a37a51911ee6f4785b38ccf4d0723b7c Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:30 +0800 Subject: [PATCH 33/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/local.rs (production). No tests changed or deleted except additions. Production LOC: 109,906 -> 109,958 (delta +52) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/local.rs | 106 +++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 22 deletions(-) diff --git a/sdk/js/native/src/local.rs b/sdk/js/native/src/local.rs index 4ca4201219..57acba623a 100644 --- a/sdk/js/native/src/local.rs +++ b/sdk/js/native/src/local.rs @@ -23,7 +23,7 @@ use layerfs_bridge::provision::StoreManifest; use layerfs_sandbox::backend::local::{Local, LocalCreateFailure, Sandbox}; use layerfs_sandbox::CommandIdentity; use std::net::{SocketAddr, TcpStream}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::Arc; /// Starts the project's next Sandbox as a local daemon. Called by @@ -93,12 +93,7 @@ fn start_claimed( (local, executable, store): (Local, PathBuf, String), ) -> Result, SdkRefusal> { let name = text(args, "project")?; - let prefix = args - .get("deploy") - .and_then(|d| d.get_text("containerPrefix")) - .filter(|prefix| !prefix.is_empty()) - .unwrap_or(crate::sandbox::PREFIX); - let sandbox_name = format!("{prefix}{name}-{id}"); + let sandbox_name = sandbox_name(args, id)?; let local = local .naming(&sandbox_name) .map_err(|_| refused("InvalidInput", format!("local Sandbox name {sandbox_name}")))?; @@ -167,7 +162,7 @@ fn start_claimed( peer: daemon.public, instance, }; - let state = Arc::new(SandboxState { + let state = SandboxState { id: id.to_owned(), project: name.to_owned(), name: sandbox_name, @@ -177,24 +172,91 @@ fn start_claimed( controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), - }); - crate::discovery::save_sandbox( - &state.directory, - &state, - args, - controller.private, - daemon.public, - instance, - )?; - crate::discovery::save_text( - &state.directory.join("container"), - &state.runtime.identity().to_string(), - )?; - std::fs::remove_file(directory.join("create.intent")).map_err(failed)?; + }; + let recorded = (|| { + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed) + })(); + if let Err(refusal) = recorded { + // The daemon runs and is this start's own: it is stopped and its + // directory deleted like any failed start's, and the records written + // so far are removed, so its number is free again. + let SandboxState { runtime, .. } = state; + let Runtime::Local(sandbox) = runtime else { + unreachable!("a local start owns a local Sandbox") + }; + for record in ["sandbox.record", "container"] { + let _ = std::fs::remove_file(saved_directory.join(record)); + } + return Err(abandon( + directory, + id, + format!("local sandbox records: {}", refusal.message), + Some(sandbox), + )); + } + let state = Arc::new(state); lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); Ok(state) } +/// A local Sandbox's directory name: `-`, the name its +/// container would have. +fn sandbox_name(args: &SdkValue, id: &str) -> Result { + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + Ok(format!("{prefix}{}-{id}", text(args, "project")?)) +} + +/// Where this deployment puts Sandbox `id`: +/// `/-`, computed from the +/// configuration alone, as a record's directory must be. +pub(crate) fn directory(args: &SdkValue, id: &str) -> Result { + let root = (args.get("deploy")) + .and_then(|deploy| deploy.get_text("localDir")) + .map(Path::new) + .ok_or_else(|| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a local Sandbox; its record is used only with \ + LAYERFS_LOCAL_DIR (deploy.localDir) configured" + ), + ) + })?; + let name = sandbox_name(args, id)?; + let plain = |path: &Path| { + !path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + }; + let single = matches!( + Path::new(&name).components().collect::>()[..], + [Component::Normal(_)] + ); + if !root.is_absolute() || !plain(root) || !single { + return Err(refused( + "InvalidInput", + format!("local Sandbox directory {}/{name}", root.display()), + )); + } + Ok(root.join(name)) +} + /// The first control connection to a new daemon: one connect, one /// authentication, one Hello that waits for the Store and must find the /// daemon ControlReady; its instance identifies the daemon from then on. From 0cf539a392fa25bb9a3279bf2d047478689af31d Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:32 +0800 Subject: [PATCH 34/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/discovery.rs (production). No tests changed or deleted except additions. Production LOC: 109,958 -> 109,995 (delta +37) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/discovery.rs | 75 ++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/sdk/js/native/src/discovery.rs b/sdk/js/native/src/discovery.rs index 7862715723..c2725160f0 100644 --- a/sdk/js/native/src/discovery.rs +++ b/sdk/js/native/src/discovery.rs @@ -10,9 +10,10 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; +#[cfg(target_os = "linux")] +use layerfs_sandbox::backend::local::{self, LocalProcess}; use layerfs_sandbox::{ backend::docker::{Docker, Sandbox, SandboxRequest}, - backend::local::{self, LocalProcess}, CommandIdentity, ContainerId, WireFailure, }; use std::{ @@ -72,6 +73,7 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } /// Whether this process runs as root, which the local runtime needs: its /// daemon mounts FUSE and its commands run as another uid/gid. +#[cfg(target_os = "linux")] pub(crate) fn root() -> bool { unsafe { geteuid() == 0 } } @@ -374,10 +376,14 @@ pub(crate) fn save_sandbox( ("volume", SdkValue::text(&runtime.request().store_volume)), ("labels", labels(&runtime.request().labels)), ]), + #[cfg(target_os = "linux")] Runtime::Local(runtime) => { let process = runtime .process() .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + let boot = process + .boot + .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; fields.extend([ ("runtime", SdkValue::text(crate::runtime::LOCAL)), ( @@ -386,6 +392,7 @@ pub(crate) fn save_sandbox( ), ("pid", SdkValue::Int(i64::from(process.pid))), ("start", SdkValue::text(process.start.to_string())), + ("boot", SdkValue::text(format!("{boot:032x}"))), ("port", SdkValue::Int(i64::from(runtime.port()))), ("image", SdkValue::text(crate::runtime::LOCAL)), ("labels", labels(runtime.labels())), @@ -470,6 +477,7 @@ fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { /// a local daemon process, its directory and port. pub(crate) enum Recorded { Docker(Docker, ContainerId, SandboxRequest), + #[cfg(target_os = "linux")] Local { directory: PathBuf, id: ContainerId, @@ -478,38 +486,61 @@ pub(crate) enum Recorded { labels: Vec<(String, String)>, }, } -/// Reads a Sandbox record's runtime. A local record's directory must be -/// absolute, plain and named for its Sandbox, so a damaged record never -/// points a delete elsewhere. -pub(crate) fn recorded(value: &SdkValue) -> Result { +/// Reads Sandbox `id`'s record's runtime. +pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { if value.get_text("runtime") != Some(crate::runtime::LOCAL) { let (docker, container, request) = runtime(value)?; return Ok(Recorded::Docker(docker, container, request)); } + local_record(args, id, value) +} +/// A local record is used only where this deployment puts that Sandbox: +/// its directory must be exactly `/-` +/// and named so, so a damaged, edited or stale record (another +/// LAYERFS_LOCAL_DIR or prefix) never points a delete elsewhere. A record +/// without `boot` was written before boot ids were kept: its daemon is +/// matched by id and start time as then, and never signalled. +#[cfg(target_os = "linux")] +fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { let directory = PathBuf::from(registry::text(value, "directory")?); - let name = registry::text(value, "name")?; - if name.is_empty() - || !directory.is_absolute() - || directory.file_name().and_then(|n| n.to_str()) != Some(name) - || directory.components().any(|c| { - matches!( - c, - std::path::Component::ParentDir | std::path::Component::CurDir - ) - }) + let expected = crate::local::directory(args, id)?; + if directory != expected + || expected.file_name().and_then(|n| n.to_str()) != Some(registry::text(value, "name")?) { - return Err(refused("InvalidInput", "local Sandbox record directory")); + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id}'s local record names {} (as {}), but this deployment puts it at \ + {}: the record is damaged or from another LAYERFS_LOCAL_DIR or prefix, so it is \ + neither used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + directory.display(), + value.get_text("name").unwrap_or_default(), + expected.display(), + registry::text(args, "project")? + ), + )); } let number = |key: &str| { value .get_int(key) .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) }; + let boot = match value.get_text("boot") { + None => None, + Some(hex) => Some( + (hex.len() == 32) + .then(|| u128::from_str_radix(hex, 16).ok()) + .flatten() + .ok_or_else(|| refused("InvalidInput", "local Sandbox record boot"))?, + ), + }; let process = LocalProcess { pid: u32::try_from(number("pid")?).map_err(failed)?, start: registry::text(value, "start")? .parse::() .map_err(failed)?, + boot, }; Ok(Recorded::Local { directory, @@ -519,6 +550,13 @@ pub(crate) fn recorded(value: &SdkValue) -> Result { labels: labels(value)?, }) } +#[cfg(not(target_os = "linux"))] +fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { + Err(refused( + "Unsupported", + format!("Sandbox {id} is a local Sandbox (LAYERFS_SANDBOX=local), which needs Linux"), + )) +} impl Recorded { /// The runtime, observed running once (it starts nothing). pub(crate) fn reopen(self) -> std::result::Result> { @@ -526,6 +564,7 @@ impl Recorded { Self::Docker(docker, id, request) => { Sandbox::reopen(docker, id, request).map(Runtime::Docker) } + #[cfg(target_os = "linux")] Self::Local { directory, id, @@ -541,6 +580,7 @@ impl Recorded { Self::Docker(docker, id, request) => { Runtime::Docker(Sandbox::stopped(docker, id, request)) } + #[cfg(target_os = "linux")] Self::Local { directory, id, @@ -569,10 +609,11 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let recorded = recorded(&value)?; + let recorded = recorded(args, id, &value)?; // a local daemon listens on this kernel's loopback; no other host reaches it let host = match recorded { Recorded::Docker(..) => crate::pool::daemon_host(args), + #[cfg(target_os = "linux")] Recorded::Local { .. } => None, }; let runtime = recorded.reopen().map_err(|failure| { From e88bd55aa9a1a05aa5ef7a4a254d83f68cc711e7 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:35 +0800 Subject: [PATCH 35/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/exec.rs (production). No tests changed or deleted except additions. Production LOC: 109,995 -> 110,005 (delta +10) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/exec.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sdk/js/native/src/exec.rs b/sdk/js/native/src/exec.rs index 1ee27b45dd..985d853975 100644 --- a/sdk/js/native/src/exec.rs +++ b/sdk/js/native/src/exec.rs @@ -110,6 +110,18 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { "background process exited before its identity was observed", ) })?; + // its own session leaves the command's process group; named, it still + // ends with a local Sandbox. Unnamed it would outlive it: it ends here, once. + if let Err(refusal) = state.runtime.track(pid, &start) { + let end = format!("kill -KILL -- -{pid} 2>/dev/null; true"); + let _ = run( + &state, + &directory, + vec!["bash".into(), "-c".into(), end], + false, + ); + return Err(refusal); + } let process = Process { path: path.to_owned(), command: command.to_owned(), From 0f156e3f6e09b4a2e31e33ba32b2b191d132617e Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:37 +0800 Subject: [PATCH 36/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/sandbox.rs (production). No tests changed or deleted except additions. Production LOC: 110,005 -> 110,015 (delta +10) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/sandbox.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sdk/js/native/src/sandbox.rs b/sdk/js/native/src/sandbox.rs index 90d665e27f..d8d69c7996 100644 --- a/sdk/js/native/src/sandbox.rs +++ b/sdk/js/native/src/sandbox.rs @@ -114,7 +114,14 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result return Err(failed(failure)), _ => {} } From 3fa366c68f869760b9cc6bc850478f9256dd3400 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:39 +0800 Subject: [PATCH 37/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/workspace.rs (production). No tests changed or deleted except additions. Production LOC: 110,015 -> 110,018 (delta +3) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/workspace.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sdk/js/native/src/workspace.rs b/sdk/js/native/src/workspace.rs index 696e7de15f..fb52f24210 100644 --- a/sdk/js/native/src/workspace.rs +++ b/sdk/js/native/src/workspace.rs @@ -1,7 +1,8 @@ //! Workspace calls (#373). `workspace.mount(branch)` places the new //! Workspace: under the project's guard it takes the lowest-numbered running -//! Sandbox of the call's image (`deploy.image`) with a free slot, reclaiming -//! any whose container is gone, and starts the next Sandbox, on that image, +//! Sandbox of the call's runtime and image (`deploy.image`) with a free +//! slot, reclaiming any whose container is gone, and starts the next +//! Sandbox, on that image, //! when every such one is full. Sandboxes of one project may run different //! images. A Workspace is //! `{ sandbox, path }`; status, Commit and unmount go to its Sandbox's @@ -35,9 +36,8 @@ fn recorded(directory: &Path) -> Result { /// lowest-numbered running one of the call's image with a free slot, else a /// new one. fn place(args: &SdkValue, project: &Path) -> Result, SdkRefusal> { - // `local` for the local runtime: local and Docker Sandboxes never share - let image = crate::runtime::image(args)?; - let image = image.as_deref(); + // by runtime, then image: local and Docker Sandboxes never share + let placement = crate::runtime::placement(args)?; for id in crate::discovery::sandbox_ids(args)? { let directory = crate::discovery::sandbox_dir(args, &id)?; let record = directory.join("sandbox.record"); @@ -45,8 +45,10 @@ fn place(args: &SdkValue, project: &Path) -> Result, SdkRefusa if !record.exists() || recorded(&directory)? >= WORKSPACES as usize { continue; } - if image.is_some() && crate::discovery::load(&record)?.get_text("image") != image { - continue; + if let Some(placement) = &placement { + if !placement.admits(&crate::discovery::load(&record)?) { + continue; + } } match sandbox_live(&naming(args, &id)) { Ok(state) => return Ok(state), @@ -124,7 +126,9 @@ pub(crate) fn list(args: &SdkValue) -> Outcome { let cached = lock(&crate::registry::SANDBOXES)?.get(&directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => crate::discovery::recorded(&value)?.reopen().map(drop), + None => crate::discovery::recorded(args, &id, &value)? + .reopen() + .map(drop), }; match observed { Ok(()) => items.push(item), From d53d4fb6916c83dbe5037b1e3c3918fc83b2da68 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:41 +0800 Subject: [PATCH 38/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/src/lib.rs (production). No tests changed or deleted except additions. Production LOC: 110,018 -> 110,019 (delta +1) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/js/native/src/lib.rs b/sdk/js/native/src/lib.rs index 5f9d2570ec..94b70f5f88 100644 --- a/sdk/js/native/src/lib.rs +++ b/sdk/js/native/src/lib.rs @@ -5,6 +5,7 @@ mod discovery; mod dispatch; mod exec; mod job; +#[cfg(target_os = "linux")] mod local; mod output; mod pool; From d46306711e0913d52cbc60b1347af137f8b9158c Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:43 +0800 Subject: [PATCH 39/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/tests/unit/runtime.rs (not production (test)). No tests changed or deleted except additions. Production LOC: 110,019 -> 110,019 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/tests/unit/runtime.rs | 160 ++++++++++++++++++++++------ 1 file changed, 129 insertions(+), 31 deletions(-) diff --git a/sdk/js/native/tests/unit/runtime.rs b/sdk/js/native/tests/unit/runtime.rs index c2dc1518a7..7dc96571bc 100644 --- a/sdk/js/native/tests/unit/runtime.rs +++ b/sdk/js/native/tests/unit/runtime.rs @@ -1,6 +1,6 @@ //! Which runtime a deployment selects, and what a local Sandbox record may //! name. Included by runtime.rs. -use super::{image, local, LOCAL}; +use super::{local, placement}; use layerfs_bridge::control::SdkValue; fn deploy(fields: &[(&str, &str)]) -> SdkValue { @@ -14,28 +14,13 @@ fn deploy(fields: &[(&str, &str)]) -> SdkValue { #[test] fn docker_stays_the_default_and_only_local_selects_local() { let image_digest = format!("sha256:{}", "a".repeat(64)); - // unset and `docker` are Docker, placed by the deployment's image as before + // unset and `docker` are Docker for fields in [vec![], vec![("sandbox", "docker")]] { let mut fields = fields; fields.push(("image", image_digest.as_str())); - let deploy = deploy(&fields); - assert!(!local(&deploy).unwrap()); - let args = SdkValue::map([("deploy", deploy)]); - assert_eq!( - image(&args).unwrap().as_deref(), - Some(image_digest.as_str()) - ); + assert!(!local(&deploy(&fields)).unwrap()); } - // no deployment: no image filter, as before - assert_eq!( - image(&SdkValue::map([("project", SdkValue::text("p"))])).unwrap(), - None - ); - // local ignores any image: its Sandboxes are placed as `local` - let deploy_local = deploy(&[("sandbox", "local"), ("image", image_digest.as_str())]); - assert!(local(&deploy_local).unwrap()); - let args = SdkValue::map([("deploy", deploy_local)]); - assert_eq!(image(&args).unwrap().as_deref(), Some(LOCAL)); + assert!(local(&deploy(&[("sandbox", "local")])).unwrap()); // anything else is refused, never taken as Docker or local for typo in ["Local", "lokal", "LOCAL", " local", "docker "] { let refusal = local(&deploy(&[("sandbox", typo)])).unwrap_err(); @@ -48,6 +33,68 @@ fn docker_stays_the_default_and_only_local_selects_local() { } } +/// A Sandbox record's placement fields: `runtime` (absent on a Docker +/// record) and `image`. +fn placed(runtime: Option<&str>, image: &str) -> SdkValue { + let mut fields = vec![("image".to_owned(), SdkValue::text(image))]; + if let Some(runtime) = runtime { + fields.push(("runtime".to_owned(), SdkValue::text(runtime))); + } + SdkValue::Map(fields) +} + +#[test] +fn placement_keeps_runtimes_apart_even_for_an_image_named_local() { + let digest = format!("sha256:{}", "a".repeat(64)); + let local_record = placed(Some("local"), "local"); + let docker_record = placed(None, &digest); + // a Docker deployment whose image is `local`, a valid Docker image name + let docker_local = SdkValue::map([("deploy", deploy(&[("image", "local")]))]); + let docker_local = placement(&docker_local).unwrap().unwrap(); + assert!(!docker_local.admits(&local_record)); + assert!(docker_local.admits(&placed(None, "local"))); + // the same image is still the Docker filter, as before + let docker = SdkValue::map([("deploy", deploy(&[("image", digest.as_str())]))]); + let docker = placement(&docker).unwrap().unwrap(); + assert!(docker.admits(&docker_record)); + assert!(!docker.admits(&placed(None, &format!("sha256:{}", "b".repeat(64))))); + assert!(!docker.admits(&local_record)); + // a Docker deployment without an image still never takes a local Sandbox + let unset = SdkValue::map([("deploy", deploy(&[]))]); + let unset = placement(&unset).unwrap().unwrap(); + assert!(unset.admits(&docker_record)); + assert!(!unset.admits(&local_record)); + // a local deployment takes only local Sandboxes, whatever image it names + let local = SdkValue::map([( + "deploy", + deploy(&[("sandbox", "local"), ("image", "local")]), + )]); + let local = placement(&local).unwrap().unwrap(); + assert!(local.admits(&local_record)); + assert!(!local.admits(&placed(None, "local"))); + assert!(!local.admits(&docker_record)); + // no deployment: no filter, as before + assert!( + placement(&SdkValue::map([("project", SdkValue::text("p"))])) + .unwrap() + .is_none() + ); +} + +/// A call's fields for Sandbox 1 of project `p` deployed locally under `root`. +#[cfg(target_os = "linux")] +fn call(root: Option<&str>) -> SdkValue { + let mut fields = vec![("sandbox", "local")]; + if let Some(root) = root { + fields.push(("localDir", root)); + } + SdkValue::map([ + ("project", SdkValue::text("p")), + ("deploy", deploy(&fields)), + ]) +} + +#[cfg(target_os = "linux")] fn record(directory: &str, name: &str) -> SdkValue { SdkValue::map([ ("id", SdkValue::text("1")), @@ -57,39 +104,90 @@ fn record(directory: &str, name: &str) -> SdkValue { ("directory", SdkValue::text(directory)), ("pid", SdkValue::Int(4242)), ("start", SdkValue::text("123456")), + ("boot", SdkValue::text("0123456789abcdef0123456789abcdef")), ("port", SdkValue::Int(40404)), ]) } +#[cfg(target_os = "linux")] #[test] -fn a_local_record_names_its_own_directory_only() { - match crate::discovery::recorded(&record("/lfs/local/layerfs-p-1", "layerfs-p-1")) { - Ok(crate::discovery::Recorded::Local { +fn a_local_record_is_used_only_where_this_deployment_puts_its_sandbox() { + use crate::discovery::{recorded, Recorded}; + let args = call(Some("/lfs/local")); + match recorded(&args, "1", &record("/lfs/local/layerfs-p-1", "layerfs-p-1")) { + Ok(Recorded::Local { directory, process, port, .. }) => { assert_eq!(directory, std::path::Path::new("/lfs/local/layerfs-p-1")); - assert_eq!((process.pid, process.start, port), (4242, 123456, 40404)); + assert_eq!( + (process.pid, process.start, process.boot, port), + ( + 4242, + 123456, + Some(0x0123456789abcdef0123456789abcdef), + 40404 + ) + ); } _ => panic!("a valid local record was refused"), } for (directory, name) in [ + // a damaged or edited record naming any other directory by its own name + ("/srv/data", "data"), + ("/lfs/local/data", "data"), + // another LAYERFS_LOCAL_DIR, another Sandbox, another name + ("/old/local/layerfs-p-1", "layerfs-p-1"), + ("/lfs/local/layerfs-p-2", "layerfs-p-2"), + ("/lfs/local/layerfs-p-1", "layerfs-p-2"), + ("/lfs/local/layerfs-p-1/sub", "sub"), ("relative/layerfs-p-1", "layerfs-p-1"), - ("/lfs/local/other", "layerfs-p-1"), ("/lfs/../etc/layerfs-p-1", "layerfs-p-1"), ("/", ""), ] { + let refusal = match recorded(&args, "1", &record(directory, name)) { + Err(refusal) => refusal, + Ok(_) => panic!("{directory} admitted for {name}"), + }; + assert_eq!(refusal.code, "InvalidInput"); assert!( - crate::discovery::recorded(&record(directory, name)).is_err(), - "{directory} admitted for {name}" + refusal.message.contains("neither used nor deleted"), + "{}", + refusal.message ); } - let mut bad_port = record("/lfs/local/layerfs-p-1", "layerfs-p-1"); - if let SdkValue::Map(fields) = &mut bad_port { - fields.retain(|(k, _)| k != "port"); - fields.push(("port".into(), SdkValue::Int(70000))); + // the configured directory is needed to use a local record at all + for root in [None, Some("relative"), Some("/lfs/../local")] { + let refusal = match recorded( + &call(root), + "1", + &record("/lfs/local/layerfs-p-1", "layerfs-p-1"), + ) { + Err(refusal) => refusal, + Ok(_) => panic!("{root:?} admitted"), + }; + assert_eq!(refusal.code, "InvalidInput", "{}", refusal.message); + } + // a record from before boot ids is read without one; a bad one is refused + let mut legacy = record("/lfs/local/layerfs-p-1", "layerfs-p-1"); + if let SdkValue::Map(fields) = &mut legacy { + fields.retain(|(k, _)| k != "boot"); + } + match recorded(&args, "1", &legacy) { + Ok(Recorded::Local { process, .. }) => assert_eq!(process.boot, None), + _ => panic!("a record without a boot id was refused"), + } + for (key, value) in [ + ("boot", SdkValue::text("not-hex")), + ("port", SdkValue::Int(70000)), + ] { + let mut bad = record("/lfs/local/layerfs-p-1", "layerfs-p-1"); + if let SdkValue::Map(fields) = &mut bad { + fields.retain(|(k, _)| k != key); + fields.push((key.into(), value)); + } + assert!(recorded(&args, "1", &bad).is_err(), "{key}"); } - assert!(crate::discovery::recorded(&bad_port).is_err()); } From 8c22d1754f7b7f86fd5b8f8e6b4531338f819813 Mon Sep 17 00:00:00 2001 From: jiangkoumo <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:46:47 +0800 Subject: [PATCH 40/64] fix(sandbox): address the first review of the local runtime (#402) Part of one change pushed file by file through the GitHub contents API; the reviewable unit is the whole set (local commit 9334dbd22, one patch). This commit: sdk/js/native/tests/unit/reclaim.rs (not production (test)). No tests changed or deleted except additions. Production LOC: 110,019 -> 110,019 (delta 0) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 85255766e. --- sdk/js/native/tests/unit/reclaim.rs | 83 +++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/sdk/js/native/tests/unit/reclaim.rs b/sdk/js/native/tests/unit/reclaim.rs index da68c815e1..122df8ec63 100644 --- a/sdk/js/native/tests/unit/reclaim.rs +++ b/sdk/js/native/tests/unit/reclaim.rs @@ -226,3 +226,86 @@ fn a_stopped_container_the_engine_will_not_delete_keeps_its_records() { assert!(dir(&home).join("sandbox.record").exists()); std::fs::remove_dir_all(home).unwrap(); } + +/// Sandbox 1 recorded as a local Sandbox in `directory` named `name`, its +/// daemon long gone (no process has id u32::MAX). +#[cfg(target_os = "linux")] +fn local_record(home: &Path, directory: &Path, name: &str) { + let bytes = |n: u8| SdkValue::Bytes(vec![n; 32]); + crate::discovery::save( + &dir(home).join("sandbox.record"), + SdkValue::map([ + ("id", SdkValue::text(ID)), + ("project", SdkValue::text("proj")), + ("name", SdkValue::text(name)), + ("container", SdkValue::text(CID)), + ("runtime", SdkValue::text("local")), + ("directory", SdkValue::text(directory.to_str().unwrap())), + ("pid", SdkValue::Int(i64::from(u32::MAX))), + ("start", SdkValue::text("1")), + ("port", SdkValue::Int(40404)), + ("image", SdkValue::text("local")), + ("secret", bytes(1)), + ("peer", bytes(2)), + ("instance", bytes(3)), + ]), + ) + .unwrap(); +} + +#[cfg(target_os = "linux")] +#[test] +fn a_local_record_outside_its_configured_directory_is_never_deleted() { + let home = home(); + let root = home.join("local"); + std::fs::create_dir(&root).unwrap(); + let local = || { + let mut args = args(&home); + if let SdkValue::Map(fields) = &mut args { + fields.push(( + "deploy".into(), + SdkValue::map([ + ("sandbox", SdkValue::text("local")), + ("localDir", SdkValue::text(root.to_str().unwrap())), + ]), + )); + } + args + }; + // an edited or damaged record naming a data directory by its own name + let data = home.join("data"); + std::fs::create_dir(&data).unwrap(); + std::fs::write(data.join("keep"), b"x").unwrap(); + local_record(&home, &data, "data"); + let refusal = reclaim(&local(), ID, CID, true).unwrap_err(); + assert_eq!(refusal.code, "InvalidInput"); + assert!( + refusal.message.contains("neither used nor deleted"), + "{}", + refusal.message + ); + assert!( + data.join("keep").exists(), + "a delete reached {}", + data.display() + ); + assert!(dir(&home).join("sandbox.record").exists()); + // the same from a record of another LAYERFS_LOCAL_DIR + let old = home.join("old"); + std::fs::create_dir_all(old.join("layerfs-proj-1")).unwrap(); + local_record(&home, &old.join("layerfs-proj-1"), "layerfs-proj-1"); + assert_eq!( + reclaim(&local(), ID, CID, true).unwrap_err().code, + "InvalidInput" + ); + assert!(old.join("layerfs-proj-1").exists()); + // where this deployment puts it, the stopped Sandbox is deleted and reclaimed + let own = root.join("layerfs-proj-1"); + std::fs::create_dir_all(own.join("tmp")).unwrap(); + local_record(&home, &own, "layerfs-proj-1"); + let reclaimed = reclaim(&local(), ID, CID, true).unwrap(); + assert_eq!(reclaimed.get_text("kind"), Some("Reclaimed")); + assert!(!own.exists()); + assert!(!dir(&home).exists(), "its number is free again"); + std::fs::remove_dir_all(home).unwrap(); +} From e63af4c10a619d11af78d4bd25cd115e2cf28ba9 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Fri, 23 Oct 2026 00:30:00 +0800 Subject: [PATCH 41/64] feat(nfs): a loopback NFSv3 export of one mounted Workspace (layerfs-nfs) New crate layerfs-nfs serves one mounted Workspace to the host's own NFSv3 client on 127.0.0.1, so a Workspace can be mounted on macOS with the system /sbin/mount_nfs as an ordinary user: no kernel extension, no macFUSE or FUSE-T. It is the macOS native route's mount entry, beside FUSE. - Its own XDR, ONC RPC, MOUNT v3 and NFSv3 codecs; no third-party dependency, #![forbid(unsafe_code)]. - Lookups, attributes, reads and directory listings go through the same kernel request ports FUSE uses (layerfs-fuse, layerfs-workspace). Every changing procedure is one KernelMutation::perform, and its reply ticket is recorded before the reply is sent: a lost reply never replays a mutation. - Silly-rename (.nfs*) and AppleDouble (._*) names are handled inside the NFS layer; they never reach the Workspace or a Commit. - Bounded kernel references: past ExportConfig.lookups_held (default 65,536) the least recently used leaves are forgotten and re-resolved by their remembered name when a client presents them again. - Refresh marks (src/refresh.rs): a mode-only SETATTR on a marked inode answers the current attributes and changes nothing, once, within 30 s. The daemon uses it to make a macOS client drop stale caches after a Commit installs another root (next commit). - Boundary guard: layerfs-nfs is unsafe-free and may depend on content, overlay, workspace and fuse only, never on the daemon. The tests that drive the daemon's NFS entry (tests/daemon.rs, examples/daemon_nfs.rs) come with the daemon wiring in the next commit. No existing test changed or deleted. Part of the macOS native route; builds on #402 (local Sandbox runtime). Production LOC: 110,624 -> 113,293 (delta +2669) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 5f83fabe1. --- core/Cargo.lock | 17 + core/Cargo.toml | 2 + core/crates/layerfs-nfs/Cargo.toml | 22 + .../crates/layerfs-nfs/examples/serve_tree.rs | 101 +++ core/crates/layerfs-nfs/src/attributes.rs | 204 +++++ core/crates/layerfs-nfs/src/block.rs | 32 + core/crates/layerfs-nfs/src/change.rs | 531 ++++++++++++ core/crates/layerfs-nfs/src/changing.rs | 268 ++++++ core/crates/layerfs-nfs/src/directory.rs | 138 +++ core/crates/layerfs-nfs/src/handle.rs | 69 ++ core/crates/layerfs-nfs/src/kernel.rs | 811 ++++++++++++++++++ core/crates/layerfs-nfs/src/lib.rs | 35 + core/crates/layerfs-nfs/src/mount.rs | 66 ++ core/crates/layerfs-nfs/src/names.rs | 38 + core/crates/layerfs-nfs/src/nfs.rs | 261 ++++++ core/crates/layerfs-nfs/src/refresh.rs | 75 ++ core/crates/layerfs-nfs/src/rpc.rs | 181 ++++ core/crates/layerfs-nfs/src/server.rs | 333 +++++++ core/crates/layerfs-nfs/src/xdr.rs | 120 +++ core/crates/layerfs-nfs/tests/collect.rs | 170 ++++ core/crates/layerfs-nfs/tests/export.rs | 185 ++++ core/crates/layerfs-nfs/tests/protocol.rs | 123 +++ core/crates/layerfs-nfs/tests/refresh.rs | 117 +++ .../layerfs-nfs/tests/support/client.rs | 588 +++++++++++++ core/crates/layerfs-nfs/tests/support/rig.rs | 295 +++++++ core/crates/layerfs-nfs/tests/support/spy.rs | 199 +++++ core/crates/layerfs-nfs/tests/support/tree.rs | 136 +++ core/crates/layerfs-nfs/tests/write.rs | 590 +++++++++++++ core/tools/check_product_boundary.py | 7 +- 29 files changed, 5712 insertions(+), 2 deletions(-) create mode 100644 core/crates/layerfs-nfs/Cargo.toml create mode 100644 core/crates/layerfs-nfs/examples/serve_tree.rs create mode 100644 core/crates/layerfs-nfs/src/attributes.rs create mode 100644 core/crates/layerfs-nfs/src/block.rs create mode 100644 core/crates/layerfs-nfs/src/change.rs create mode 100644 core/crates/layerfs-nfs/src/changing.rs create mode 100644 core/crates/layerfs-nfs/src/directory.rs create mode 100644 core/crates/layerfs-nfs/src/handle.rs create mode 100644 core/crates/layerfs-nfs/src/kernel.rs create mode 100644 core/crates/layerfs-nfs/src/lib.rs create mode 100644 core/crates/layerfs-nfs/src/mount.rs create mode 100644 core/crates/layerfs-nfs/src/names.rs create mode 100644 core/crates/layerfs-nfs/src/nfs.rs create mode 100644 core/crates/layerfs-nfs/src/refresh.rs create mode 100644 core/crates/layerfs-nfs/src/rpc.rs create mode 100644 core/crates/layerfs-nfs/src/server.rs create mode 100644 core/crates/layerfs-nfs/src/xdr.rs create mode 100644 core/crates/layerfs-nfs/tests/collect.rs create mode 100644 core/crates/layerfs-nfs/tests/export.rs create mode 100644 core/crates/layerfs-nfs/tests/protocol.rs create mode 100644 core/crates/layerfs-nfs/tests/refresh.rs create mode 100644 core/crates/layerfs-nfs/tests/support/client.rs create mode 100644 core/crates/layerfs-nfs/tests/support/rig.rs create mode 100644 core/crates/layerfs-nfs/tests/support/spy.rs create mode 100644 core/crates/layerfs-nfs/tests/support/tree.rs create mode 100644 core/crates/layerfs-nfs/tests/write.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index 307a94e171..b9c1005ad3 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -618,6 +618,23 @@ dependencies = [ "layerfs-content", ] +[[package]] +name = "layerfs-nfs" +version = "1.0.0" +dependencies = [ + "layerfs-bridge", + "layerfs-content", + "layerfs-daemon", + "layerfs-fuse", + "layerfs-history", + "layerfs-overlay", + "layerfs-persistence", + "layerfs-project", + "layerfs-storage", + "layerfs-telemetry", + "layerfs-workspace", +] + [[package]] name = "layerfs-node" version = "1.0.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index deaf432445..4ce224a851 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -23,6 +23,8 @@ members = [ "crates/layerfs-bridge", "crates/layerfs-sandbox", "crates/layerfs-fuse", + # Loopback NFSv3 export over the same kernel request ports (native macOS route). + "crates/layerfs-nfs", # Node binding of the JavaScript SDK (owner-approved napi-rs, 2026-10-15). "../sdk/js/native", ] diff --git a/core/crates/layerfs-nfs/Cargo.toml b/core/crates/layerfs-nfs/Cargo.toml new file mode 100644 index 0000000000..5f39b20d10 --- /dev/null +++ b/core/crates/layerfs-nfs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "layerfs-nfs" +description = "Loopback NFSv3 service of one mounted Workspace over the kernel request ports" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +layerfs-content = { path = "../layerfs-content" } +layerfs-overlay = { path = "../layerfs-overlay" } +layerfs-workspace = { path = "../layerfs-workspace" } +layerfs-fuse = { path = "../layerfs-fuse" } + +[dev-dependencies] +layerfs-bridge = { path = "../layerfs-bridge" } +layerfs-daemon = { path = "../layerfs-daemon" } +layerfs-history = { path = "../layerfs-history" } +layerfs-persistence = { path = "../layerfs-persistence" } +layerfs-project = { path = "../layerfs-project" } +layerfs-storage = { path = "../layerfs-storage" } +layerfs-telemetry = { path = "../layerfs-telemetry" } diff --git a/core/crates/layerfs-nfs/examples/serve_tree.rs b/core/crates/layerfs-nfs/examples/serve_tree.rs new file mode 100644 index 0000000000..676d063b45 --- /dev/null +++ b/core/crates/layerfs-nfs/examples/serve_tree.rs @@ -0,0 +1,101 @@ +//! Serves a host tree, committed as a Branch, over loopback NFSv3. +//! +//! usage: serve_tree [uid gid] [write] +//! +//! Prints `READY
` once the export accepts calls. With `write` the +//! export is writable, and whenever `.commit` appears the export +//! closes its implicit file descriptors, the Workspace is committed with the +//! product Commit (`COMMITTED …`), and a second, independent Workspace bound +//! to the committed Branch is served read-only on `+1` (`VIEW …`); +//! the trigger file is then removed. It serves until `.stop` +//! exists; it then prints per-procedure call counts and time, gives back +//! every engine reference, revokes the engine mounts and removes +//! ``. Mount on Linux with, for example: +//! `mount -t nfs -o vers=3,proto=tcp,port=P,mountport=P,mountproto=tcp,nolock HOST:/ DIR`. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use std::{net::TcpListener, path::PathBuf, time::Duration}; + +fn main() { + let mut arguments: Vec = std::env::args().collect(); + let writable = arguments.last().is_some_and(|last| last == "write"); + if writable { + arguments.pop(); + } + if arguments.len() != 4 && arguments.len() != 6 { + eprintln!("usage: serve_tree [uid gid] [write]"); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let port: u16 = arguments[3].parse().expect("port"); + let (uid, gid) = if arguments.len() == 6 { + (arguments[4].parse().unwrap(), arguments[5].parse().unwrap()) + } else { + (501, 20) + }; + let stop = PathBuf::from(format!("{}.stop", work.display())); + let commit = PathBuf::from(format!("{}.commit", work.display())); + let started = std::time::Instant::now(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!( + "INIT entries={} seconds={:.2}", + rig.entries, + started.elapsed().as_secs_f64() + ); + let listener = TcpListener::bind(("127.0.0.1", port)).expect("bind"); + let server = rig.export(listener, uid, gid, !writable); + println!("READY {} writable={writable}", server.local_addr()); + let mut views = Vec::new(); + while !stop.exists() { + if writable && commit.exists() { + let closed = server.close_files(); + let at = std::time::Instant::now(); + let done = rig.commit(); + println!( + "COMMITTED closed_files={closed} ms={} history={:?} replaced={:?}", + at.elapsed().as_millis(), + done.history, + done.replaced + ); + drop(done); + let mut authority = [80; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let listener = + TcpListener::bind(("127.0.0.1", port + 1 + views.len() as u16)).expect("bind"); + let viewer = rig.serve_view(listener, &view, uid, gid); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + drop(std::fs::remove_file(&commit)); + } + std::thread::sleep(Duration::from_millis(200)); + } + let mut total = (0, 0); + for count in server.counters() { + total.0 += count.calls; + total.1 += count.nanoseconds; + println!( + "PROC {:<12} calls={:<8} total_ms={:<10.1} avg_us={:.1}", + count.procedure, + count.calls, + count.nanoseconds as f64 / 1e6, + count.nanoseconds as f64 / 1e3 / count.calls as f64 + ); + } + println!( + "TOTAL calls={} server_ms={:.1}", + total.0, + total.1 as f64 / 1e6 + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + let released = server.stop(); + println!("RELEASED {released:?}"); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); + drop(std::fs::remove_file(stop)); +} diff --git a/core/crates/layerfs-nfs/src/attributes.rs b/core/crates/layerfs-nfs/src/attributes.rs new file mode 100644 index 0000000000..70fd740c1d --- /dev/null +++ b/core/crates/layerfs-nfs/src/attributes.rs @@ -0,0 +1,204 @@ +//! NFSv3 (RFC 1813) status codes and the projection of Workspace inode facts. +use crate::xdr::Writer; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_workspace::{Refusal, ViewStat}; + +/// `nfsstat3` values this server returns. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum Status { + Ok = 0, + Perm = 1, + NoEnt = 2, + Io = 5, + Access = 13, + Exist = 17, + NotDir = 20, + IsDir = 21, + Inval = 22, + FBig = 27, + NoSpc = 28, + RoFs = 30, + MLink = 31, + NameTooLong = 63, + NotEmpty = 66, + Stale = 70, + BadHandle = 10001, + NotSync = 10002, + BadCookie = 10003, + NotSupp = 10004, + TooSmall = 10005, + ServerFault = 10006, + /// The request may succeed later; the client retries after a delay. + Jukebox = 10008, +} +impl Status { + pub const fn code(self) -> u32 { + self as u32 + } +} +/// A Workspace refusal as the status the client sees; the same errnos the +/// FUSE adapter replies with. +pub fn refusal(value: Refusal) -> Status { + match value { + Refusal::Exists => Status::Exist, + Refusal::Missing => Status::NoEnt, + Refusal::NotDirectory => Status::NotDir, + Refusal::IsDirectory => Status::IsDir, + Refusal::NotEmpty => Status::NotEmpty, + Refusal::NotPermitted => Status::Perm, + Refusal::Unsupported => Status::NotSupp, + Refusal::Invalid | Refusal::AncestryRequired | Refusal::AncestryMismatch => Status::Inval, + Refusal::TooManyLinks => Status::MLink, + Refusal::TooLarge => Status::FBig, + } +} + +/// `ftype3` of a Workspace inode kind. +pub const fn file_type(kind: InodeKind) -> u32 { + match kind { + InodeKind::RegularFile => 1, + InodeKind::Directory => 2, + InodeKind::Symlink => 5, + } +} + +/// The identity every inode is shown as, and the export's fixed facts. +#[derive(Clone, Copy, Debug)] +pub struct Identity { + /// Serial of the mount's root directory. + pub root: u64, + pub uid: u32, + pub gid: u32, + /// `fsid` of every attribute; one export is one filesystem. + pub fsid: u64, +} +impl Identity { + /// POSIX link count, as the FUSE adapter projects it: a directory's is + /// derived (0 once removed, else two plus its child directories); any + /// other kind's is its namespace reference count. + pub fn nlink(self, value: &ViewStat) -> u32 { + if value.kind == InodeKind::Directory { + if value.namespace_refs == 0 && value.serial != self.root { + 0 + } else { + u32::try_from(value.subdirs.saturating_add(2)).unwrap_or(u32::MAX) + } + } else { + u32::try_from(value.namespace_refs).unwrap_or(u32::MAX) + } + } + /// Encodes one `fattr3`. The Workspace keeps one timestamp; atime, mtime + /// and ctime all show it. `fileid` is the serial itself. + pub fn fattr3(self, out: &mut Writer, value: &ViewStat) { + let (seconds, nanoseconds) = time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ); + let used = value.logical_len.div_ceil(512).saturating_mul(512); + out.u32(file_type(value.kind)); + out.u32(value.metadata.mode & 0o7777); + out.u32(self.nlink(value)); + out.u32(self.uid).u32(self.gid); + out.u64(value.logical_len).u64(used); + out.u32(0).u32(0); // rdev + out.u64(self.fsid).u64(value.serial); + for _ in 0..3 { + out.u32(seconds).u32(nanoseconds); + } + } + /// `post_op_attr` carrying attributes. + pub fn post_op(self, out: &mut Writer, value: &ViewStat) { + out.bool(true); + self.fattr3(out, value); + } +} +/// The `ctime` a client sees for `value` (the one stored timestamp), as the +/// SETATTR guard compares it. +pub fn ctime(value: &ViewStat) -> (u32, u32) { + time( + value.metadata.mtime_seconds, + value.metadata.mtime_nanoseconds, + ) +} +/// `wcc_attr` (size, mtime, ctime) of a `pre_op_attr` carrying attributes. +pub fn pre_op(out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => { + let (seconds, nanoseconds) = ctime(value); + out.bool(true).u64(value.logical_len); + out.u32(seconds).u32(nanoseconds); + out.u32(seconds).u32(nanoseconds); + } + None => { + out.bool(false); + } + } +} +/// `post_op_attr`, present or absent. +pub fn post_op_maybe(identity: Identity, out: &mut Writer, value: Option<&ViewStat>) { + match value { + Some(value) => identity.post_op(out, value), + None => { + out.bool(false); + } + } +} +/// `nfstime3` holds unsigned seconds; times before the epoch show as the epoch. +fn time(seconds: i64, nanoseconds: u32) -> (u32, u32) { + match u32::try_from(seconds) { + Ok(seconds) => (seconds, nanoseconds.min(999_999_999)), + Err(_) if seconds < 0 => (0, 0), + Err(_) => (u32::MAX, 999_999_999), + } +} + +/// ACCESS3 bits. +pub const ACCESS_READ: u32 = 0x01; +pub const ACCESS_LOOKUP: u32 = 0x02; +pub const ACCESS_MODIFY: u32 = 0x04; +pub const ACCESS_EXTEND: u32 = 0x08; +pub const ACCESS_DELETE: u32 = 0x10; +pub const ACCESS_EXECUTE: u32 = 0x20; + +/// The ACCESS answer for a caller. Every inode belongs to the one command +/// identity; a caller with that uid gets the owner bits, a caller in that +/// group the group bits, anyone else the other bits. uid 0 gets no +/// superuser override: LayerFS has no root semantics, and root inside a +/// sandbox must not bypass the mode. A read-only export grants no change. +pub fn access( + identity: Identity, + value: &ViewStat, + caller: Option<(u32, u32, &[u32])>, + requested: u32, + read_only: bool, +) -> u32 { + let mode = value.metadata.mode; + let bits = match caller { + Some((uid, _, _)) if uid == identity.uid || uid == 0 => (mode >> 6) & 7, + Some((_, gid, groups)) if gid == identity.gid || groups.contains(&identity.gid) => { + (mode >> 3) & 7 + } + _ => mode & 7, + }; + let (read, write, execute) = (bits & 4 != 0, bits & 2 != 0, bits & 1 != 0); + let directory = value.kind == InodeKind::Directory; + let mut granted = 0; + if read { + granted |= ACCESS_READ; + } + if write && !read_only { + granted |= ACCESS_MODIFY | ACCESS_EXTEND; + if directory { + granted |= ACCESS_DELETE; + } + } + if execute { + granted |= if directory { + ACCESS_LOOKUP + } else { + ACCESS_EXECUTE + }; + } + granted & requested +} diff --git a/core/crates/layerfs-nfs/src/block.rs b/core/crates/layerfs-nfs/src/block.rs new file mode 100644 index 0000000000..7449bf9d34 --- /dev/null +++ b/core/crates/layerfs-nfs/src/block.rs @@ -0,0 +1,32 @@ +//! Runs one port future to completion on the connection thread. +//! +//! The service ports return boxed futures; the daemon's implementations run +//! their owner job on the calling thread and are ready on first poll. A +//! future that is not ready parks this thread until its waker unparks it. +use std::{ + future::Future, + sync::Arc, + task::{Context, Poll, Wake, Waker}, + thread::{self, Thread}, +}; + +struct Unpark(Thread); +impl Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + fn wake_by_ref(self: &Arc) { + self.0.unpark(); + } +} +pub fn wait(future: impl Future) -> T { + let mut future = std::pin::pin!(future); + let waker = Waker::from(Arc::new(Unpark(thread::current()))); + let mut context = Context::from_waker(&waker); + loop { + if let Poll::Ready(value) = future.as_mut().poll(&mut context) { + return value; + } + thread::park(); + } +} diff --git a/core/crates/layerfs-nfs/src/change.rs b/core/crates/layerfs-nfs/src/change.rs new file mode 100644 index 0000000000..7122181f94 --- /dev/null +++ b/core/crates/layerfs-nfs/src/change.rs @@ -0,0 +1,531 @@ +//! The changing procedures over the kernel mutation ports: one decided plan +//! per engine visit, and every publication ticket released only after the +//! call's single reply attempt. +//! +//! Each procedure runs [`KernelMutation::perform`], the same orchestration +//! the FUSE connection uses: the serial of a creating operation is reserved +//! there (an exact allocator contention is the declined `Contended`, served +//! as `NFS3ERR_JUKEBOX` so the client retries later with nothing created), +//! need rounds read base facts, and the one publishing visit hands back a +//! [`KernelMutation`] that still owns its ticket. The procedure puts it in +//! the call's [`Tickets`]; the connection releases them with +//! [`Tickets::replied`] after `write_record` returned, whether or not the +//! reply reached the client. A failure that keeps custody is retained, never +//! released or undone on a guess; a fenced step or a failed base demand of +//! this request alone is relinquished, as the FUSE connection does. +use crate::{ + attributes::{refusal, Status}, + block::wait, + kernel::Kernel, + names, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::{ + create, link, remove, rename, + write::{self, AttributeChange, Stamp}, + Declined, KernelMutation, MutationFailure, MutationInput, MutationRequest, +}; +use layerfs_overlay::WRITE_WINDOW; +use layerfs_workspace::{Time, ViewStat}; +use std::{io, sync::atomic::Ordering, time::SystemTime}; + +/// The page a client reads a symlink target into; a longer target could not +/// be read back whole and is refused at creation, as on the FUSE side. +pub const LINK_PAGE: u32 = 4096; +/// Modes when a CREATE or MKDIR names none (clients normally send one). +const FILE_MODE: u32 = 0o644; +const DIRECTORY_MODE: u32 = 0o755; +/// EXCLUSIVE CREATE carries no attributes; the client sets them right after. +const EXCLUSIVE_MODE: u32 = 0o600; + +/// Mutations decided while serving one call, each owed exactly one release +/// of its publication ticket after that call's reply attempt. A declined or +/// unchanged mutation holds no ticket; its release issues no job. +#[derive(Default)] +#[must_use = "every decided mutation is owed its post-reply release"] +pub struct Tickets(Vec); +impl Tickets { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + /// Called only after the reply's single send attempt returned. A release + /// that fails is retained with its remaining custody. + pub fn replied(self, kernel: &Kernel) { + for mutation in self.0 { + if let Err(failure) = wait(mutation.replied()) { + kernel.retain(failure); + } + } + } +} + +/// The settable attributes of `sattr3`. Access time is accepted and not +/// stored (the Workspace keeps one timestamp; the mount is `noatime`). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Sattr { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + pub mtime: Option, +} +impl Sattr { + fn change(self) -> AttributeChange { + AttributeChange { + mode: self.mode, + owner: self.uid, + group: self.gid, + size: self.size, + mtime: self.mtime, + } + } + fn is_empty(self) -> bool { + self.change() == AttributeChange::default() + } +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Sattr), + Guarded(Sattr), + Exclusive([u8; 8]), +} +/// What a published result must carry for its reply to be composed. +#[derive(Clone, Copy)] +enum Expect { + Nothing, + /// The published inode, with this serial when given. + Stat(Option), +} +/// A declined mutation as the status the client sees. +pub fn declined(reason: Declined) -> Status { + match reason { + Declined::Refused(value) => refusal(value), + Declined::NoSpace => Status::NoSpc, + Declined::Contended => Status::Jukebox, + Declined::TargetTooLong => Status::NameTooLong, + } +} + +impl Kernel { + /// One kernel mutation. Its decided [`KernelMutation`] goes to `tickets` + /// whatever the answer; a failure is relinquished or retained. + fn mutate( + &self, + protected: u64, + handle: Option, + expect: Expect, + tickets: &mut Tickets, + build: impl FnOnce(Time) -> Result, + ) -> Result, Status> { + let now = write::time(SystemTime::now()).ok_or(Status::Io)?; + let input = build(now).map_err(declined)?; + let (services, request) = self.request()?; + let request = MutationRequest { + mount: self.mount(), + request, + protected, + handle, + input, + open: None, + now, + }; + let mutation = match wait(KernelMutation::perform(services, request)) { + Ok(mutation) => mutation, + Err(failure) => { + let status = if crate::kernel::failure_is_stale(&failure.reason) { + Status::Stale + } else { + Status::Io + }; + self.failed(failure); + return Err(status); + } + }; + let stat = match mutation.value() { + Err(reason) => { + tickets.0.push(mutation); + return Err(declined(reason)); + } + Ok(published) => published.stat.clone(), + }; + if let Expect::Stat(serial) = expect { + let composable = stat + .as_ref() + .is_some_and(|stat| serial.is_none_or(|serial| serial == stat.serial)); + if !composable { + // The reply cannot be composed from the published result: + // an error for the client and a retained mutation. + let reason = io::Error::other("NFS reply composition").into(); + self.retain(mutation.retain(reason)); + return Err(Status::Io); + } + } + tickets.0.push(mutation); + Ok(stat) + } + fn failed(&self, failure: MutationFailure) { + if failure.fenced() || failure.base_demand().is_some() { + if let Err(failure) = wait(failure.relinquish()) { + self.retain(failure); + } + } else { + self.retain(failure); + } + } + /// A name a changing procedure may bind (`creating`) or remove. + fn checked(&self, name: &[u8], creating: bool) -> Result { + match name { + b"" => return Err(Status::Inval), + b"." | b".." => { + return Err(if creating { + Status::Exist + } else { + Status::Inval + }) + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ if creating && self.refuses_apple_double() && names::apple_double(name) => { + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + _ => {} + } + PathName::from_bytes(name).map_err(|_| Status::Inval) + } + /// The entry a name binds now, or none. + fn existing(&self, parent: u64, name: &[u8]) -> Result, Status> { + match self.lookup(parent, name) { + Ok(stat) => Ok(Some(stat)), + Err(Status::NoEnt) => Ok(None), + Err(status) => Err(status), + } + } + /// CREATE of a regular file. + pub fn create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let key = (parent, name.to_vec()); + if let How::Exclusive(verifier) = how { + // A retransmitted EXCLUSIVE CREATE finds its own file. + let known = self.table().verifiers.get(&key).copied(); + if let Some((stored, serial, _)) = known { + if stored == verifier { + if let Some(stat) = self.existing(parent, name)? { + if stat.serial == serial { + return Ok(stat); + } + } + } + } + } + let (mode, rest) = match how { + How::Unchecked(sattr) | How::Guarded(sattr) => ( + sattr.mode.unwrap_or(FILE_MODE), + Sattr { + mode: None, + ..sattr + }, + ), + How::Exclusive(_) => (EXCLUSIVE_MODE, Sattr::default()), + }; + let created = self.mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::create(parent, checked, mode))) + }); + let stat = match (created, how) { + (Ok(stat), _) => stat.ok_or(Status::ServerFault)?, + (Err(Status::Exist), How::Unchecked(sattr)) => { + // UNCHECKED over an existing regular file: only its size is + // applied, as an open with truncation would. + let stat = self.existing(parent, name)?.ok_or(Status::Exist)?; + if stat.kind != InodeKind::RegularFile { + return Err(Status::Exist); + } + return match sattr.size { + Some(size) if size != stat.logical_len => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.setattr(stat.serial, sized, None, tickets) + } + _ => Ok(stat), + }; + } + (Err(status), _) => return Err(status), + }; + self.entry_taken(&stat, parent, name); + if let How::Exclusive(verifier) = how { + let mut table = self.table(); + let tick = table.tick(); + if table.verifiers.len() >= crate::kernel::VERIFIERS_HELD { + let oldest = table + .verifiers + .iter() + .min_by_key(|(_, entry)| entry.2) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + table.verifiers.remove(&oldest); + } + } + table.verifiers.insert(key, (verifier, stat.serial, tick)); + } + self.settle(stat, rest, tickets) + } + /// Applies what a creating call's attributes ask beyond the mode. + fn settle( + &self, + stat: ViewStat, + rest: Sattr, + tickets: &mut Tickets, + ) -> Result { + let rest = Sattr { + size: rest.size.filter(|size| *size != stat.logical_len), + ..rest + }; + if rest.is_empty() { + return Ok(stat); + } + self.setattr(stat.serial, rest, None, tickets) + } + /// MKDIR. + pub fn mkdir( + &self, + parent: u64, + name: &[u8], + sattr: Sattr, + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let mode = sattr.mode.unwrap_or(DIRECTORY_MODE); + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + Ok(MutationInput::Named(create::mkdir(parent, checked, mode))) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + let rest = Sattr { + mode: None, + size: None, + ..sattr + }; + self.settle(stat, rest, tickets) + } + /// SYMLINK. A symlink's mode is fixed; its attributes are not applied. + pub fn symlink( + &self, + parent: u64, + name: &[u8], + target: &[u8], + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(None), tickets, |_| { + create::symlink(parent, checked, target, LINK_PAGE).map(MutationInput::Named) + })? + .ok_or(Status::ServerFault)?; + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// The silly-named serial `name` denotes in `parent`, if any. + fn silly_entry(&self, parent: u64, name: &[u8]) -> Option { + self.table().silly.get(&(parent, name.to_vec())).copied() + } + /// Drops one silly name; the inode goes back once no silly name keeps it. + fn drop_silly(&self, parent: u64, name: &[u8]) { + let serial = { + let mut table = self.table(); + let serial = table.silly.remove(&(parent, name.to_vec())); + serial.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = serial { + self.unlinked(serial); + } + } + /// REMOVE of a non-directory name. + pub fn remove(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + if self.silly_entry(parent, name).is_some() { + self.drop_silly(parent, name); + return Ok(()); + } + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(parent, checked))) + })?; + if target.namespace_refs <= 1 && !self.table().pinned(target.serial) { + self.unlinked(target.serial); + } + Ok(()) + } + /// RMDIR. + pub fn rmdir(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + let checked = self.checked(name, false)?; + let target = self.existing(parent, name)?.ok_or(Status::NoEnt)?; + self.mutate(parent, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::rmdir(parent, checked))) + })?; + self.unlinked(target.serial); + Ok(()) + } + /// RENAME, replacing the destination. A rename onto a client silly name + /// is the source's unlink with the inode kept alive by the export. + pub fn rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + tickets: &mut Tickets, + ) -> Result<(), Status> { + if let Some(serial) = self.silly_entry(from, name) { + // Moving a silly name to another silly name keeps it ours; + // giving it a real name again would need a link of an unlinked + // inode, which the Workspace refuses. + if !names::silly(new_name) { + return Err(Status::Access); + } + let mut table = self.table(); + table.silly.remove(&(from, name.to_vec())); + table.silly.insert((to, new_name.to_vec()), serial); + return Ok(()); + } + let source_name = self.checked(name, false)?; + let destination_name = self.checked(new_name, true)?; + let source = self.existing(from, name)?.ok_or(Status::NoEnt)?; + if names::silly(new_name) && source.kind == InodeKind::RegularFile { + // Keep a writable descriptor before the last name may go. + self.open_file(source.serial, true)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + Ok(MutationInput::Named(remove::unlink(from, source_name))) + })?; + self.silly_renamed.fetch_add(1, Ordering::Relaxed); + let replaced = { + let mut table = self.table(); + let replaced = table.silly.insert((to, new_name.to_vec()), source.serial); + replaced.filter(|serial| !table.pinned(*serial)) + }; + if let Some(serial) = replaced { + self.unlinked(serial); + } + return Ok(()); + } + let destination = self.existing(to, new_name)?; + self.mutate(from, None, Expect::Nothing, tickets, |_| { + rename::rename(from, source_name, to, destination_name, 0).map(MutationInput::Named) + })?; + if source.kind == InodeKind::Directory { + self.saw_directory(source.serial, to); + } + self.renamed(source.serial, to, new_name); + if let Some(replaced) = destination.filter(|stat| stat.serial != source.serial) { + let last = replaced.kind == InodeKind::Directory || replaced.namespace_refs <= 1; + if last && !self.table().pinned(replaced.serial) { + self.unlinked(replaced.serial); + } + } + Ok(()) + } + /// LINK: one more name for `serial`. + pub fn link( + &self, + serial: u64, + parent: u64, + name: &[u8], + tickets: &mut Tickets, + ) -> Result { + let checked = self.checked(name, true)?; + let stat = self + .mutate(parent, None, Expect::Stat(Some(serial)), tickets, |_| { + Ok(MutationInput::Named(link::link(serial, parent, checked))) + })? + .ok_or(Status::ServerFault)?; + // The engine took one more lookup reference; the export holds one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// SETATTR. `guard` is the client's expected ctime (seconds, + /// nanoseconds), compared with a fresh GETATTR first: the check is not + /// atomic with the change. + pub fn setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + tickets: &mut Tickets, + ) -> Result { + if let Some(expected) = guard { + let current = self.getattr(serial)?; + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if sattr.is_empty() || self.refreshed(serial, &sattr) { + return self.getattr(serial); + } + let identity = (self.identity().uid, self.identity().gid); + let change = sattr.change(); + // A silly-named inode has no name left: a size change goes through + // its descriptor, as an ftruncate would. + let pinned = self.table().pinned(serial); + let handle = if pinned && change.size.is_some() { + Some(self.open_file(serial, true)?.handle) + } else { + None + }; + let stat = self + .mutate(serial, handle, Expect::Stat(Some(serial)), tickets, |now| { + write::set_attributes(serial, handle.is_some(), identity, change, now) + })? + .ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok(stat) + } + /// WRITE through the implicit writable descriptor, in engine windows of + /// at most `WRITE_WINDOW` bytes, each its own publication. A failure after + /// the first window is a short write of the windows already published. + pub fn write( + &self, + serial: u64, + offset: u64, + data: &[u8], + tickets: &mut Tickets, + ) -> Result<(u32, ViewStat), Status> { + let open = self.open_file(serial, true)?; + if data.is_empty() { + return Ok((0, self.getattr(serial)?)); + } + let mut written = 0usize; + let mut last = None; + for window in data.chunks(WRITE_WINDOW) { + let at = offset.checked_add(written as u64).ok_or(Status::FBig)?; + let published = self.mutate( + serial, + Some(open.handle), + Expect::Stat(Some(serial)), + tickets, + |_| write::write(at, window, false), + ); + match published { + Ok(stat) => { + written += window.len(); + last = stat; + } + Err(status) if written == 0 => return Err(status), + Err(_) => break, + } + } + let stat = last.ok_or(Status::ServerFault)?; + self.observed_size(&stat); + Ok((u32::try_from(written).unwrap_or(u32::MAX), stat)) + } +} diff --git a/core/crates/layerfs-nfs/src/changing.rs b/core/crates/layerfs-nfs/src/changing.rs new file mode 100644 index 0000000000..408c914639 --- /dev/null +++ b/core/crates/layerfs-nfs/src/changing.rs @@ -0,0 +1,268 @@ +//! Arguments and results of the changing NFSv3 procedures (RFC 1813): +//! SETATTR, WRITE, CREATE, MKDIR, SYMLINK, MKNOD, REMOVE, RMDIR, RENAME, +//! LINK and COMMIT. +//! +//! Every directory a procedure changes is answered with `wcc_data`: its +//! attributes read just before the change and just after. The two reads are +//! separate visits, not atomic with the change; the connection serves one +//! call at a time, so they bracket it exactly unless another connection or +//! a Commit installs a change in between, in which case the client sees an +//! unexplained difference and drops its cache, which is safe. +//! +//! WRITE answers `FILE_SYNC`: every window was published to the Workspace +//! before the reply, as every FUSE write is. That is publication, not +//! durability; the write verifier changes with every export, so a client +//! that ever sent UNSTABLE data resends it to a restarted export. +use crate::{ + attributes::{post_op_maybe, pre_op, Status}, + change::{How, Sattr, Tickets}, + kernel::Kernel, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::{Time, ViewStat}; + +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Longest symlink target a SYMLINK may carry. +const PATH_LIMIT: usize = 64 * 1024; +/// Longest WRITE payload accepted in one call (two engine windows of slack +/// over what FSINFO declares). +const WRITE_LIMIT: usize = 1 << 20; +/// `stable_how` FILE_SYNC. +const FILE_SYNC: u32 = 2; + +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +fn time_how(arguments: &mut Reader<'_>) -> Result, Garbage> { + Ok(match arguments.u32()? { + 0 => None, + 1 => Some(Stamp::Processing), + 2 => { + let seconds = arguments.u32()?; + let nanoseconds = arguments.u32()?; + Some(Stamp::At(Time { + seconds: i64::from(seconds), + nanoseconds: nanoseconds.min(999_999_999), + })) + } + _ => return Err(Garbage), + }) +} +/// `sattr3`. The access time is decoded and dropped. +pub fn sattr(arguments: &mut Reader<'_>) -> Result { + let mode = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let uid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let gid = arguments.bool()?.then(|| arguments.u32()).transpose()?; + let size = arguments.bool()?.then(|| arguments.u64()).transpose()?; + let _atime = time_how(arguments)?; + let mtime = time_how(arguments)?; + Ok(Sattr { + mode, + uid, + gid, + size, + mtime, + }) +} +/// `wcc_data` of one object around a change. +fn wcc(kernel: &Kernel, out: &mut Writer, before: Option<&ViewStat>, after: Option<&ViewStat>) { + pre_op(out, before); + post_op_maybe(kernel.identity(), out, after); +} +/// Attributes of `serial` when the handle was accepted. +fn attributes(kernel: &Kernel, serial: &Result) -> Option { + serial + .as_ref() + .ok() + .and_then(|serial| kernel.getattr(*serial).ok()) +} +/// The results of CREATE, MKDIR and SYMLINK. +fn created( + kernel: &Kernel, + out: &mut Writer, + result: Result, + directory: &Result, + before: Option<&ViewStat>, +) { + let after = attributes(kernel, directory); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.bool(true).opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + wcc(kernel, out, before, after.as_ref()); +} +/// Decodes the arguments of changing `procedure`, runs it and appends its +/// results. Mutations it decides go to `tickets`, released by the caller +/// after the reply attempt. `Err` is a garbage-arguments refusal. +pub fn call( + kernel: &Kernel, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Garbage> { + match procedure { + 2 => { + let serial = handle(kernel, arguments)?; + let change = sattr(arguments)?; + let guard = if arguments.bool()? { + Some((arguments.u32()?, arguments.u32()?)) + } else { + None + }; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.setattr(serial, change, guard, tickets)); + match result { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 7 => { + let serial = handle(kernel, arguments)?; + let offset = arguments.u64()?; + let count = arguments.u32()? as usize; + let _stable = arguments.u32()?; + let data = arguments.opaque(WRITE_LIMIT)?; + let data = &data[..count.min(data.len())]; + let before = attributes(kernel, &serial); + let result = serial.and_then(|serial| kernel.write(serial, offset, data, tickets)); + match result { + Ok((written, stat)) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, before.as_ref(), Some(&stat)); + out.u32(written).u32(FILE_SYNC).fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + let after = attributes(kernel, &serial); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + } + } + 8..=10 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match procedure { + 8 => { + let how = match arguments.u32()? { + 0 => How::Unchecked(sattr(arguments)?), + 1 => How::Guarded(sattr(arguments)?), + 2 => { + let mut verifier = [0; 8]; + verifier.copy_from_slice(arguments.fixed(8)?); + How::Exclusive(verifier) + } + _ => return Err(Garbage), + }; + directory.and_then(|parent| kernel.create(parent, &name, how, tickets)) + } + 9 => { + let change = sattr(arguments)?; + directory.and_then(|parent| kernel.mkdir(parent, &name, change, tickets)) + } + _ => { + let _attributes = sattr(arguments)?; + let target = arguments.opaque(PATH_LIMIT)?.to_vec(); + directory.and_then(|parent| kernel.symlink(parent, &name, &target, tickets)) + } + }; + created(kernel, out, result, &directory, before.as_ref()); + } + 11 => { + // Only regular files, directories and symlinks exist in the + // canonical format: no device, socket or FIFO node. + let directory = handle(kernel, arguments)?; + let _name = arguments.opaque(NAME_LIMIT)?; + let before = attributes(kernel, &directory); + let status = directory.err().unwrap_or(Status::NotSupp); + created(kernel, out, Err(status), &directory, before.as_ref()); + } + 12 | 13 => { + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = directory.and_then(|parent| { + if procedure == 12 { + kernel.remove(parent, &name, tickets) + } else { + kernel.rmdir(parent, &name, tickets) + } + }); + let after = attributes(kernel, &directory); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 14 => { + let from = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let to = handle(kernel, arguments)?; + let new_name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before_from = attributes(kernel, &from); + let before_to = attributes(kernel, &to); + let result = match (&from, &to) { + (Ok(from), Ok(to)) => kernel.rename(*from, &name, *to, &new_name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after_from = attributes(kernel, &from); + let after_to = attributes(kernel, &to); + out.u32(result.err().unwrap_or(Status::Ok).code()); + wcc(kernel, out, before_from.as_ref(), after_from.as_ref()); + wcc(kernel, out, before_to.as_ref(), after_to.as_ref()); + } + 15 => { + let file = handle(kernel, arguments)?; + let directory = handle(kernel, arguments)?; + let name = arguments.opaque(NAME_LIMIT)?.to_vec(); + let before = attributes(kernel, &directory); + let result = match (&file, &directory) { + (Ok(file), Ok(parent)) => kernel.link(*file, *parent, &name, tickets), + (Err(status), _) | (_, Err(status)) => Err(*status), + }; + let after = attributes(kernel, &directory); + let (status, object) = match result { + Ok(stat) => (Status::Ok, Some(stat)), + Err(status) => (status, attributes(kernel, &file)), + }; + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, object.as_ref()); + wcc(kernel, out, before.as_ref(), after.as_ref()); + } + 21 => { + // Every WRITE was answered FILE_SYNC, so nothing is uncommitted. + let serial = handle(kernel, arguments)?; + let _offset = arguments.u64()?; + let _count = arguments.u32()?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + wcc(kernel, out, None, Some(&stat)); + out.fixed(&verifier); + } + Err(status) => { + out.u32(status.code()); + wcc(kernel, out, None, None); + } + } + } + _ => return Err(Garbage), + } + Ok(()) +} diff --git a/core/crates/layerfs-nfs/src/directory.rs b/core/crates/layerfs-nfs/src/directory.rs new file mode 100644 index 0000000000..92a984bbb0 --- /dev/null +++ b/core/crates/layerfs-nfs/src/directory.rs @@ -0,0 +1,138 @@ +//! READDIR and READDIRPLUS over the engine's published directory cookies. +//! +//! The NFS cookie is the engine's kernel offset (1 for `.`, 2 for `..`, then +//! the offsets the engine publishes) on one implicit directory descriptor, +//! named to the client by `cookieverf`. Cookie 0 opens a fresh descriptor and +//! a fresh verifier. A verifier whose descriptor was closed, or an offset a +//! rewind retired, is `NFS3ERR_BAD_COOKIE`; the client then lists again from 0. +//! +//! One reply carries at most one engine window: exactly the names that fit +//! the client's byte budget are published, and the reply is not at EOF. The +//! next call resumes after the last accepted cookie; a call that finds no +//! names left replies EOF. READDIRPLUS entries carry no attributes and no +//! handle (both optional in RFC 1813), so the client looks each name up. +use crate::{ + attributes::Status, + block::wait, + kernel::{failure, Kernel}, + xdr::{opaque_size, Writer}, +}; +use layerfs_content::object::inode_leaf::InodeKind; +use layerfs_fuse::operations::{DirectoryFailure, DirectoryStep, DirectoryStream}; + +/// One enumeration request. +pub struct Listing { + pub serial: u64, + pub cookie: u64, + pub verifier: u64, + /// The client's byte budget for the whole reply (`count` / `maxcount`). + pub budget: usize, + /// READDIRPLUS: each entry also carries two absent optional fields. + pub plus: bool, +} +/// Bytes of the reply around the entries: status, absent post_op_attr, +/// cookieverf, the list terminator and the EOF flag, plus the RPC reply header. +const FIXED: usize = 4 + 4 + 8 + 4 + 4 + 24; + +struct Entries { + out: Writer, + used: usize, + budget: usize, + plus: bool, + count: usize, +} +impl Entries { + fn size(&self, name: &[u8]) -> usize { + 4 + 8 + opaque_size(name.len()) + 8 + if self.plus { 8 } else { 0 } + } + /// Appends one entry when it fits the budget. + fn push(&mut self, fileid: u64, name: &[u8], cookie: u64) -> bool { + let size = self.size(name); + if self.used + size > self.budget { + return false; + } + self.used += size; + self.count += 1; + self.out.bool(true).u64(fileid).opaque(name).u64(cookie); + if self.plus { + self.out.bool(false).bool(false); + } + true + } +} + +/// The encoded `READDIR3resok`/`READDIRPLUS3resok` body after the status, +/// or the failure status. +pub fn list(kernel: &Kernel, request: Listing) -> Result, Status> { + let handle = if request.cookie == 0 { + kernel.open_directory(request.serial)? + } else { + kernel + .directory(request.serial, request.verifier) + .ok_or(Status::BadCookie)? + }; + let (services, id) = kernel.request()?; + let stream = wait(DirectoryStream::prepare( + services, + kernel.mount(), + id, + request.serial, + handle, + request.cookie, + )) + .map_err(directory_failure)?; + let mut entries = Entries { + out: Writer::with_capacity(request.budget.min(64 * 1024)), + used: FIXED, + budget: request.budget, + plus: request.plus, + count: 0, + }; + let dots: Vec<(&str, u64, u64)> = stream.dots().collect(); + for (name, serial, cookie) in dots { + if name == ".." && serial != request.serial { + kernel.saw_directory(request.serial, serial); + } + if !entries.push(serial, name.as_bytes(), cookie) { + return finish(entries, handle, false); + } + } + let eof = match wait(stream.next()).map_err(directory_failure)? { + DirectoryStep::End(_) => true, + DirectoryStep::Batch(batch) => { + let mut accepted = 0; + for (entry, cookie) in batch.entries() { + if !entries.push(entry.serial, &entry.name, cookie) { + break; + } + if entry.kind == InodeKind::Directory { + kernel.saw_directory(entry.serial, request.serial); + } + accepted += 1; + } + // Publishing ends this reply's window even when names remain. + drop(wait(batch.accept(accepted)).map_err(directory_failure)?); + false + } + }; + finish(entries, handle, eof) +} +fn finish(entries: Entries, verifier: u64, eof: bool) -> Result, Status> { + if entries.count == 0 && !eof { + return Err(Status::TooSmall); + } + let mut out = Writer::with_capacity(entries.out.len() + 24); + out.bool(false); // dir_attributes + out.u64(verifier); + out.fixed(entries.out.as_bytes()); + out.bool(false); // no further entry + out.bool(eof); + Ok(out.into_bytes()) +} +fn directory_failure(failure: DirectoryFailure) -> Status { + if failure.rewound() { + Status::BadCookie + } else { + self::failure(failure.reason) + } +} diff --git a/core/crates/layerfs-nfs/src/handle.rs b/core/crates/layerfs-nfs/src/handle.rs new file mode 100644 index 0000000000..1599a69205 --- /dev/null +++ b/core/crates/layerfs-nfs/src/handle.rs @@ -0,0 +1,69 @@ +//! Stable NFS file handles for one kernel mount. +//! +//! A handle is `[version][mount owner][serial][tag]`: 25 bytes, well inside +//! the 64-byte NFSv3 limit. The same serial always encodes to the same bytes +//! for the life of one mount, so a client may compare handles. The mount owner +//! is fresh for every engine mount; a handle from an earlier mount decodes as +//! stale. The tag is a keyed hash under a key drawn once per export, so a +//! local process cannot forge a handle for a serial it was never given. It is +//! not a cryptographic MAC (std SipHash, no third-party dependency); it raises +//! the cost of guessing, it does not authenticate the caller. +use std::{collections::hash_map::RandomState, hash::BuildHasher}; + +const VERSION: u8 = 1; +/// Encoded length of every handle this export issues. +pub const HANDLE_BYTES: usize = 25; + +/// Why a client's handle was not accepted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum HandleError { + /// Not a handle this export ever issued (length, version or tag). + Bad, + /// Issued for an earlier mount of this export. + Stale, +} + +/// Encodes and checks the handles of one kernel mount. +pub struct Handles { + owner: u64, + key: RandomState, +} +impl Handles { + pub fn new(owner: u64) -> Self { + Self { + owner, + key: RandomState::new(), + } + } + fn tag(&self, owner: u64, serial: u64) -> u64 { + self.key.hash_one((VERSION, owner, serial)) + } + pub fn encode(&self, serial: u64) -> [u8; HANDLE_BYTES] { + let mut handle = [0; HANDLE_BYTES]; + handle[0] = VERSION; + handle[1..9].copy_from_slice(&self.owner.to_be_bytes()); + handle[9..17].copy_from_slice(&serial.to_be_bytes()); + handle[17..25].copy_from_slice(&self.tag(self.owner, serial).to_be_bytes()); + handle + } + /// The serial a handle names. A handle of another mount is stale even + /// when its tag cannot be checked, since its key is gone. + pub fn decode(&self, handle: &[u8]) -> Result { + if handle.len() != HANDLE_BYTES || handle[0] != VERSION { + return Err(HandleError::Bad); + } + let word = |range: std::ops::Range| { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&handle[range]); + u64::from_be_bytes(bytes) + }; + let (owner, serial, tag) = (word(1..9), word(9..17), word(17..25)); + if owner != self.owner { + return Err(HandleError::Stale); + } + if serial == 0 || tag != self.tag(owner, serial) { + return Err(HandleError::Bad); + } + Ok(serial) + } +} diff --git a/core/crates/layerfs-nfs/src/kernel.rs b/core/crates/layerfs-nfs/src/kernel.rs new file mode 100644 index 0000000000..5df1044bb6 --- /dev/null +++ b/core/crates/layerfs-nfs/src/kernel.rs @@ -0,0 +1,811 @@ +//! The synthetic kernel: the lookup references, open descriptors and parent +//! index a FUSE kernel would hold, kept by the export for a stateless client. +//! +//! The engine fences every request on a kernel reference: the root's implicit +//! one, a lookup reference, or an open descriptor. An NFS client presents a +//! handle at any time and never sends FORGET, OPEN or RELEASE, so: +//! +//! - every serial the export hands out in a handle holds exactly one lookup +//! reference, taken by the LOOKUP visit or the creating mutation that +//! decided it; a later visit that adds one more for the same serial is +//! answered by forgetting the extra one at once; +//! - READ and WRITE use one implicit descriptor per file (read-only until the +//! first WRITE, which reopens it writable), and READDIR one implicit +//! directory descriptor per enumeration (`cookieverf`); both are bounded +//! and the least recently used is closed when the bound is reached; +//! - `..` is answered from the parent recorded when a directory was looked +//! up, created, moved or listed, since the engine resolves names only +//! downward; +//! - a serial whose last name a REMOVE, RMDIR or replacing RENAME of this +//! export took away gives its reference and descriptor back at once, so the +//! engine can reclaim it; its handle is stale afterwards, as on any server; +//! - a client's silly rename (`.nfs…`) is not a namespace change: the source +//! name is unlinked and the inode is kept alive by the export alone, under +//! a name only this export knows (see [`crate::names`]). +//! +//! - the references are bounded (reference collection, "plan A" of the A2 +//! design, with no engine interface added): every handed-out serial keeps +//! the name it was last reached by. When more than the configured number +//! of serials hold a reference, the least recently used quarter that holds +//! no descriptor and no silly name is forgotten, its name memory kept. A +//! later handle of a forgotten serial is resolved again by the ordinary +//! LOOKUP of that remembered name (its parent first, recursively): the +//! same serial continues; another serial or none is a stale handle, as for +//! a file renamed or removed elsewhere. +//! +//! Everything still held is given back by [`Kernel::release`], which runs +//! before the engine mount is revoked. +use crate::{ + attributes::{refusal, Identity, Status}, + block::wait, + handle::{HandleError, Handles}, +}; +use layerfs_content::{filesystem::PathName, object::inode_leaf::InodeKind}; +use layerfs_fuse::{ + operations::{KernelData, KernelRead, MutationFailure, ReadDataInput, ReadFailure}, + ports::{Fence, MountServices, RequestServices, ServiceError}, +}; +use layerfs_overlay::{KernelMount, OverlayError, READ_WINDOW}; +use layerfs_workspace::{KernelReadOperation, ViewStat}; +use std::{ + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, MutexGuard, + }, +}; + +/// Implicit file descriptors kept open at once. +pub const FILES_HELD: usize = 256; +/// Implicit directory descriptors (enumerations) kept open at once. +pub const DIRECTORIES_HELD: usize = 256; +/// EXCLUSIVE CREATE verifiers remembered for retransmissions. +pub const VERIFIERS_HELD: usize = 1024; +/// Lookup references an export keeps by default before collecting. +pub const LOOKUPS_HELD: usize = 65_536; +/// Remembered names beyond which a handle of a forgotten serial is stale. +const NAMES_HELD: usize = 4 * 1024 * 1024; +/// Parents resolved again for one handle at most (directory depth). +const DEPTH: usize = 4096; +/// Serials used within this many most recent uses are never collected: a +/// call another connection is serving may be about to use them. +pub const RECENT_KEPT: u64 = 64; + +/// A name in one directory: (directory serial, name bytes). +pub(crate) type Named = (u64, Vec); +/// An EXCLUSIVE CREATE's verifier, the serial it created and its last use. +pub(crate) type Verified = ([u8; 8], u64, u64); + +/// One implicit file descriptor. +#[derive(Clone, Copy)] +pub(crate) struct Held { + pub handle: u64, + /// The size last published or observed through this export. + pub size: u64, + pub tick: u64, + pub writable: bool, +} + +#[derive(Default)] +pub(crate) struct Table { + /// Serials holding the export's one lookup reference (the root's is implicit). + pub lookups: HashSet, + /// Directory serial -> the parent it was last seen under. + pub parents: HashMap, + /// File serial -> its implicit descriptor. + pub files: HashMap, + /// Directory descriptor -> (directory serial, last use). + pub directories: HashMap, + /// Silly-renamed inodes: (directory, client name) -> serial. Their + /// descriptors are never evicted; their source name is gone. + pub silly: HashMap, + /// EXCLUSIVE CREATE: (directory, name) -> (verifier, serial, last use). + pub verifiers: HashMap, + /// Serial -> the name it was last reached by, kept after its reference + /// is collected so a later handle can be resolved again. + pub names: HashMap, + /// Serial -> last use, for the collection order. + pub used: HashMap, + /// Serials marked for a cache refresh, and when (see [`crate::refresh`]). + pub refreshing: HashMap, + pub tick: u64, +} +impl Table { + pub fn tick(&mut self) -> u64 { + self.tick += 1; + self.tick + } + /// Serials kept alive by a silly name. + pub fn pinned(&self, serial: u64) -> bool { + self.silly.values().any(|held| *held == serial) + } +} +/// One opened file as READ and WRITE need it. +#[derive(Clone, Copy)] +pub(crate) struct OpenFile { + pub handle: u64, + pub size: u64, +} + +/// One kernel mount served over NFS: its ports and its synthetic kernel state. +pub struct Kernel { + services: Arc, + fence: Fence, + mount: KernelMount, + identity: Identity, + handles: Handles, + table: Mutex, + requests: AtomicU64, + /// Mutations whose failure keeps engine custody (possibly a publication + /// ticket): kept, never released or undone on a guess, as the FUSE + /// connection retains them. + retained: Mutex>, + /// Refuse to create macOS AppleDouble (`._*`) names. + refuse_apple_double: bool, + /// Lookup references kept before the least recently used are collected. + lookups_held: usize, + /// References collected, handles resolved again, and resolutions that + /// found another serial or none (stale), over the export's life. + collected: AtomicU64, + resolved: AtomicU64, + unresolved: AtomicU64, + /// Silly renames served as unlinks, and creations refused by name. + pub(crate) silly_renamed: AtomicU64, + pub(crate) names_refused: AtomicU64, +} +impl Kernel { + pub fn new(services: Arc, mount: KernelMount, uid: u32, gid: u32) -> Self { + let root = mount.root_serial(); + Self { + services, + fence: Fence::default(), + mount, + identity: Identity { + root, + uid, + gid, + fsid: mount.owner_id(), + }, + handles: Handles::new(mount.owner_id()), + table: Mutex::default(), + requests: AtomicU64::new(1), + retained: Mutex::default(), + refuse_apple_double: false, + lookups_held: LOOKUPS_HELD, + collected: AtomicU64::new(0), + resolved: AtomicU64::new(0), + unresolved: AtomicU64::new(0), + silly_renamed: AtomicU64::new(0), + names_refused: AtomicU64::new(0), + } + } + /// Whether creating `._*` names is refused (see [`crate::names`]). + pub fn refusing_apple_double(mut self, refuse: bool) -> Self { + self.refuse_apple_double = refuse; + self + } + pub fn refuses_apple_double(&self) -> bool { + self.refuse_apple_double + } + /// How many lookup references are kept before collection (at least 4). + pub fn holding(mut self, lookups: usize) -> Self { + self.lookups_held = lookups.max(4); + self + } + pub fn identity(&self) -> Identity { + self.identity + } + pub fn root(&self) -> u64 { + self.identity.root + } + pub fn mount(&self) -> KernelMount { + self.mount + } + pub fn encode(&self, serial: u64) -> Vec { + self.handles.encode(serial).to_vec() + } + pub(crate) fn table(&self) -> MutexGuard<'_, Table> { + self.table + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// Keeps a failed mutation's remaining custody. + pub(crate) fn retain(&self, failure: MutationFailure) { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(failure); + } + /// The serial a client handle names, holding its reference: a collected + /// one is resolved again by its remembered name. + pub fn decode(&self, handle: &[u8]) -> Result { + let serial = self.handles.decode(handle).map_err(|error| match error { + HandleError::Bad => Status::BadHandle, + HandleError::Stale => Status::Stale, + })?; + self.held_reference(serial)?; + Ok(serial) + } + /// Ensures the export holds `serial`'s reference, resolving a collected + /// serial again; marks it used. + pub(crate) fn held_reference(&self, serial: u64) -> Result<(), Status> { + if serial == self.root() { + return Ok(()); + } + { + let mut table = self.table(); + if table.lookups.contains(&serial) { + let tick = table.tick(); + table.used.insert(serial, tick); + return Ok(()); + } + } + self.resolve(serial, 0) + } + /// One collected serial resolved by the LOOKUP of its remembered name, + /// after its parent. Another serial, or none, there is a stale handle. + fn resolve(&self, serial: u64, depth: usize) -> Result<(), Status> { + let named = self.table().names.get(&serial).cloned(); + let Some((parent, name)) = named else { + return Err(Status::Stale); + }; + if depth >= DEPTH { + return Err(Status::Stale); + } + let parent_held = parent == self.root() || self.table().lookups.contains(&parent); + if !parent_held { + self.resolve(parent, depth + 1)?; + } + let found = PathName::from_bytes(&name) + .map_err(|_| Status::Stale) + .and_then(|path| { + self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + ) + }); + match found { + Ok((stat, _, _)) => { + self.entry_taken(&stat, parent, &name); + if stat.serial == serial { + self.resolved.fetch_add(1, Ordering::Relaxed); + return Ok(()); + } + } + Err(Status::NoEnt | Status::Stale | Status::NotDir) => {} + Err(status) => return Err(status), + } + let mut table = self.table(); + if table + .names + .get(&serial) + .is_some_and(|(p, n)| *p == parent && *n == name) + { + table.names.remove(&serial); + } + self.unresolved.fetch_add(1, Ordering::Relaxed); + Err(Status::Stale) + } + pub fn request(&self) -> Result<(Arc, u64), Status> { + let services = self.services.request(&self.fence).map_err(failure)?; + Ok((services, self.requests.fetch_add(1, Ordering::Relaxed))) + } + fn observe( + &self, + protected: u64, + handle: Option, + operation: KernelReadOperation, + ) -> Result<(ViewStat, Option, Option), Status> { + let (services, request) = self.request()?; + let found = wait(KernelRead::prepare( + services, self.mount, request, protected, handle, operation, + )) + .map_err(read_failure)?; + let answer = found.value().map(|value| { + ( + value.stat.clone(), + value.file.map(|file| file.owner_id()), + value.directory.map(|directory| directory.owner_id()), + ) + }); + wait(found.dispose()).map_err(read_failure)?; + answer.map_err(refusal) + } + /// GETATTR. An inode that lost its last name while a silly name keeps it + /// is observed through its descriptor; any other inode no name reaches + /// any more is stale. + pub fn getattr(&self, serial: u64) -> Result { + let handle = { + let table = self.table(); + if table.pinned(serial) { + table.files.get(&serial).map(|held| held.handle) + } else { + None + } + }; + let stat = self + .observe(serial, handle, KernelReadOperation::Getattr { serial }) + .map(|(stat, _, _)| stat) + .map_err(|status| { + if status == Status::NoEnt { + Status::Stale + } else { + status + } + })?; + self.observed_size(&stat); + Ok(stat) + } + /// LOOKUP of `name` in directory `parent`, including `.`, `..` and the + /// export's own silly names. + pub fn lookup(&self, parent: u64, name: &[u8]) -> Result { + match name { + b"" => return Err(Status::NoEnt), + b"." => return self.getattr(parent), + b".." => { + let up = if parent == self.root() { + parent + } else { + let known = self.table().parents.get(&parent).copied(); + known.ok_or(Status::NoEnt)? + }; + self.held_reference(up)?; + return self.getattr(up); + } + _ if name.len() > 255 => return Err(Status::NameTooLong), + _ => {} + } + let silly = self.table().silly.get(&(parent, name.to_vec())).copied(); + if let Some(serial) = silly { + return self.getattr(serial); + } + let path = PathName::from_bytes(name).map_err(|_| Status::Inval)?; + let (stat, _, _) = self.observe( + parent, + None, + KernelReadOperation::Lookup { parent, name: path }, + )?; + // The deciding visit took one engine reference; keep exactly one. + self.entry_taken(&stat, parent, name); + Ok(stat) + } + /// A visit or mutation that decided `stat` as the entry `name` of + /// `parent` took one engine lookup reference on it: the export keeps + /// exactly one per serial, remembers the name and records a directory's + /// parent. Beyond the bound, the least recently used are collected. + pub(crate) fn entry_taken(&self, stat: &ViewStat, parent: u64, name: &[u8]) { + let (extra, collect) = { + let mut table = self.table(); + if stat.kind == InodeKind::Directory { + table.parents.insert(stat.serial, parent); + } + let root = stat.serial == self.root(); + if !root { + let tick = table.tick(); + table.used.insert(stat.serial, tick); + if table.names.len() < NAMES_HELD || table.names.contains_key(&stat.serial) { + table.names.insert(stat.serial, (parent, name.to_vec())); + } + } + let extra = root || !table.lookups.insert(stat.serial); + let collect = if table.lookups.len() > self.lookups_held { + collectable(&mut table, self.lookups_held - self.lookups_held / 4) + } else { + Vec::new() + }; + (extra, collect) + }; + if extra { + self.forget(stat.serial, 1); + } + self.collected + .fetch_add(collect.len() as u64, Ordering::Relaxed); + for serial in collect { + self.forget(serial, 1); + } + self.observed_size(stat); + } + /// Records that `serial` is now reached as `name` in `parent` (RENAME). + pub(crate) fn renamed(&self, serial: u64, parent: u64, name: &[u8]) { + let mut table = self.table(); + if table.names.contains_key(&serial) { + table.names.insert(serial, (parent, name.to_vec())); + } + } + pub(crate) fn forget(&self, serial: u64, count: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.forget(self.mount, serial, count))); + } + } + /// Gives back everything the export holds for `serial`, whose last name + /// is gone: its descriptor, its reference and its parent record. Later + /// handles of it are stale. + pub(crate) fn unlinked(&self, serial: u64) { + if serial == self.root() { + return; + } + let (file, directories, held) = { + let mut table = self.table(); + table.parents.remove(&serial); + table.names.remove(&serial); + table.used.remove(&serial); + table.verifiers.retain(|_, entry| entry.1 != serial); + let directories: Vec = table + .directories + .iter() + .filter(|(_, entry)| entry.0 == serial) + .map(|(handle, _)| *handle) + .collect(); + for handle in &directories { + table.directories.remove(handle); + } + ( + table.files.remove(&serial), + directories, + table.lookups.remove(&serial), + ) + }; + if let Some(file) = file { + self.close_file(serial, file.handle); + } + for handle in directories { + self.close_directory(serial, handle); + } + if held { + self.forget(serial, 1); + } + } + /// Records the parent of a directory seen in a listing or moved by RENAME. + pub fn saw_directory(&self, serial: u64, parent: u64) { + self.table().parents.insert(serial, parent); + } + /// The size a published or observed stat reports, for READ's EOF. + pub(crate) fn observed_size(&self, stat: &ViewStat) { + if let Some(held) = self.table().files.get_mut(&stat.serial) { + held.size = stat.logical_len; + } + } + /// READLINK: the raw target bytes. + pub fn readlink(&self, serial: u64) -> Result, Status> { + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + None, + ReadDataInput::Link, + )) + .map_err(read_failure)? + .map_err(refusal)?; + Ok(data.bytes().to_vec()) + } + /// The implicit descriptor of `serial`, opened (or reopened writable) + /// when needed. + pub(crate) fn open_file(&self, serial: u64, writable: bool) -> Result { + { + let mut table = self.table(); + let tick = table.tick(); + if let Some(held) = table.files.get_mut(&serial) { + if held.writable || !writable { + held.tick = tick; + return Ok(OpenFile { + handle: held.handle, + size: held.size, + }); + } + } + } + let operation = KernelReadOperation::Open { serial, writable }; + let (stat, file, _) = self.observe(serial, None, operation)?; + let handle = file.ok_or(Status::ServerFault)?; + let (evicted, closed, chosen) = { + let mut table = self.table(); + let tick = table.tick(); + let current = table.files.get(&serial).copied(); + match current { + // Another connection opened a good enough one meanwhile. + Some(theirs) if theirs.writable || !writable => { + (None, Some((serial, handle)), theirs) + } + _ => { + let evicted = if current.is_none() && table.files.len() >= FILES_HELD { + let pinned: HashSet = table.silly.values().copied().collect(); + oldest( + table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.tick)), + ) + .and_then(|serial| { + table + .files + .remove(&serial) + .map(|held| (serial, held.handle)) + }) + } else { + None + }; + let held = Held { + handle, + size: stat.logical_len, + tick, + writable, + }; + table.files.insert(serial, held); + (evicted, current.map(|old| (serial, old.handle)), held) + } + } + }; + if let Some((old, descriptor)) = evicted { + self.close_file(old, descriptor); + } + if let Some((old, descriptor)) = closed { + self.close_file(old, descriptor); + } + Ok(OpenFile { + handle: chosen.handle, + size: chosen.size, + }) + } + pub(crate) fn close_file(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_file(self.mount, serial, handle))); + } + } + /// READ of at most one engine window. `eof` is set when the reply reaches + /// the size last seen for the file, or comes back short. + pub fn read(&self, serial: u64, offset: u64, count: u32) -> Result<(Vec, bool), Status> { + let length = count.min(READ_WINDOW as u32); + let open = self.open_file(serial, false)?; + if offset >= open.size { + return Ok((Vec::new(), true)); + } + let (services, request) = self.request()?; + let data = wait(KernelData::read( + services, + self.mount, + request, + serial, + Some(open.handle), + ReadDataInput::File { offset, length }, + )) + .map_err(read_failure)? + .map_err(refusal)?; + let bytes = data.bytes().to_vec(); + let end = offset.saturating_add(bytes.len() as u64); + let eof = bytes.len() < length as usize || end >= open.size; + Ok((bytes, eof)) + } + /// A fresh implicit directory descriptor for an enumeration from cookie 0. + pub fn open_directory(&self, serial: u64) -> Result { + let (_, _, directory) = + self.observe(serial, None, KernelReadOperation::Opendir { serial })?; + let handle = directory.ok_or(Status::ServerFault)?; + let evicted = { + let mut table = self.table(); + let tick = table.tick(); + let evicted = if table.directories.len() >= DIRECTORIES_HELD { + oldest( + table + .directories + .iter() + .map(|(handle, entry)| (*handle, entry.1)), + ) + .and_then(|handle| { + table + .directories + .remove(&handle) + .map(|entry| (entry.0, handle)) + }) + } else { + None + }; + table.directories.insert(handle, (serial, tick)); + evicted + }; + if let Some((old, descriptor)) = evicted { + self.close_directory(old, descriptor); + } + Ok(handle) + } + /// The descriptor of a continuing enumeration, if it is still open. + pub fn directory(&self, serial: u64, handle: u64) -> Option { + let mut table = self.table(); + let tick = table.tick(); + let entry = table.directories.get_mut(&handle)?; + if entry.0 != serial { + return None; + } + entry.1 = tick; + Some(handle) + } + fn close_directory(&self, serial: u64, handle: u64) { + if let Ok((services, _)) = self.request() { + drop(wait(services.close_directory(self.mount, serial, handle))); + } + } + /// Gives back everything the synthetic kernel holds: descriptors first, + /// then lookup references. Run once, before the engine mount is revoked. + /// Retained mutation failures keep their custody and are only counted. + pub fn release(&self) -> Released { + let table = std::mem::take(&mut *self.table()); + let released = Released { + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + }; + for (serial, held) in table.files { + self.close_file(serial, held.handle); + } + for (handle, (serial, _)) in table.directories { + self.close_directory(serial, handle); + } + for serial in table.lookups { + self.forget(serial, 1); + } + released + } + fn retained_count(&self) -> usize { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .len() + } + /// Current holdings, for diagnostics. + pub fn held(&self) -> Released { + let table = self.table(); + Released { + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + } + } + /// Closes every implicit file descriptor that no silly name needs, so + /// nothing the export opened is held across a Commit. Returns how many. + pub fn close_files(&self) -> usize { + let closing: Vec<(u64, u64)> = { + let mut table = self.table(); + let pinned: HashSet = table.silly.values().copied().collect(); + let closing: Vec<(u64, u64)> = table + .files + .iter() + .filter(|(serial, _)| !pinned.contains(serial)) + .map(|(serial, held)| (*serial, held.handle)) + .collect(); + for (serial, _) in &closing { + table.files.remove(serial); + } + closing + }; + for (serial, handle) in &closing { + self.close_file(*serial, *handle); + } + closing.len() + } + /// Closes every enumeration descriptor; a continuing READDIR then + /// answers `NFS3ERR_BAD_COOKIE` and the client lists again from the + /// start. Used after a Commit installed another root. Returns how many. + pub fn close_directories(&self) -> usize { + let closing: Vec<(u64, u64)> = std::mem::take(&mut self.table().directories) + .into_iter() + .map(|(handle, (serial, _))| (serial, handle)) + .collect(); + for (serial, handle) in &closing { + self.close_directory(*serial, *handle); + } + closing.len() + } +} +/// Takes the least recently used references out of the table until `keep` +/// remain, skipping recently used serials, serials that hold a descriptor +/// or a silly name, and +/// directories a held serial was reached through: the engine's own parent +/// index of a held inode never loses its link, so collection proceeds from +/// the leaves. Names and parent records stay for a later resolution. +fn collectable(table: &mut Table, keep: usize) -> Vec { + let held_parents = table.lookups.iter().filter_map(|serial| { + table + .names + .get(serial) + .map(|named| named.0) + .or_else(|| table.parents.get(serial).copied()) + }); + let busy: HashSet = table + .silly + .values() + .copied() + .chain(table.files.keys().copied()) + .chain(table.directories.values().map(|entry| entry.0)) + .chain(held_parents) + .collect(); + let recent = table.tick.saturating_sub(RECENT_KEPT); + let mut candidates: Vec<(u64, u64)> = table + .lookups + .iter() + .filter(|serial| !busy.contains(serial)) + .map(|serial| (table.used.get(serial).copied().unwrap_or(0), *serial)) + .filter(|(used, _)| *used < recent) + .collect(); + let excess = table.lookups.len().saturating_sub(keep); + candidates.sort_unstable(); + candidates.truncate(excess); + for (_, serial) in &candidates { + table.lookups.remove(serial); + table.used.remove(serial); + } + candidates.into_iter().map(|(_, serial)| serial).collect() +} +/// What [`Kernel::release`] gave back. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Released { + pub files: usize, + pub directories: usize, + pub lookups: usize, + /// Silly names whose inode was kept alive until release. + pub silly: usize, + /// Failed mutations still holding engine custody. + pub retained: usize, + /// Client silly renames served as unlinks, over the export's life. + pub silly_renamed: u64, + /// Creations refused for their AppleDouble name, over the export's life. + pub names_refused: u64, + /// References collected beyond the bound, over the export's life. + pub collected: u64, + /// Handles of collected serials resolved again by their name. + pub resolved: u64, + /// Handles of collected serials whose name no longer reaches them. + pub unresolved: u64, +} +fn oldest(entries: impl Iterator) -> Option { + entries.min_by_key(|(_, tick)| *tick).map(|(key, _)| key) +} +fn stale(error: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(error); + while let Some(error) = current { + if matches!( + error.downcast_ref::(), + Some(OverlayError::Stale) + ) { + return true; + } + current = error.source(); + } + false +} +/// Whether a failed step's cause chain holds an engine `Stale`. +pub(crate) fn failure_is_stale(error: &ServiceError) -> bool { + stale(error.as_ref()) +} +/// A failed port step: an engine `Stale` is the client's stale handle; any +/// other failure is an I/O error of this request alone. +pub fn failure(error: ServiceError) -> Status { + if stale(error.as_ref()) { + Status::Stale + } else { + Status::Io + } +} +fn read_failure(failure: ReadFailure) -> Status { + if stale(failure.reason.as_ref()) { + Status::Stale + } else { + Status::Io + } +} diff --git a/core/crates/layerfs-nfs/src/lib.rs b/core/crates/layerfs-nfs/src/lib.rs new file mode 100644 index 0000000000..f9377c13ed --- /dev/null +++ b/core/crates/layerfs-nfs/src/lib.rs @@ -0,0 +1,35 @@ +//! A mounted Workspace served as a loopback NFSv3 export. +//! +//! The platform-independent half of the native macOS route: the same +//! [`MountServices`](layerfs_fuse::ports::MountServices) ports and kernel +//! operations the FUSE connection uses, answered over ONC RPC instead of +//! `/dev/fuse`. A stateless NFS client holds no lookup references or open +//! descriptors, so the export keeps them itself (see [`kernel`]). +//! +//! Every NFSv3 procedure is served (MKNOD answers `NFS3ERR_NOTSUPP`: the +//! canonical format has no special files) and the MOUNT protocol; an export +//! configured read-only answers every changing procedure `NFS3ERR_ROFS`. +//! A changing procedure's publication tickets are released only after its +//! reply was sent (see [`change`]); names a client invents for itself are +//! kept out of the Workspace (see [`names`]). +//! There is no third-party dependency: XDR, RPC record marking and both +//! programs are implemented here over `std::net`. +#![forbid(unsafe_code)] + +pub mod attributes; +mod block; +pub mod change; +mod changing; +mod directory; +pub mod handle; +pub mod kernel; +mod mount; +pub mod names; +mod nfs; +pub mod refresh; +pub mod rpc; +mod server; +pub mod xdr; + +pub use kernel::Released; +pub use server::{Count, ExportConfig, Server}; diff --git a/core/crates/layerfs-nfs/src/mount.rs b/core/crates/layerfs-nfs/src/mount.rs new file mode 100644 index 0000000000..4b6656de04 --- /dev/null +++ b/core/crates/layerfs-nfs/src/mount.rs @@ -0,0 +1,66 @@ +//! MOUNT program 100005 version 3 (RFC 1813 appendix I): one export. +use crate::{ + kernel::Kernel, + rpc::Accept, + xdr::{Reader, Writer}, +}; + +pub const PROGRAM: u32 = 100_005; +pub const VERSION: u32 = 3; +const MNTPATHLEN: usize = 1024; +const MNT3_OK: u32 = 0; +const MNT3ERR_NOENT: u32 = 2; +const AUTH_UNIX: u32 = 1; + +/// Decodes and answers one MOUNT call for the export named `path`. +pub fn call( + kernel: &Kernel, + path: &str, + procedure: u32, + arguments: &mut Reader<'_>, + out: &mut Writer, +) -> Result<(), Accept> { + match procedure { + 0 | 4 => Ok(()), + 1 => { + let asked = arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + if exported(asked, path) { + out.u32(MNT3_OK); + out.opaque(&kernel.encode(kernel.root())); + out.u32(1).u32(AUTH_UNIX); + } else { + out.u32(MNT3ERR_NOENT); + } + Ok(()) + } + 2 => { + out.bool(false); + Ok(()) + } + 3 => { + arguments + .opaque(MNTPATHLEN) + .map_err(|_| Accept::GarbageArguments)?; + Ok(()) + } + 5 => { + out.bool(true).opaque(path.as_bytes()).bool(false); + out.bool(false); + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The export path, compared without trailing slashes ("/" names the root). +fn exported(asked: &[u8], path: &str) -> bool { + let trim = |bytes: &[u8]| -> Vec { + let mut bytes = bytes.to_vec(); + while bytes.len() > 1 && bytes.last() == Some(&b'/') { + bytes.pop(); + } + bytes + }; + trim(asked) == trim(path.as_bytes()) +} diff --git a/core/crates/layerfs-nfs/src/names.rs b/core/crates/layerfs-nfs/src/names.rs new file mode 100644 index 0000000000..1bc440f639 --- /dev/null +++ b/core/crates/layerfs-nfs/src/names.rs @@ -0,0 +1,38 @@ +//! Names an NFS client invents for its own bookkeeping, kept out of the +//! Workspace by the export itself so they never reach a Commit. +//! +//! - **Silly renames.** A client that unlinks a file it still has open +//! renames it instead, and removes that name after the last close. Linux +//! names it `.nfs` + 16 hex digits (fileid) + 8 hex digits (counter); +//! macOS `.nfs.` + 8 hex + `.` + 4 hex. Such a RENAME is served as an +//! UNLINK of the source name while the export keeps the inode alive by its +//! own reference and an open descriptor; the silly name exists only in the +//! export's table (LOOKUP, GETATTR, READ, WRITE, SETATTR and REMOVE of it +//! work; READDIR does not list it). The final REMOVE gives the inode back. +//! - **AppleDouble.** On NFSv3 the macOS client stores extended attributes +//! and resource forks in `._` companion files. Creating one is +//! refused with `NFS3ERR_ACCES` (when the export is configured to), so no +//! such file enters the Workspace; the client reports the extended +//! attribute as not settable. +//! +//! The patterns are exact so that a user's own `.nfsrc` or `._config` is an +//! ordinary name (the AppleDouble refusal is the one deliberate exception +//! and is configurable). + +/// A client-generated silly-rename name. +pub fn silly(name: &[u8]) -> bool { + let hex = |bytes: &[u8]| bytes.iter().all(u8::is_ascii_hexdigit); + if let Some(rest) = name.strip_prefix(b".nfs.") { + // macOS: .nfs.XXXXXXXX.XXXX + return rest.len() == 13 && rest[8] == b'.' && hex(&rest[..8]) && hex(&rest[9..]); + } + if let Some(rest) = name.strip_prefix(b".nfs") { + // Linux: .nfs + 16 + 8 hex digits + return rest.len() == 24 && hex(rest); + } + false +} +/// A macOS AppleDouble companion name (`._` followed by at least one byte). +pub fn apple_double(name: &[u8]) -> bool { + name.len() > 2 && name.starts_with(b"._") +} diff --git a/core/crates/layerfs-nfs/src/nfs.rs b/core/crates/layerfs-nfs/src/nfs.rs new file mode 100644 index 0000000000..7b208f8d9d --- /dev/null +++ b/core/crates/layerfs-nfs/src/nfs.rs @@ -0,0 +1,261 @@ +//! NFS program 100003 version 3: dispatch of every procedure. +//! +//! The reading procedures are answered here; the changing ones in +//! [`crate::changing`]. A read-only export answers every changing procedure +//! `NFS3ERR_ROFS` with its empty weak cache consistency data. +use crate::{ + attributes::{access, Status}, + change::Tickets, + changing, + directory::{list, Listing}, + kernel::Kernel, + rpc::{Accept, Credentials}, + xdr::{Garbage, Reader, Writer}, +}; +use layerfs_overlay::{READ_WINDOW, WRITE_WINDOW}; + +pub const PROGRAM: u32 = 100_003; +pub const VERSION: u32 = 3; +/// Procedure names by number, for diagnostics. +pub const PROCEDURES: [&str; 22] = [ + "NULL", + "GETATTR", + "SETATTR", + "LOOKUP", + "ACCESS", + "READLINK", + "READ", + "WRITE", + "CREATE", + "MKDIR", + "SYMLINK", + "MKNOD", + "REMOVE", + "RMDIR", + "RENAME", + "LINK", + "READDIR", + "READDIRPLUS", + "FSSTAT", + "FSINFO", + "PATHCONF", + "COMMIT", +]; +/// Longest handle (`NFS3_FHSIZE`) and the longest name a client may send. +const FHSIZE: usize = 64; +const NAME_LIMIT: usize = 4096; +/// Declared capacity: fixed values, never an admission signal (as for FUSE statfs). +const DECLARED_BLOCKS: u64 = 1 << 30; +const DECLARED_FILES: u64 = 1 << 30; +const BLOCK_BYTES: u64 = 4096; +/// FSINFO properties: FSF3_LINK | FSF3_SYMLINK | FSF3_HOMOGENEOUS | FSF3_CANSETTIME. +const PROPERTIES: u32 = 0x01 | 0x02 | 0x08 | 0x10; +/// Preferred READDIR reply size the client is told to ask for. +const DIRECTORY_PREFERENCE: u32 = 32 * 1024; + +/// The export's fixed answers. +#[derive(Clone, Copy, Debug)] +pub struct Policy { + pub read_only: bool, +} + +/// Decodes the arguments of `procedure`, runs it and appends its results. +/// `Err` is an RPC-level refusal of the call. +#[allow(clippy::too_many_arguments)] +pub fn call( + kernel: &Kernel, + policy: Policy, + procedure: u32, + credentials: &Credentials, + arguments: &mut Reader<'_>, + out: &mut Writer, + tickets: &mut Tickets, + verifier: [u8; 8], +) -> Result<(), Accept> { + let garbage = |_: Garbage| Accept::GarbageArguments; + match procedure { + 0 => Ok(()), + 1 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().fattr3(out, &stat); + } + Err(status) => { + out.u32(status.code()); + } + } + Ok(()) + } + 3 => { + let parent = handle(kernel, arguments).map_err(garbage)?; + let name = arguments.opaque(NAME_LIMIT).map_err(garbage)?; + match parent.and_then(|parent| kernel.lookup(parent, name)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + out.opaque(&kernel.encode(stat.serial)); + kernel.identity().post_op(out, &stat); + out.bool(false); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 4 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let requested = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + let caller = credentials + .unix + .as_ref() + .map(|unix| (unix.uid, unix.gid, unix.groups.as_slice())); + let granted = access( + kernel.identity(), + &stat, + caller, + requested, + policy.read_only, + ); + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + out.u32(granted); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 5 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.readlink(serial)) { + Ok(target) => { + out.u32(Status::Ok.code()).bool(false).opaque(&target); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 6 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + let offset = arguments.u64().map_err(garbage)?; + let count = arguments.u32().map_err(garbage)?; + match serial.and_then(|serial| kernel.read(serial, offset, count)) { + Ok((data, eof)) => { + out.u32(Status::Ok.code()).bool(false); + out.u32(data.len() as u32).bool(eof).opaque(&data); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 16 | 17 => { + let plus = procedure == 17; + let serial = handle(kernel, arguments).map_err(garbage)?; + let cookie = arguments.u64().map_err(garbage)?; + let verifier = arguments.fixed(8).map_err(garbage)?; + let mut budget = arguments.u32().map_err(garbage)?; + if plus { + // dircount bounds the names alone; maxcount the whole reply. + let _dircount = budget; + budget = arguments.u32().map_err(garbage)?; + } + let mut word = [0; 8]; + word.copy_from_slice(verifier); + let listing = |serial| Listing { + serial, + cookie, + verifier: u64::from_be_bytes(word), + budget: budget as usize, + plus, + }; + match serial.and_then(|serial| list(kernel, listing(serial))) { + Ok(body) => { + out.u32(Status::Ok.code()).fixed(&body); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 18..=20 => { + let serial = handle(kernel, arguments).map_err(garbage)?; + match serial.and_then(|serial| kernel.getattr(serial)) { + Ok(stat) => { + out.u32(Status::Ok.code()); + kernel.identity().post_op(out, &stat); + filesystem(procedure, out); + } + Err(status) => { + out.u32(status.code()).bool(false); + } + } + Ok(()) + } + 2 | 7..=15 | 21 if !policy.read_only => { + changing::call(kernel, procedure, arguments, out, tickets, verifier).map_err(garbage) + } + 2 | 7..=15 | 21 => { + // Each changing procedure's failure carries wcc_data (two absent + // optional fields) per directory it names; LINK also its object's + // post_op_attr. + let absent = match procedure { + 14 => 4, + 15 => 3, + _ => 2, + }; + out.u32(Status::RoFs.code()); + for _ in 0..absent { + out.bool(false); + } + Ok(()) + } + _ => Err(Accept::ProcedureUnavailable), + } +} +/// The serial of the call's handle argument; a handle that is not accepted is +/// the procedure's own failure status, not an RPC refusal. +fn handle(kernel: &Kernel, arguments: &mut Reader<'_>) -> Result, Garbage> { + let bytes = arguments.opaque(FHSIZE)?; + Ok(kernel.decode(bytes)) +} +/// FSSTAT, FSINFO and PATHCONF results after the object's attributes. +fn filesystem(procedure: u32, out: &mut Writer) { + match procedure { + 18 => { + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_BLOCKS * BLOCK_BYTES); + out.u64(DECLARED_FILES) + .u64(DECLARED_FILES) + .u64(DECLARED_FILES); + out.u32(0); // invarsec + } + 19 => { + let (read, write) = (READ_WINDOW as u32, WRITE_WINDOW as u32); + out.u32(read).u32(read).u32(BLOCK_BYTES as u32); + out.u32(write).u32(write).u32(BLOCK_BYTES as u32); + out.u32(DIRECTORY_PREFERENCE); + out.u64(i64::MAX as u64); + out.u32(0).u32(1); // time_delta: 1 ns + out.u32(PROPERTIES); + } + _ => { + out.u32(u32::MAX); // linkmax + out.u32(255); // name_max + out.bool(true); // no_trunc + out.bool(true); // chown_restricted + out.bool(false); // case_insensitive + out.bool(true); // case_preserving + } + } +} diff --git a/core/crates/layerfs-nfs/src/refresh.rs b/core/crates/layerfs-nfs/src/refresh.rs new file mode 100644 index 0000000000..32a7f1e071 --- /dev/null +++ b/core/crates/layerfs-nfs/src/refresh.rs @@ -0,0 +1,75 @@ +//! Cache refresh after a Commit installed another root (the EdenFS method). +//! +//! An NFS client has no invalidation channel: it keeps attributes and +//! listings until `actimeo` or its close-to-open check. A SETATTR it sends +//! itself is answered with fresh post-operation attributes, and a client +//! that sees them changed drops what it cached for that inode. So the daemon +//! names the inodes an install made stale ([`Kernel::refresh`]) and changes +//! their mode to the value it already has through the mount point. The +//! export answers each such mode-only SETATTR on a marked inode with its +//! current attributes and changes nothing (the mark is taken once and ends +//! after [`MARK_LIFE`]), so the refresh never writes into the Workspace and +//! never reverts a mode the install changed. +use crate::{change::Sattr, kernel::Kernel}; +use std::time::{Duration, Instant}; + +/// How long one refresh mark waits for its SETATTR. +pub const MARK_LIFE: Duration = Duration::from_secs(30); +/// Directory depth one path is built to at most. +const DEPTH: usize = 4096; + +impl Kernel { + /// Marks each of `serials` this export handed out and still knows by + /// name, and returns their paths below the export root (empty for the + /// root itself), in the order given. A serial the client never saw has + /// nothing cached and is left out. + pub fn refresh(&self, serials: impl IntoIterator) -> Vec>> { + let root = self.root(); + let mut table = self.table(); + let now = Instant::now(); + table + .refreshing + .retain(|_, marked| now.duration_since(*marked) < MARK_LIFE); + let mut paths = Vec::new(); + for serial in serials { + let mut path = Vec::new(); + let mut at = serial; + let mut known = true; + while at != root { + let Some((parent, name)) = table.names.get(&at) else { + known = false; + break; + }; + if path.len() >= DEPTH { + known = false; + break; + } + path.push(name.clone()); + at = *parent; + } + if known { + path.reverse(); + table.refreshing.insert(serial, now); + paths.push(path); + } + } + paths + } + /// Whether a SETATTR of `serial` is a pending refresh: mode only, on a + /// marked inode whose mark has not ended. Takes the mark. + pub(crate) fn refreshed(&self, serial: u64, sattr: &Sattr) -> bool { + let mode_only = sattr.mode.is_some() + && sattr.uid.is_none() + && sattr.gid.is_none() + && sattr.size.is_none() + && sattr.mtime.is_none(); + if !mode_only { + return false; + } + let mut table = self.table(); + match table.refreshing.remove(&serial) { + Some(marked) => marked.elapsed() < MARK_LIFE, + None => false, + } + } +} diff --git a/core/crates/layerfs-nfs/src/rpc.rs b/core/crates/layerfs-nfs/src/rpc.rs new file mode 100644 index 0000000000..43c2ded345 --- /dev/null +++ b/core/crates/layerfs-nfs/src/rpc.rs @@ -0,0 +1,181 @@ +//! ONC RPC v2 (RFC 5531) over TCP record marking: calls in, replies out. +use crate::xdr::{Garbage, Reader, Writer}; +use std::io::{self, Read, Write}; + +const CALL: u32 = 0; +const REPLY: u32 = 1; +const MSG_ACCEPTED: u32 = 0; +const MSG_DENIED: u32 = 1; +const RPC_MISMATCH: u32 = 0; +const AUTH_ERROR: u32 = 1; +const AUTH_NONE: u32 = 0; +const AUTH_UNIX: u32 = 1; +/// AUTH_TOOWEAK: the credential flavor is not accepted. +const AUTH_TOOWEAK: u32 = 5; +const LAST_FRAGMENT: u32 = 1 << 31; +/// Largest credential or verifier body RFC 5531 allows. +const AUTH_BODY: usize = 400; +/// Largest whole record accepted. A1 serves no WRITE, so every call is small; +/// the bound still admits a full-window WRITE for the later write path. +pub const RECORD_LIMIT: usize = 2 * 1024 * 1024; + +/// The accepted-reply status of one call. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Accept { + Success, + ProgramUnavailable, + /// The program exists in versions `low..=high` only. + ProgramMismatch { + low: u32, + high: u32, + }, + ProcedureUnavailable, + GarbageArguments, +} +/// The caller's identity as AUTH_UNIX states it. AUTH_NONE has none. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Credentials { + pub unix: Option, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UnixCredentials { + pub uid: u32, + pub gid: u32, + pub groups: Vec, +} +/// One decoded call header; its arguments follow in the same record. +#[derive(Debug)] +pub struct Call { + pub xid: u32, + pub program: u32, + pub version: u32, + pub procedure: u32, + pub credentials: Credentials, +} +/// A record that is not an answerable call. +#[derive(Debug)] +pub enum Refused { + /// Not a call message, or not decodable: no reply is possible. + Undecodable, + /// A call whose header can be answered only with a rejection. + Reply(Vec), +} + +/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`]. +/// `Ok(None)` is an orderly end of stream before a record started. +pub fn read_record(stream: &mut impl Read) -> io::Result>> { + let mut record = Vec::new(); + loop { + let mut header = [0; 4]; + if let Err(error) = stream.read_exact(&mut header) { + if error.kind() == io::ErrorKind::UnexpectedEof && record.is_empty() { + return Ok(None); + } + return Err(error); + } + let header = u32::from_be_bytes(header); + let length = (header & !LAST_FRAGMENT) as usize; + if record.len() + length > RECORD_LIMIT { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "RPC record exceeds its bound", + )); + } + let start = record.len(); + record.resize(start + length, 0); + stream.read_exact(&mut record[start..])?; + if header & LAST_FRAGMENT != 0 { + return Ok(Some(record)); + } + } +} +/// Writes one reply as a single last fragment. +pub fn write_record(stream: &mut impl Write, reply: &[u8]) -> io::Result<()> { + let mut framed = Vec::with_capacity(reply.len() + 4); + framed.extend_from_slice(&(reply.len() as u32 | LAST_FRAGMENT).to_be_bytes()); + framed.extend_from_slice(reply); + stream.write_all(&framed) +} + +/// Decodes the call header; the returned reader is positioned at the arguments. +pub fn parse_call(record: &[u8]) -> Result<(Call, Reader<'_>), Refused> { + let mut reader = Reader::new(record); + let header = (|| -> Result<(u32, u32, u32), Garbage> { + Ok((reader.u32()?, reader.u32()?, reader.u32()?)) + })(); + let (xid, kind, rpcvers) = header.map_err(|_| Refused::Undecodable)?; + if kind != CALL { + return Err(Refused::Undecodable); + } + if rpcvers != 2 { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(RPC_MISMATCH); + reply.u32(2).u32(2); + return Err(Refused::Reply(reply.into_bytes())); + } + let body = (|| -> Result<(u32, u32, u32, u32, &[u8]), Garbage> { + let program = reader.u32()?; + let version = reader.u32()?; + let procedure = reader.u32()?; + let flavor = reader.u32()?; + let credential = reader.opaque(AUTH_BODY)?; + let _verifier_flavor = reader.u32()?; + let _verifier = reader.opaque(AUTH_BODY)?; + Ok((program, version, procedure, flavor, credential)) + })(); + let (program, version, procedure, flavor, credential) = + body.map_err(|_| Refused::Undecodable)?; + let credentials = match flavor { + AUTH_NONE => Credentials::default(), + AUTH_UNIX => match unix_credentials(credential) { + Ok(unix) => Credentials { unix: Some(unix) }, + Err(Garbage) => return Err(Refused::Reply(denied_auth(xid))), + }, + _ => return Err(Refused::Reply(denied_auth(xid))), + }; + Ok(( + Call { + xid, + program, + version, + procedure, + credentials, + }, + reader, + )) +} +fn unix_credentials(body: &[u8]) -> Result { + let mut reader = Reader::new(body); + let _stamp = reader.u32()?; + let _machine = reader.opaque(255)?; + let uid = reader.u32()?; + let gid = reader.u32()?; + let count = reader.u32()? as usize; + if count > 16 { + return Err(Garbage); + } + let groups = (0..count) + .map(|_| reader.u32()) + .collect::, _>>()?; + Ok(UnixCredentials { uid, gid, groups }) +} +fn denied_auth(xid: u32) -> Vec { + let mut reply = Writer::new(); + reply.u32(xid).u32(REPLY).u32(MSG_DENIED).u32(AUTH_ERROR); + reply.u32(AUTH_TOOWEAK); + reply.into_bytes() +} +/// The accepted-reply header for `xid`; procedure results follow a success. +pub fn accepted(xid: u32, status: Accept) -> Writer { + let mut reply = Writer::with_capacity(256); + reply.u32(xid).u32(REPLY).u32(MSG_ACCEPTED); + reply.u32(AUTH_NONE).u32(0); + match status { + Accept::Success => reply.u32(0), + Accept::ProgramUnavailable => reply.u32(1), + Accept::ProgramMismatch { low, high } => reply.u32(2).u32(low).u32(high), + Accept::ProcedureUnavailable => reply.u32(3), + Accept::GarbageArguments => reply.u32(4), + }; + reply +} diff --git a/core/crates/layerfs-nfs/src/server.rs b/core/crates/layerfs-nfs/src/server.rs new file mode 100644 index 0000000000..91ccbdf807 --- /dev/null +++ b/core/crates/layerfs-nfs/src/server.rs @@ -0,0 +1,333 @@ +//! One export on one loopback TCP listener: MOUNT and NFS on the same port. +//! +//! Each accepted connection is served by its own thread, one call at a time, +//! in arrival order. A call's reply is written first; only after that single +//! send attempt returned are the publication tickets of the mutations it +//! decided released (see [`crate::change::Tickets`]). Stop closes the listener and every connection, joins +//! their threads and only then gives back the synthetic kernel's holdings, +//! so no call is running when references are released. +use crate::{ + change::Tickets, + kernel::{Kernel, Released}, + mount, + nfs::{self, Policy}, + rpc::{self, Accept, Refused}, +}; +use layerfs_fuse::ports::MountServices; +use layerfs_overlay::KernelMount; +use std::{ + io, + net::{Shutdown, SocketAddr, TcpListener, TcpStream}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Mutex, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +/// What the export shows and allows. +#[derive(Clone, Debug)] +pub struct ExportConfig { + /// The path MNT accepts, normally "/". + pub path: String, + /// The identity every inode is shown as. + pub uid: u32, + pub gid: u32, + /// Changing procedures answer `NFS3ERR_ROFS`. + pub read_only: bool, + /// Refuse to create macOS AppleDouble `._*` names (see [`crate::names`]). + pub refuse_apple_double: bool, + /// Lookup references kept before the least recently used are collected + /// (see [`crate::kernel`]); [`crate::kernel::LOOKUPS_HELD`] by default. + pub lookups_held: usize, +} +impl ExportConfig { + /// A writable export of "/" shown as `uid`:`gid`, refusing AppleDouble + /// names, with the default reference bound. + pub fn writable(uid: u32, gid: u32) -> Self { + Self { + path: "/".into(), + uid, + gid, + read_only: false, + refuse_apple_double: true, + lookups_held: crate::kernel::LOOKUPS_HELD, + } + } +} + +/// Calls served and time spent, per procedure. +pub struct Counters { + nfs: [(AtomicU64, AtomicU64); 22], + mount: [(AtomicU64, AtomicU64); 6], +} +impl Default for Counters { + fn default() -> Self { + Self { + nfs: std::array::from_fn(|_| Default::default()), + mount: std::array::from_fn(|_| Default::default()), + } + } +} +/// One procedure's totals. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Count { + pub procedure: &'static str, + pub calls: u64, + pub nanoseconds: u64, +} +const MOUNT_PROCEDURES: [&str; 6] = ["MNT_NULL", "MNT", "DUMP", "UMNT", "UMNTALL", "EXPORT"]; +impl Counters { + fn charge(&self, program: u32, procedure: u32, started: Instant) { + let slot = match program { + nfs::PROGRAM => self.nfs.get(procedure as usize), + mount::PROGRAM => self.mount.get(procedure as usize), + _ => None, + }; + if let Some((calls, time)) = slot { + calls.fetch_add(1, Ordering::Relaxed); + let spent = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); + time.fetch_add(spent, Ordering::Relaxed); + } + } + /// Procedures called at least once. + pub fn snapshot(&self) -> Vec { + let rows = |names: &[&'static str], slots: &[(AtomicU64, AtomicU64)]| { + names + .iter() + .zip(slots) + .map(|(name, (calls, time))| Count { + procedure: name, + calls: calls.load(Ordering::Relaxed), + nanoseconds: time.load(Ordering::Relaxed), + }) + .filter(|count| count.calls > 0) + .collect::>() + }; + let mut all = rows(&MOUNT_PROCEDURES, &self.mount); + all.extend(rows(&nfs::PROCEDURES, &self.nfs)); + all + } +} + +struct Shared { + kernel: Kernel, + /// The WRITE/COMMIT verifier: fresh for every export. + verifier: [u8; 8], + config: ExportConfig, + counters: Counters, + stopping: AtomicBool, + connections: Mutex)>>, +} +/// A running export. +pub struct Server { + address: SocketAddr, + shared: Arc, + accept: Option>, +} +impl Server { + /// Serves the engine mount `mount` over `services` on `listener`. + pub fn start( + listener: TcpListener, + services: Arc, + mount: KernelMount, + config: ExportConfig, + ) -> io::Result { + let address = listener.local_addr()?; + let shared = Arc::new(Shared { + kernel: Kernel::new(services, mount, config.uid, config.gid) + .refusing_apple_double(config.refuse_apple_double) + .holding(config.lookups_held), + verifier: fresh_verifier(), + config, + counters: Counters::default(), + stopping: AtomicBool::new(false), + connections: Mutex::default(), + }); + let accepting = shared.clone(); + let accept = thread::Builder::new() + .name("layerfs-nfs-accept".into()) + .spawn(move || accept_loop(&listener, &accepting))?; + Ok(Self { + address, + shared, + accept: Some(accept), + }) + } + pub fn local_addr(&self) -> SocketAddr { + self.address + } + pub fn counters(&self) -> Vec { + self.shared.counters.snapshot() + } + /// The synthetic kernel's current holdings. + pub fn held(&self) -> Released { + self.shared.kernel.held() + } + /// Closes the implicit file descriptors no silly name needs, for example + /// before a Commit; the next READ or WRITE reopens one. Returns how many. + pub fn close_files(&self) -> usize { + self.shared.kernel.close_files() + } + /// After a Commit installed a root other than the one the client saw: + /// closes the implicit file and enumeration descriptors, so the next + /// READ reopens the installed file and a continuing READDIR restarts. + /// Attributes the client cached are revalidated on its own schedule + /// (`actimeo`, close-to-open). Returns (files, directories) closed. + pub fn installed(&self) -> (usize, usize) { + ( + self.shared.kernel.close_files(), + self.shared.kernel.close_directories(), + ) + } + /// After an install: marks the stale `serials` the client may cache and + /// returns their paths below the mount point, for the mode-preserving + /// SETATTR that makes the client drop its cache (see [`crate::refresh`]). + pub fn refresh(&self, serials: impl IntoIterator) -> Vec { + use std::os::unix::ffi::OsStrExt; + (self.shared.kernel.refresh(serials).into_iter()) + .map(|names| { + (names.iter()) + .map(|name| std::ffi::OsStr::from_bytes(name)) + .collect() + }) + .collect() + } + /// The write verifier this export answers WRITE and COMMIT with. + pub fn verifier(&self) -> [u8; 8] { + self.shared.verifier + } + /// Stops accepting, closes every connection, joins every thread, then + /// releases the synthetic kernel's descriptors and references. The engine + /// mount itself stays with the caller, who revokes it afterwards. + pub fn stop(mut self) -> Released { + self.shutdown() + } + fn shutdown(&mut self) -> Released { + let Some(accept) = self.accept.take() else { + return Released::default(); + }; + self.shared.stopping.store(true, Ordering::Release); + // Wake the blocking accept; it sees the flag and returns. + drop(TcpStream::connect(self.address)); + drop(accept.join()); + let connections = std::mem::take( + &mut *self + .shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ); + for (stream, _) in &connections { + drop(stream.shutdown(Shutdown::Both)); + } + for (_, thread) in connections { + drop(thread.join()); + } + self.shared.kernel.release() + } +} +impl Drop for Server { + fn drop(&mut self) { + self.shutdown(); + } +} +fn accept_loop(listener: &TcpListener, shared: &Arc) { + for stream in listener.incoming() { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let Ok(stream) = stream else { continue }; + drop(stream.set_nodelay(true)); + let Ok(control) = stream.try_clone() else { + continue; + }; + let serving = shared.clone(); + let spawned = thread::Builder::new() + .name("layerfs-nfs-connection".into()) + .spawn(move || connection(stream, &serving)); + if let Ok(thread) = spawned { + let mut connections = shared + .connections + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + // A closed connection's thread has ended; forget it. + connections.retain(|(_, thread)| !thread.is_finished()); + connections.push((control, thread)); + } + } +} +fn connection(mut stream: TcpStream, shared: &Shared) { + while let Ok(Some(record)) = rpc::read_record(&mut stream) { + if shared.stopping.load(Ordering::Acquire) { + return; + } + let mut tickets = Tickets::default(); + let Some(reply) = answer(shared, &record, &mut tickets) else { + tickets.replied(&shared.kernel); + continue; + }; + // The single reply attempt, then the release of every ticket it + // owed, whether or not the reply reached the client. + let sent = rpc::write_record(&mut stream, &reply); + tickets.replied(&shared.kernel); + if sent.is_err() { + return; + } + } +} +/// A verifier no earlier export of this process or machine is likely to +/// have used: the clock and the process, under a per-process random key. +fn fresh_verifier() -> [u8; 8] { + use std::hash::BuildHasher; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |since| since.as_nanos()); + std::collections::hash_map::RandomState::new() + .hash_one((now, std::process::id())) + .to_be_bytes() +} +/// The reply to one record, or none when the record is not a call. +/// Mutations the call decided go to `tickets`. +fn answer(shared: &Shared, record: &[u8], tickets: &mut Tickets) -> Option> { + let started = Instant::now(); + let (call, mut arguments) = match rpc::parse_call(record) { + Ok(parsed) => parsed, + Err(Refused::Reply(reply)) => return Some(reply), + Err(Refused::Undecodable) => return None, + }; + let mut out = rpc::accepted(call.xid, Accept::Success); + let result = match (call.program, call.version) { + (nfs::PROGRAM, nfs::VERSION) => nfs::call( + &shared.kernel, + Policy { + read_only: shared.config.read_only, + }, + call.procedure, + &call.credentials, + &mut arguments, + &mut out, + tickets, + shared.verifier, + ), + (mount::PROGRAM, mount::VERSION) => mount::call( + &shared.kernel, + &shared.config.path, + call.procedure, + &mut arguments, + &mut out, + ), + (nfs::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + (mount::PROGRAM, _) => Err(Accept::ProgramMismatch { low: 3, high: 3 }), + _ => Err(Accept::ProgramUnavailable), + }; + let reply = match result { + Ok(()) => out.into_bytes(), + Err(status) => rpc::accepted(call.xid, status).into_bytes(), + }; + shared + .counters + .charge(call.program, call.procedure, started); + Some(reply) +} diff --git a/core/crates/layerfs-nfs/src/xdr.rs b/core/crates/layerfs-nfs/src/xdr.rs new file mode 100644 index 0000000000..18f572ed27 --- /dev/null +++ b/core/crates/layerfs-nfs/src/xdr.rs @@ -0,0 +1,120 @@ +//! XDR (RFC 4506) primitives over one received record and one reply buffer. +use std::fmt; + +/// The record ended before a field, or a length exceeded its declared bound. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Garbage; +impl fmt::Display for Garbage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("malformed XDR arguments") + } +} +impl std::error::Error for Garbage {} + +/// Sequential decoding of one record. Every read is bounds-checked. +pub struct Reader<'a> { + data: &'a [u8], + at: usize, +} +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data, at: 0 } + } + fn take(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let end = self.at.checked_add(length).ok_or(Garbage)?; + let slice = self.data.get(self.at..end).ok_or(Garbage)?; + self.at = end; + Ok(slice) + } + pub fn u32(&mut self) -> Result { + let bytes = self.take(4)?; + Ok(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + pub fn u64(&mut self) -> Result { + Ok((u64::from(self.u32()?) << 32) | u64::from(self.u32()?)) + } + pub fn bool(&mut self) -> Result { + match self.u32()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(Garbage), + } + } + /// Fixed-length opaque data, followed by its padding. + pub fn fixed(&mut self, length: usize) -> Result<&'a [u8], Garbage> { + let value = self.take(length)?; + self.take(pad(length))?; + Ok(value) + } + /// Variable-length opaque data of at most `limit` bytes. + pub fn opaque(&mut self, limit: usize) -> Result<&'a [u8], Garbage> { + let length = self.u32()? as usize; + if length > limit { + return Err(Garbage); + } + self.fixed(length) + } + /// The bytes not yet decoded. + pub fn remaining(&self) -> usize { + self.data.len() - self.at + } +} + +/// Bytes of padding after `length` bytes of opaque data. +pub const fn pad(length: usize) -> usize { + (4 - length % 4) % 4 +} +/// Encoded size of variable-length opaque data of `length` bytes. +pub const fn opaque_size(length: usize) -> usize { + 4 + length + pad(length) +} + +/// One reply under construction. +#[derive(Default)] +pub struct Writer { + data: Vec, +} +impl Writer { + pub fn new() -> Self { + Self::default() + } + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: Vec::with_capacity(capacity), + } + } + pub fn u32(&mut self, value: u32) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn u64(&mut self, value: u64) -> &mut Self { + self.data.extend_from_slice(&value.to_be_bytes()); + self + } + pub fn bool(&mut self, value: bool) -> &mut Self { + self.u32(u32::from(value)) + } + /// Fixed-length opaque data and its padding. + pub fn fixed(&mut self, value: &[u8]) -> &mut Self { + self.data.extend_from_slice(value); + self.data.extend_from_slice(&[0; 3][..pad(value.len())]); + self + } + /// Variable-length opaque data; callers bound `value` by its protocol limit. + pub fn opaque(&mut self, value: &[u8]) -> &mut Self { + self.u32(value.len() as u32); + self.fixed(value) + } + pub fn len(&self) -> usize { + self.data.len() + } + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + pub fn into_bytes(self) -> Vec { + self.data + } + pub fn as_bytes(&self) -> &[u8] { + &self.data + } +} diff --git a/core/crates/layerfs-nfs/tests/collect.rs b/core/crates/layerfs-nfs/tests/collect.rs new file mode 100644 index 0000000000..b4dfaddbe9 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/collect.rs @@ -0,0 +1,170 @@ +//! Reference collection (A3, plan A): an export configured with a small +//! reference bound walks a tree far larger than the bound, repeatedly, and +//! keeps both its own holdings and the engine's kernel entries bounded. +//! Handles of collected serials keep working (resolved again by their name); +//! a handle whose name now reaches another inode is stale; the product +//! Commit afterwards holds exactly what the client made. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, OK, STALE}; +use std::{fs, net::TcpListener, path::PathBuf, time::Duration}; + +const UID: u32 = 501; +const GID: u32 = 20; +const BOUND: usize = 64; +const DIRECTORIES: usize = 8; +const FILES: usize = 120; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-c{}-{label}", std::process::id())) +} +fn content(directory: usize, file: usize) -> Vec { + format!("d{directory}/f{file}\n") + .repeat(file % 7 + 1) + .into_bytes() +} +fn kernel_entries(rig: &rig::Rig) -> u64 { + let operation = rig.bound.operation().unwrap(); + operation.overlay().resources().unwrap().open.kernel_entries +} + +#[test] +fn references_stay_bounded_and_collected_handles_resolve() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + for directory in 0..DIRECTORIES { + let path = source.join(format!("d{directory}/inner")); + fs::create_dir_all(&path).unwrap(); + for file in 0..FILES { + fs::write( + source.join(format!("d{directory}/f{file}")), + content(directory, file), + ) + .unwrap(); + } + fs::write(path.join("leaf"), b"leaf\n").unwrap(); + } + let mut rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + rig.lookups_held = BOUND; + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let total = DIRECTORIES * (FILES + 3); + // The bound is soft: the most recently used serials, the directories a + // held serial was reached through and serials holding a descriptor are + // never collected. + let ceiling = BOUND + layerfs_nfs::kernel::RECENT_KEPT as usize + DIRECTORIES * 2 + 8; + let mut first: Vec<(Vec, u64, usize, usize)> = Vec::new(); + let mut peak_held = 0; + let mut peak_engine = 0; + for round in 0..6 { + for directory in 0..DIRECTORIES { + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + for file in 0..FILES { + let (handle, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + if round == 0 && file % 15 == 0 { + first.push((handle, attr.fileid, directory, file)); + } + } + let (inner, _) = client.lookup(&dir, b"inner").unwrap(); + client.lookup(&inner, b"leaf").unwrap(); + // Churn: a file made and removed in every directory, every round. + let made = format!("tmp{round}"); + client.touch(&dir, made.as_bytes(), 0o644).unwrap(); + client.remove(&dir, made.as_bytes()).unwrap(); + peak_held = peak_held.max(server.held().lookups); + peak_engine = peak_engine.max(kernel_entries(&rig)); + } + println!( + "ROUND {round} held={:?} engine_kernel_entries={}", + server.held(), + kernel_entries(&rig) + ); + } + assert!(peak_held <= ceiling, "export held {peak_held} > {ceiling}"); + // The engine counts the export's references (plus the root's). + assert!( + peak_engine <= ceiling as u64 + 2, + "engine kernel entries {peak_engine} > {ceiling}" + ); + assert!( + server.held().collected as usize >= total * 5, + "{:?}", + server.held() + ); + // Every early handle was collected long ago; each resolves to its file. + for (handle, fileid, directory, file) in &first { + let attr = client.getattr(handle).unwrap(); + assert_eq!(attr.fileid, *fileid); + assert_eq!(client.read_all(handle, 4096), content(*directory, *file)); + } + let resolved = server.held().resolved; + assert!(resolved >= first.len() as u64, "{:?}", server.held()); + // A collected handle whose name was replaced by another file is stale. + let (victim, victim_fileid, directory, file) = first[1].clone(); + for _ in 0..3 { + // collect it again + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d7").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + } + let (dir, _) = client + .lookup(&root, format!("d{directory}").as_bytes()) + .unwrap(); + let replacement = client.touch(&dir, b"replacement", 0o644).unwrap(); + client + .rename(&dir, b"replacement", &dir, format!("f{file}").as_bytes()) + .unwrap(); + assert_eq!(client.getattr(&victim).map(|attr| attr.fileid), Err(STALE)); + let (now, attr) = client.lookup(&dir, format!("f{file}").as_bytes()).unwrap(); + assert_ne!(attr.fileid, victim_fileid); + assert_eq!(now, replacement.handle); + // `..` of a directory whose parent was collected still answers. + let (d3, _) = client.lookup(&root, b"d3").unwrap(); + let (inner, _) = client.lookup(&d3, b"inner").unwrap(); + for other in 0..FILES { + let (dir, _) = client.lookup(&root, b"d6").unwrap(); + client.lookup(&dir, format!("f{other}").as_bytes()).unwrap(); + } + let (up, up_attr) = client.lookup(&inner, b"..").unwrap(); + assert_eq!(up_attr.fileid, client.getattr(&d3).unwrap().fileid); + assert_eq!(client.getattr(&up).map(|_| OK), Ok(OK)); + // The Commit holds what the client made. + let closed = server.close_files(); + let done = rig.commit(); + assert!(done.history.is_some()); + println!("COMMITTED closed={closed} held={:?}", server.held()); + let view = rig.view([93; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (replaced, _) = reader + .walk_to(&view_root, &format!("d{directory}/f{file}")) + .unwrap(); + assert!(reader.read_all(&replaced, 4096).is_empty()); + let (kept, _) = reader.walk_to(&view_root, "d5/f119").unwrap(); + assert_eq!(reader.read_all(&kept, 4096), content(5, 119)); + assert_eq!( + reader.lookup(&view_root, b"tmp0").map(|_| ()), + Err(client::NOENT) + ); + drop(reader); + viewer.stop(); + rig.close_view(view); + drop(client); + let released = server.stop(); + println!("RELEASED {released:?}"); + assert!(released.lookups <= ceiling); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(&source).unwrap(); +} diff --git a/core/crates/layerfs-nfs/tests/export.rs b/core/crates/layerfs-nfs/tests/export.rs new file mode 100644 index 0000000000..6c0a7c7617 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/export.rs @@ -0,0 +1,185 @@ +//! A committed Branch served read-only over loopback NFSv3: every path, +//! byte, link target, mode and identity of the source tree through the +//! protocol alone, the synthetic kernel's reference discipline, cookies, +//! refusals and stale handles across engine mounts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{Attr, Client, BADHANDLE, BAD_COOKIE, NOENT, ROFS, STALE}; +use layerfs_nfs::xdr::Writer; +use std::{ + collections::BTreeSet, + fs, + net::TcpListener, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + time::Duration, +}; +use tree::{oracle, walk, Node}; + +const UID: u32 = 501; +const GID: u32 = 20; +const MANY: usize = 700; +const BIG: usize = 300_001; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested/deep")).unwrap(); + fs::create_dir_all(root.join("many")).unwrap(); + fs::create_dir_all(root.join("emptydir")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write( + root.join("big.bin"), + (0..BIG).map(|n| (n % 251) as u8).collect::>(), + ) + .unwrap(); + fs::write(root.join("empty"), b"").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep/file.txt"), b"deep\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/lib-alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + std::os::unix::fs::symlink("../nowhere", root.join("dangling")).unwrap(); + fs::write(root.join("unicod\u{e9}.txt"), "caf\u{e9}\n").unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\necho run\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); + for n in 0..MANY { + fs::write(root.join(format!("many/f-{n:04}")), format!("{n}\n")).unwrap(); + } +} +#[test] +fn a_committed_branch_reads_completely_over_nfs() { + let base = temp("complete"); + let tree = base.join("tree"); + source(&tree); + let expected = oracle(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + assert_eq!(rig.entries as usize, expected.len()); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + + let (paths, bytes, identities) = walk(&mut client, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + assert!(bytes > BIG); + // One inode behind both names of the hard link, two names for it, and + // distinct inodes everywhere else. + let alias = expected.iter().find_map(|(path, node)| match node { + Node::File { ino, .. } if path == Path::new("src/lib.rs") => Some(*ino), + _ => None, + }); + let fileids = &identities[&alias.unwrap()]; + assert_eq!(fileids.len(), 1, "both names show one fileid"); + let lib = client.mnt("/"); + let (src, _) = client.lookup(&lib, b"src").unwrap(); + let (_, lib_attr) = client.lookup(&src, b"lib.rs").unwrap(); + assert_eq!(lib_attr.nlink, 2); + let all: BTreeSet = identities.values().flatten().copied().collect(); + assert_eq!( + all.len(), + identities.len(), + "distinct inodes, distinct fileids" + ); + + // Re-walking takes no further engine references: one per handed-out serial. + // Every path but the root holds one reference; the hard link's two names + // share one serial. + let serials = expected.len() - 2; + let held = server.held(); + assert_eq!(held.lookups, serials, "one reference per non-root serial"); + walk(&mut client, &expected, 64 * 1024, (UID, GID)); + assert_eq!(server.held().lookups, held.lookups); + + let released = server.stop(); + assert_eq!(released.lookups, serials); + assert!(released.files > 0 && released.directories > 0); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0, "every Store reader was given back"); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn cookies_refusals_and_stale_handles() { + let base = temp("protocol"); + let tree = base.join("tree"); + source(&tree); + let mut rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (many, _) = client.lookup(&root, b"many").unwrap(); + + // A small budget spreads one enumeration over many replies; every name + // appears exactly once, in both procedures. + for plus in [false, true] { + let (entries, calls) = client.list(&many, 1024, plus); + let names: BTreeSet> = entries.iter().map(|entry| entry.name.clone()).collect(); + assert_eq!(names.len(), entries.len(), "no duplicate (plus={plus})"); + assert_eq!(entries.len(), MANY + 2, "plus={plus}"); + assert!(calls > 10, "{calls} calls (plus={plus})"); + } + // A verifier the export never issued, and a budget too small for one entry. + assert_eq!( + client.readdir(&many, 5, 0xdead_beef, 4096, false), + Err(BAD_COOKIE) + ); + // Missing names, a foreign handle and every changing procedure. + assert_eq!(client.lookup(&root, b"absent").map(|_| ()), Err(NOENT)); + // Not a handle this export issues: wrong version or length. A version-1 + // handle of another engine mount is stale (its key is gone); a forged + // tag under this mount is refused (see protocol.rs). + assert_eq!(client.getattr(&[2; 25]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 10]), Err(BADHANDLE)); + assert_eq!(client.getattr(&[1; 25]), Err(STALE)); + for procedure in [2, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21] { + let mut arguments = Writer::new(); + arguments.opaque(&root); + assert_eq!(client.nfs(procedure, &arguments).0, ROFS, "{procedure}"); + } + // A read-only export grants reads only, whatever the mode allows. + let (readme, readme_attr): (Vec, Attr) = client.lookup(&root, b"README.md").unwrap(); + let host = fs::metadata(tree.join("README.md")).unwrap().mode() & 0o7777; + assert_eq!(readme_attr.mode, host); + assert_eq!(host & 0o444, 0o444, "the fixture is readable by everyone"); + assert_eq!(client.access(&readme, 0x3f), Ok(0x01)); + let mut other = Client::connect(server.local_addr(), 1234, 1234); + assert_eq!(other.access(&readme, 0x3f), Ok(0x01), "other: r--"); + let (run, _) = client.lookup(&root, b"run.sh").unwrap(); + assert_eq!(client.access(&run, 0x3f), Ok(0x01 | 0x20)); + // Reads beyond the end, and windows that are not page aligned. + let (big, big_attr) = client.lookup(&root, b"big.bin").unwrap(); + assert_eq!( + client.read(&big, big_attr.size + 10, 4096), + Ok((vec![], true)) + ); + let (window, eof) = client.read(&big, 1000, 70_000).unwrap(); + assert!(!eof); + assert_eq!( + window, + (1000..71_000).map(|n| (n % 251) as u8).collect::>() + ); + + // A restarted export is a new engine mount: earlier handles are stale. + drop(client); + drop(other); + let released = server.stop(); + rig.remount(); + let server = rig.serve(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID); + let mut client = Client::connect(server.local_addr(), UID, GID); + assert_eq!(client.getattr(&readme), Err(STALE)); + assert_eq!(client.getattr(&root), Err(STALE)); + let fresh = client.mnt("/"); + assert!(client.getattr(&fresh).is_ok()); + drop(client); + let released_again = server.stop(); + assert!(released.lookups > 0); + let (_, outstanding) = rig.stop(released_again); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/core/crates/layerfs-nfs/tests/protocol.rs b/core/crates/layerfs-nfs/tests/protocol.rs new file mode 100644 index 0000000000..c513df60aa --- /dev/null +++ b/core/crates/layerfs-nfs/tests/protocol.rs @@ -0,0 +1,123 @@ +//! XDR, RPC record marking, call headers and file handles through the public API. +use layerfs_nfs::{ + handle::{HandleError, Handles, HANDLE_BYTES}, + rpc::{parse_call, read_record, write_record, Refused, RECORD_LIMIT}, + xdr::{opaque_size, Reader, Writer}, +}; + +#[test] +fn xdr_round_trips_with_padding() { + let mut out = Writer::new(); + out.u32(7) + .u64(u64::MAX - 1) + .bool(true) + .opaque(b"abcde") + .fixed(b"xy"); + assert_eq!(out.len(), 4 + 8 + 4 + opaque_size(5) + 4); + let bytes = out.into_bytes(); + let mut reader = Reader::new(&bytes); + assert_eq!(reader.u32(), Ok(7)); + assert_eq!(reader.u64(), Ok(u64::MAX - 1)); + assert_eq!(reader.bool(), Ok(true)); + assert_eq!(reader.opaque(5), Ok(&b"abcde"[..])); + assert_eq!(reader.fixed(2), Ok(&b"xy"[..])); + assert_eq!(reader.remaining(), 0); + assert!(reader.u32().is_err(), "reads past the record fail"); + let mut long = Writer::new(); + long.opaque(&[0; 9]); + let long = long.into_bytes(); + assert!( + Reader::new(&long).opaque(8).is_err(), + "a length beyond its bound" + ); + assert!( + Reader::new(&[0, 0, 0, 2]).bool().is_err(), + "a bool is 0 or 1" + ); +} + +#[test] +fn records_join_fragments_and_respect_their_bound() { + let mut wire = Vec::new(); + wire.extend_from_slice(&3u32.to_be_bytes()); + wire.extend_from_slice(b"abc"); + wire.extend_from_slice(&(2u32 | (1 << 31)).to_be_bytes()); + wire.extend_from_slice(b"de"); + write_record(&mut wire, b"next").unwrap(); + let mut stream = &wire[..]; + assert_eq!(read_record(&mut stream).unwrap(), Some(b"abcde".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), Some(b"next".to_vec())); + assert_eq!(read_record(&mut stream).unwrap(), None, "orderly end"); + let huge = (RECORD_LIMIT as u32 + 1) | (1 << 31); + assert!(read_record(&mut &huge.to_be_bytes()[..]).is_err()); +} + +fn call(rpcvers: u32, flavor: u32, credential: &[u8]) -> Vec { + let mut out = Writer::new(); + out.u32(9).u32(0).u32(rpcvers).u32(100_003).u32(3).u32(1); + out.u32(flavor).opaque(credential).u32(0).u32(0); + out.u32(0xabcd); + out.into_bytes() +} + +#[test] +fn call_headers_carry_unix_credentials() { + let mut unix = Writer::new(); + unix.u32(0) + .opaque(b"host") + .u32(501) + .u32(20) + .u32(2) + .u32(12) + .u32(61); + let record = call(2, 1, unix.as_bytes()); + let (header, mut arguments) = parse_call(&record).unwrap(); + assert_eq!( + (header.xid, header.program, header.version, header.procedure), + (9, 100_003, 3, 1) + ); + let credentials = header.credentials.unix.unwrap(); + assert_eq!( + (credentials.uid, credentials.gid, credentials.groups), + (501, 20, vec![12, 61]) + ); + assert_eq!(arguments.u32(), Ok(0xabcd)); + assert!(parse_call(&call(2, 0, b"")) + .unwrap() + .0 + .credentials + .unix + .is_none()); + // RPC version 3 and an unknown flavor are answered with a denial. + for record in [call(3, 0, b""), call(2, 6, b"")] { + match parse_call(&record) { + Err(Refused::Reply(reply)) => { + assert_eq!(&reply[..12], &[0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 1]) + } + Err(Refused::Undecodable) => panic!("undecodable"), + Ok((call, _)) => panic!("{call:?}"), + } + } + assert!(matches!( + parse_call(&[0, 0, 0, 9, 0, 0, 0, 1]), + Err(Refused::Undecodable) + )); +} + +#[test] +fn handles_are_stable_per_mount_and_stale_after_it() { + let handles = Handles::new(41); + let first = handles.encode(1234); + assert_eq!(first.len(), HANDLE_BYTES); + assert_eq!( + handles.encode(1234), + first, + "the same serial, the same bytes" + ); + assert_eq!(handles.decode(&first), Ok(1234)); + let mut forged = first; + forged[16] ^= 1; + assert_eq!(handles.decode(&forged), Err(HandleError::Bad)); + assert_eq!(handles.decode(&first[..24]), Err(HandleError::Bad)); + assert_eq!(Handles::new(42).decode(&first), Err(HandleError::Stale)); +} diff --git a/core/crates/layerfs-nfs/tests/refresh.rs b/core/crates/layerfs-nfs/tests/refresh.rs new file mode 100644 index 0000000000..7ff99c4d96 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/refresh.rs @@ -0,0 +1,117 @@ +//! Cache refresh after an install (B): the inodes the daemon names are +//! marked and their paths returned; a mode-only SETATTR on a marked inode is +//! answered with its current attributes and changes nothing, once; every +//! other SETATTR still changes the inode; the Commit holds only real changes. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, Set}; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-r{}-{label}", std::process::id())) +} + +#[test] +fn a_refresh_chmod_changes_nothing_and_other_setattrs_still_do() { + let source = temp("src"); + let work = temp("work"); + drop(fs::remove_dir_all(&source)); + drop(fs::remove_dir_all(&work)); + fs::create_dir_all(source.join("a/b")).unwrap(); + fs::write(source.join("a/b/file"), b"inner\n").unwrap(); + fs::write(source.join("top.txt"), b"top\n").unwrap(); + for (path, mode) in [("a/b/file", 0o644), ("top.txt", 0o644), ("a", 0o755)] { + fs::set_permissions(source.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let (file, file_attr) = client.walk_to(&root, "a/b/file").unwrap(); + let (a, a_attr) = client.lookup(&root, b"a").unwrap(); + let (top, top_attr) = client.lookup(&root, b"top.txt").unwrap(); + let root_id = client.getattr(&root).unwrap().fileid; + // Paths of the serials the client knows, the root's empty; an unknown serial is left out. + let paths = server.refresh([ + file_attr.fileid, + a_attr.fileid, + root_id, + u64::MAX - 7, + top_attr.fileid, + ]); + assert_eq!( + paths, + vec![ + PathBuf::from("a/b/file"), + PathBuf::from("a"), + PathBuf::new(), + PathBuf::from("top.txt"), + ] + ); + let mode = |set: Result| { + set.map(|wcc| wcc.after.map(|after| after.mode & 0o7777)) + .map_err(|(status, _)| status) + }; + let chmod = |mode: u32| Set { + mode: Some(mode), + ..Set::default() + }; + // A marked inode: the mode-only SETATTR is answered, nothing changes. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o644)) + ); + assert_eq!( + mode(client.setattr(&a, chmod(0o700), None)), + Ok(Some(0o755)) + ); + assert_eq!(client.getattr(&file).unwrap().mode & 0o7777, 0o644); + // The mark is taken once: the next chmod is real. + assert_eq!( + mode(client.setattr(&file, chmod(0o600), None)), + Ok(Some(0o600)) + ); + // A SETATTR that is not mode-only is never taken for a refresh. + let truncate = Set { + mode: Some(0o640), + size: Some(0), + ..Set::default() + }; + let after = client.setattr(&top, truncate, None).unwrap().after.unwrap(); + assert_eq!((after.mode & 0o7777, after.size), (0o640, 0)); + // The Commit holds the real changes only. + server.close_files(); + assert!(rig.commit().history.is_some()); + let view = rig.view([71; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let at = |reader: &mut Client, path: &str| reader.walk_to(&view_root, path).unwrap().1; + assert_eq!(at(&mut reader, "a/b/file").mode & 0o7777, 0o600); + assert_eq!(at(&mut reader, "a").mode & 0o7777, 0o755); + let top_view = at(&mut reader, "top.txt"); + assert_eq!((top_view.mode & 0o7777, top_view.size), (0o640, 0)); + drop(reader); + let _released = viewer.stop(); + rig.close_view(view); + drop(client); + let served = server.stop(); + let (_, outstanding) = rig.stop(served); + assert_eq!(outstanding, 0); + for path in [&source, &work] { + drop(fs::remove_dir_all(Path::new(path))); + } +} diff --git a/core/crates/layerfs-nfs/tests/support/client.rs b/core/crates/layerfs-nfs/tests/support/client.rs new file mode 100644 index 0000000000..0c579d89b2 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/support/client.rs @@ -0,0 +1,588 @@ +//! A minimal NFSv3/MOUNT client over the export's public XDR and record +//! framing: what a kernel client sends, one call at a time. +use layerfs_nfs::{ + rpc::{read_record, write_record}, + xdr::{Reader, Writer}, +}; +use std::{ + net::TcpStream, + sync::{Arc, Condvar, Mutex}, + time::Duration, +}; + +/// Calls sent and replies fully received by clients sharing it, so a test's +/// port wrapper can tell whether a reply was already on the wire. +#[derive(Default)] +pub struct Wire { + pub counts: Mutex<(u64, u64)>, + pub changed: Condvar, +} + +pub const NFS: u32 = 100_003; +pub const MOUNT: u32 = 100_005; +pub const OK: u32 = 0; +pub const NOENT: u32 = 2; +pub const ROFS: u32 = 30; +pub const STALE: u32 = 70; +pub const BADHANDLE: u32 = 10001; +pub const BAD_COOKIE: u32 = 10003; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Attr { + pub kind: u32, + pub mode: u32, + pub nlink: u32, + pub uid: u32, + pub gid: u32, + pub size: u64, + pub fileid: u64, + pub mtime: (u32, u32), +} +fn attr(reader: &mut Reader<'_>) -> Attr { + let kind = reader.u32().unwrap(); + let mode = reader.u32().unwrap(); + let nlink = reader.u32().unwrap(); + let uid = reader.u32().unwrap(); + let gid = reader.u32().unwrap(); + let size = reader.u64().unwrap(); + let _used = reader.u64().unwrap(); + let _rdev = (reader.u32().unwrap(), reader.u32().unwrap()); + let _fsid = reader.u64().unwrap(); + let fileid = reader.u64().unwrap(); + let _atime = (reader.u32().unwrap(), reader.u32().unwrap()); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let _ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + Attr { + kind, + mode, + nlink, + uid, + gid, + size, + fileid, + mtime, + } +} +fn post_op(reader: &mut Reader<'_>) -> Option { + reader.bool().unwrap().then(|| attr(reader)) +} +/// One READDIR/READDIRPLUS entry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Entry { + pub name: Vec, + pub fileid: u64, + pub cookie: u64, +} + +pub struct Client { + stream: TcpStream, + xid: u32, + uid: u32, + gid: u32, + pub calls: u64, + pub wire: Option>, +} +impl Client { + pub fn connect(address: std::net::SocketAddr, uid: u32, gid: u32) -> Self { + let stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + Self { + stream, + xid: 1, + uid, + gid, + calls: 0, + wire: None, + } + } + /// One call; the accept status and the results after it. + pub fn call(&mut self, program: u32, procedure: u32, arguments: &[u8]) -> (u32, Vec) { + self.xid += 1; + self.calls += 1; + let mut credential = Writer::new(); + credential + .u32(0) + .opaque(b"test") + .u32(self.uid) + .u32(self.gid) + .u32(0); + let mut call = Writer::new(); + call.u32(self.xid) + .u32(0) + .u32(2) + .u32(program) + .u32(3) + .u32(procedure); + call.u32(1).opaque(credential.as_bytes()); + call.u32(0).u32(0); + call.fixed(arguments); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().0 += 1; + } + write_record(&mut self.stream, call.as_bytes()).unwrap(); + let record = read_record(&mut self.stream).unwrap().unwrap(); + if let Some(wire) = &self.wire { + wire.counts.lock().unwrap().1 += 1; + wire.changed.notify_all(); + } + let mut reader = Reader::new(&record); + assert_eq!(reader.u32().unwrap(), self.xid); + assert_eq!(reader.u32().unwrap(), 1, "a reply"); + assert_eq!(reader.u32().unwrap(), 0, "accepted"); + let _flavor = reader.u32().unwrap(); + let _verifier = reader.opaque(400).unwrap(); + let status = reader.u32().unwrap(); + let rest = record[record.len() - reader.remaining()..].to_vec(); + (status, rest) + } + /// One NFS procedure that the RPC layer accepted: its nfsstat3 and results. + pub fn nfs(&mut self, procedure: u32, arguments: &Writer) -> (u32, Vec) { + let (accept, rest) = self.call(NFS, procedure, arguments.as_bytes()); + assert_eq!(accept, 0, "procedure {procedure} accepted"); + let mut reader = Reader::new(&rest); + let status = reader.u32().unwrap(); + (status, rest[4..].to_vec()) + } + pub fn mnt(&mut self, path: &str) -> Vec { + let mut arguments = Writer::new(); + arguments.opaque(path.as_bytes()); + let (accept, rest) = self.call(MOUNT, 1, arguments.as_bytes()); + assert_eq!(accept, 0); + let mut reader = Reader::new(&rest); + assert_eq!(reader.u32().unwrap(), 0, "MNT3_OK"); + reader.opaque(64).unwrap().to_vec() + } + pub fn getattr(&mut self, handle: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(1, &arguments); + if status != OK { + return Err(status); + } + Ok(attr(&mut Reader::new(&rest))) + } + pub fn lookup(&mut self, directory: &[u8], name: &[u8]) -> Result<(Vec, Attr), u32> { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let handle = reader.opaque(64).unwrap().to_vec(); + let attributes = post_op(&mut reader).expect("LOOKUP returns object attributes"); + Ok((handle, attributes)) + } + pub fn access(&mut self, handle: &[u8], requested: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle).u32(requested); + let (status, rest) = self.nfs(4, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.u32().unwrap()) + } + pub fn readlink(&mut self, handle: &[u8]) -> Result, u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle); + let (status, rest) = self.nfs(5, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + Ok(reader.opaque(4096).unwrap().to_vec()) + } + /// One READ: data and EOF. + pub fn read(&mut self, handle: &[u8], offset: u64, count: u32) -> Result<(Vec, bool), u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(offset).u32(count); + let (status, rest) = self.nfs(6, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let count = reader.u32().unwrap(); + let eof = reader.bool().unwrap(); + let data = reader.opaque(1 << 20).unwrap().to_vec(); + assert_eq!(count as usize, data.len()); + Ok((data, eof)) + } + /// READs until EOF in windows of `count` bytes. + pub fn read_all(&mut self, handle: &[u8], count: u32) -> Vec { + let mut bytes = Vec::new(); + loop { + let (data, eof) = self.read(handle, bytes.len() as u64, count).unwrap(); + bytes.extend_from_slice(&data); + if eof { + return bytes; + } + assert!(!data.is_empty(), "a short non-EOF read makes progress"); + } + } + /// One READDIR (or READDIRPLUS): entries, verifier, EOF. + pub fn readdir( + &mut self, + handle: &[u8], + cookie: u64, + verifier: u64, + budget: u32, + plus: bool, + ) -> Result<(Vec, u64, bool), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(cookie) + .u64(verifier) + .u32(budget); + if plus { + arguments.u32(budget); + } + let (status, rest) = self.nfs(if plus { 17 } else { 16 }, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + post_op(&mut reader); + let verifier = reader.u64().unwrap(); + let mut entries = Vec::new(); + while reader.bool().unwrap() { + let fileid = reader.u64().unwrap(); + let name = reader.opaque(4096).unwrap().to_vec(); + let cookie = reader.u64().unwrap(); + if plus { + assert!(post_op(&mut reader).is_none()); + assert!(!reader.bool().unwrap(), "no handle in READDIRPLUS"); + } + entries.push(Entry { + name, + fileid, + cookie, + }); + } + let eof = reader.bool().unwrap(); + Ok((entries, verifier, eof)) + } + /// A whole enumeration as a kernel client runs it, and the calls it took. + pub fn list(&mut self, handle: &[u8], budget: u32, plus: bool) -> (Vec, u64) { + let (mut cookie, mut verifier, mut calls) = (0, 0, 0); + let mut all = Vec::new(); + loop { + let (entries, next, eof) = self + .readdir(handle, cookie, verifier, budget, plus) + .unwrap(); + calls += 1; + verifier = next; + if let Some(last) = entries.last() { + cookie = last.cookie; + } + all.extend(entries); + if eof { + return (all, calls); + } + } + } +} + +pub const PERM: u32 = 1; +pub const EXIST: u32 = 17; +pub const NOTDIR: u32 = 20; +pub const ISDIR: u32 = 21; +pub const INVAL: u32 = 22; +pub const NAMETOOLONG: u32 = 63; +pub const NOTEMPTY: u32 = 66; +pub const ACCES: u32 = 13; +pub const NOT_SYNC: u32 = 10002; +pub const NOTSUPP: u32 = 10004; +pub const JUKEBOX: u32 = 10008; + +/// `wcc_attr`: size, mtime and ctime before a change. +pub type Before = (u64, (u32, u32), (u32, u32)); +/// `wcc_data`: the pre-op (size, mtime, ctime) and post-op attributes. +#[derive(Clone, Debug, Default)] +pub struct Wcc { + pub before: Option, + pub after: Option, +} +pub fn wcc(reader: &mut Reader<'_>) -> Wcc { + let before = reader.bool().unwrap().then(|| { + let size = reader.u64().unwrap(); + let mtime = (reader.u32().unwrap(), reader.u32().unwrap()); + let ctime = (reader.u32().unwrap(), reader.u32().unwrap()); + (size, mtime, ctime) + }); + Wcc { + before, + after: post_op(reader), + } +} +/// A `sattr3` to send. +#[derive(Clone, Copy, Debug, Default)] +pub struct Set { + pub mode: Option, + pub uid: Option, + pub gid: Option, + pub size: Option, + /// None: don't change; Some(None): server time; Some(Some(t)): client time. + pub mtime: Option>, +} +pub fn sattr(out: &mut Writer, set: Set) { + let field = |out: &mut Writer, value: Option| match value { + Some(value) => { + out.bool(true).u32(value); + } + None => { + out.bool(false); + } + }; + field(out, set.mode); + field(out, set.uid); + field(out, set.gid); + match set.size { + Some(size) => { + out.bool(true).u64(size); + } + None => { + out.bool(false); + } + } + out.u32(0); // atime: don't change + match set.mtime { + None => { + out.u32(0); + } + Some(None) => { + out.u32(1); + } + Some(Some((seconds, nanoseconds))) => { + out.u32(2).u32(seconds).u32(nanoseconds); + } + } +} +/// The results of CREATE, MKDIR and SYMLINK. +#[derive(Clone, Debug)] +pub struct Made { + pub handle: Vec, + pub attr: Attr, + pub directory: Wcc, +} +/// `createhow3`. +#[derive(Clone, Copy, Debug)] +pub enum How { + Unchecked(Set), + Guarded(Set), + Exclusive([u8; 8]), +} +impl Client { + fn made(&mut self, procedure: u32, arguments: &Writer) -> Result { + let (status, rest) = self.nfs(procedure, arguments); + let mut reader = Reader::new(&rest); + if status != OK { + return Err((status, wcc(&mut reader))); + } + assert!(reader.bool().unwrap(), "a created handle"); + let handle = reader.opaque(64).unwrap().to_vec(); + let attr = post_op(&mut reader).expect("created attributes"); + let directory = wcc(&mut reader); + Ok(Made { + handle, + attr, + directory, + }) + } + pub fn create(&mut self, directory: &[u8], name: &[u8], how: How) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + match how { + How::Unchecked(set) => { + arguments.u32(0); + sattr(&mut arguments, set); + } + How::Guarded(set) => { + arguments.u32(1); + sattr(&mut arguments, set); + } + How::Exclusive(verifier) => { + arguments.u32(2).fixed(&verifier); + } + } + self.made(8, &arguments).map_err(|(status, _)| status) + } + /// CREATE UNCHECKED with a mode, the common case. + pub fn touch(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.create(directory, name, How::Unchecked(set)) + } + pub fn mkdir(&mut self, directory: &[u8], name: &[u8], mode: u32) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + sattr(&mut arguments, set); + self.made(9, &arguments).map_err(|(status, _)| status) + } + pub fn symlink(&mut self, directory: &[u8], name: &[u8], target: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + sattr(&mut arguments, Set::default()); + arguments.opaque(target); + self.made(10, &arguments).map_err(|(status, _)| status) + } + pub fn mknod_fifo(&mut self, directory: &[u8], name: &[u8]) -> u32 { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name).u32(7); + sattr(&mut arguments, Set::default()); + self.nfs(11, &arguments).0 + } + /// SETATTR, optionally guarded by a ctime. + pub fn setattr( + &mut self, + handle: &[u8], + set: Set, + guard: Option<(u32, u32)>, + ) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(handle); + sattr(&mut arguments, set); + match guard { + Some((seconds, nanoseconds)) => { + arguments.bool(true).u32(seconds).u32(nanoseconds); + } + None => { + arguments.bool(false); + } + } + let (status, rest) = self.nfs(2, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err((status, found)) + } + } + /// WRITE: (count, committed, verifier, file wcc). + pub fn write( + &mut self, + handle: &[u8], + offset: u64, + data: &[u8], + ) -> Result<(u32, u32, [u8; 8], Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(handle) + .u64(offset) + .u32(data.len() as u32) + .u32(0) + .opaque(data); + let (status, rest) = self.nfs(7, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + let file = wcc(&mut reader); + let count = reader.u32().unwrap(); + let committed = reader.u32().unwrap(); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok((count, committed, verifier, file)) + } + /// WRITE of everything, in pieces of `piece` bytes. + pub fn write_all(&mut self, handle: &[u8], offset: u64, data: &[u8], piece: usize) { + let mut at = 0; + while at < data.len() { + let end = (at + piece).min(data.len()); + let (count, _, _, _) = self + .write(handle, offset + at as u64, &data[at..end]) + .unwrap(); + assert!(count > 0); + at += count as usize; + } + } + pub fn commit(&mut self, handle: &[u8]) -> Result<[u8; 8], u32> { + let mut arguments = Writer::new(); + arguments.opaque(handle).u64(0).u32(0); + let (status, rest) = self.nfs(21, &arguments); + if status != OK { + return Err(status); + } + let mut reader = Reader::new(&rest); + wcc(&mut reader); + let mut verifier = [0; 8]; + verifier.copy_from_slice(reader.fixed(8).unwrap()); + Ok(verifier) + } + /// REMOVE (`directory` false) or RMDIR. + pub fn unlink(&mut self, directory: &[u8], name: &[u8], rmdir: bool) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(if rmdir { 13 } else { 12 }, &arguments); + let found = wcc(&mut Reader::new(&rest)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn remove(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, false) + } + pub fn rmdir(&mut self, directory: &[u8], name: &[u8]) -> Result { + self.unlink(directory, name, true) + } + pub fn rename( + &mut self, + from: &[u8], + name: &[u8], + to: &[u8], + new_name: &[u8], + ) -> Result<(Wcc, Wcc), u32> { + let mut arguments = Writer::new(); + arguments + .opaque(from) + .opaque(name) + .opaque(to) + .opaque(new_name); + let (status, rest) = self.nfs(14, &arguments); + let mut reader = Reader::new(&rest); + let found = (wcc(&mut reader), wcc(&mut reader)); + if status == OK { + Ok(found) + } else { + Err(status) + } + } + pub fn link(&mut self, file: &[u8], directory: &[u8], name: &[u8]) -> Result { + let mut arguments = Writer::new(); + arguments.opaque(file).opaque(directory).opaque(name); + let (status, rest) = self.nfs(15, &arguments); + let mut reader = Reader::new(&rest); + let object = post_op(&mut reader); + wcc(&mut reader); + if status == OK { + Ok(object.expect("LINK returns the file's attributes")) + } else { + Err(status) + } + } + /// A path of names below `root`, looked up one by one. + pub fn walk_to(&mut self, root: &[u8], path: &str) -> Result<(Vec, Attr), u32> { + let mut handle = root.to_vec(); + let mut attr = self.getattr(root)?; + for name in path.split('/').filter(|name| !name.is_empty()) { + let (next, next_attr) = self.lookup(&handle, name.as_bytes())?; + handle = next; + attr = next_attr; + } + Ok((handle, attr)) + } +} diff --git a/core/crates/layerfs-nfs/tests/support/rig.rs b/core/crates/layerfs-nfs/tests/support/rig.rs new file mode 100644 index 0000000000..3c7b5ff4d1 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/support/rig.rs @@ -0,0 +1,295 @@ +//! A committed Branch from a host tree, bound to one Workspace and one engine +//! kernel mount, with no kernel connection: the NFS export is the only client. +use layerfs_daemon::{ + bootstrap::open_store, + store::{BindRequest, BoundWorkspace, Store}, + Mounted, Owner, OwnerClient, OwnerConfig, +}; +use layerfs_history::{ + BranchId, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryName, + LayerStackId, WorkspaceId, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{KernelMount, ProfileConfig}; +use layerfs_persistence::{ + Handles, PersistenceConfig, SqliteAcquisitionSchema, SqlitePersistenceProfile, +}; +use layerfs_project::{init, InitRequest}; +use layerfs_storage::{Storage, StoragePolicy}; +use layerfs_telemetry::timer::Timing; +use std::{ + net::TcpListener, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; + +const BINDING: &[u8] = b"layerfs-nfs-rig"; +const CURSOR: [u8; 32] = [71; 32]; +const CACHE: usize = 32 * 1024 * 1024; + +pub struct Rig { + pub directory: PathBuf, + store: Arc, + owner: Owner, + client: OwnerClient, + pub bound: Arc, + pub mount: KernelMount, + pub branch: BranchId, + /// Paths below the root that Init acquired. + pub entries: u64, + /// The reference bound exports of this rig are configured with. + pub lookups_held: usize, +} +impl Rig { + /// Initializes a Store from `source`, publishes Branch `main` at that + /// root, binds one Workspace and creates one engine kernel mount. + /// `directory` must not exist; everything the rig writes goes there. + pub fn new(source: &Path, directory: &Path, timeout: Duration) -> Self { + std::fs::create_dir_all(directory).unwrap(); + let sealed = directory.join("sealed.sqlite"); + let config = PersistenceConfig::sqlite(&sealed) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); + let handles = Handles::create( + config.clone(), + StoragePolicy::frozen_default(), + &HistoryCatalogConfig { + binding_key: BINDING.to_vec(), + cursor_key: CURSOR, + incarnation: 1, + }, + ) + .unwrap(); + let storage = Storage::new(handles.storage.clone()).unwrap(); + let initialized = Timing::disabled("nfs.rig.init", |timing| { + init( + &storage, + &handles.history, + InitRequest { + source, + acquisition: &handles.acquisition, + stack: LayerStackId::from_authority([72; 16]), + name: HistoryName::new("nfs").unwrap(), + scope_seed: [73; 32], + deadline: Instant::now() + timeout, + }, + timing, + ) + }) + .0 + .unwrap(); + let branch = BranchId::from_authority([74; 16]); + handles + .history + .fork(&ForkRequest { + stack: initialized.stack.id, + branch, + name: HistoryName::new("main").unwrap(), + source: ForkSource::Layer(initialized.stack.head_layer), + }) + .unwrap(); + drop(storage); + let sealed = handles.seal().unwrap(); + let store = open_store( + PersistenceConfig { + path: sealed.path.clone(), + ..config + }, + BINDING, + CURSOR, + 4, + CACHE, + Default::default(), + ) + .unwrap(); + let owner = Owner::start( + &directory.join("overlay"), + ProfileConfig::default(), + OwnerConfig::default(), + ) + .unwrap(); + let client = owner.client(); + let bound = Arc::new( + store + .bind( + client.clone(), + BindRequest { + branch, + workspace: WorkspaceId::from_authority([75; 32]).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = client.kernel_mount(bound.route(), root).unwrap(); + Self { + directory: directory.to_owned(), + store, + owner, + client, + bound, + mount, + branch, + entries: initialized.entries, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } + /// Serves this rig's engine mount read-only on `listener`. + pub fn serve(&self, listener: TcpListener, uid: u32, gid: u32) -> Server { + self.export(listener, uid, gid, true) + } + /// Serves this rig's engine mount on `listener`, writable or not. + pub fn export(&self, listener: TcpListener, uid: u32, gid: u32, read_only: bool) -> Server { + self.export_services( + listener, + Arc::new(Mounted(self.bound.clone())), + uid, + gid, + read_only, + ) + } + /// Serves this rig's engine mount over `services` (a wrapper of the + /// rig's own ports, for example). + pub fn export_services( + &self, + listener: TcpListener, + services: Arc, + uid: u32, + gid: u32, + read_only: bool, + ) -> Server { + Server::start( + listener, + services, + self.mount, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// A daemon control Service over this rig's Store and overlay owner whose + /// Attach serves NFS (the native route's mount entry). + pub fn service( + &self, + export: Arc, + ) -> Arc { + Arc::new(layerfs_daemon::control::Service::with_export( + self.store.clone(), + &self.owner, + export, + )) + } + /// The product Commit of everything the Workspace published locally. + pub fn commit(&self) -> layerfs_daemon::store::CommitSuccess { + self.bound + .commit_captured(&layerfs_daemon::store::CommitOptions::default()) + .unwrap_or_else(|failure| panic!("Commit: {:?}", failure.error)) + } + /// A second Workspace bound to Branch `main` as it is now, with its own + /// engine mount: an independent view of what was committed. + pub fn view(&self, workspace: [u8; 32]) -> (Arc, KernelMount) { + let bound = Arc::new( + self.store + .bind( + self.client.clone(), + BindRequest { + branch: self.branch, + workspace: WorkspaceId::from_authority(workspace).unwrap(), + }, + ) + .unwrap() + .workspace, + ); + let root = bound + .operation() + .unwrap() + .workspace() + .base() + .unwrap() + .root() + .root_inode() + .serial(); + let mount = self.client.kernel_mount(bound.route(), root).unwrap(); + (bound, mount) + } + /// Serves a view read-only. + pub fn serve_view( + &self, + listener: TcpListener, + view: &(Arc, KernelMount), + uid: u32, + gid: u32, + ) -> Server { + Server::start( + listener, + Arc::new(Mounted(view.0.clone())), + view.1, + ExportConfig { + path: "/".into(), + uid, + gid, + read_only: true, + refuse_apple_double: true, + lookups_held: self.lookups_held, + }, + ) + .unwrap() + } + /// Revokes and closes a view. + pub fn close_view(&self, view: (Arc, KernelMount)) { + self.client.kernel_revoke(view.0.route(), view.1).unwrap(); + view.0.close().unwrap(); + } + /// A second engine mount of the same Workspace, as a restarted export + /// would create: handles of the first are stale under it. + pub fn remount(&mut self) { + self.client + .kernel_revoke(self.bound.route(), self.mount) + .unwrap(); + self.mount = self + .client + .kernel_mount(self.bound.route(), self.mount.root_serial()) + .unwrap(); + } + /// Revokes the engine mount, closes the Workspace and stops the owner. + /// Returns the Store readers still open or quarantined afterwards. + pub fn stop(self, served: Released) -> (Released, u64) { + let Self { + directory, + store, + owner, + client, + bound, + mount, + .. + } = self; + client.kernel_revoke(bound.route(), mount).unwrap(); + bound.close().unwrap(); + drop(bound); + // A closed Workspace keeps no Store reader and none was quarantined; + // the Store's own control reader stays open. + let work = store.read_work(); + assert!(!work.poisoned && work.opening_failures == 0, "{work:?}"); + let outstanding = work.readers.saturating_sub(1) + work.quarantined; + owner.stop().unwrap(); + drop(store); + std::fs::remove_dir_all(directory).unwrap(); + (served, outstanding as u64) + } +} diff --git a/core/crates/layerfs-nfs/tests/support/spy.rs b/core/crates/layerfs-nfs/tests/support/spy.rs new file mode 100644 index 0000000000..0f329a20f3 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/support/spy.rs @@ -0,0 +1,199 @@ +//! A wrapper of the rig's request ports that watches the publication +//! ticket contract from outside the export: every `reply_attempted` waits +//! until the test's client has received the reply of the call that made the +//! publication, +//! and records whether that happened (a release before the reply reached the +//! wire would wait forever on a synchronous client, so it times out and is +//! counted as a violation). It can also refuse serial reservations, as a +//! contended allocator does. +use super::client::Wire; +use layerfs_fuse::ports::{ + MountServices, RequestServices, ServiceError, ServiceFuture, ServiceReply, +}; +use layerfs_overlay::{KernelCookieOffer, KernelMount, Publication}; +use layerfs_workspace::{ + BaseView, KernelDirectoryWindow, KernelMutationOutcome, KernelReadOperation, KernelReadOutcome, + KernelVisitRequest, KernelWindow, VisitFacts, WriteData, +}; +use std::{ + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +/// What the spy saw. +#[derive(Default)] +pub struct Watch { + pub wire: Arc, + /// Releases made after the reply was received, and before it. + pub released_after: AtomicU64, + pub released_before: AtomicU64, + /// Publishing visits (`Applied` outcomes) and reservations refused. + pub published: AtomicU64, + /// Reservations still to refuse. + pub refuse_serials: AtomicU64, + pub refused: AtomicU64, + pub log: Mutex>, + /// Each publication and the call count when it was made: the client was + /// then waiting for exactly that call's reply. + pub owed: Mutex>, +} +pub struct Spy { + pub inner: Arc, + pub watch: Arc, +} +impl MountServices for Spy { + fn request( + &self, + fence: &layerfs_fuse::ports::Fence, + ) -> Result, ServiceError> { + Ok(Arc::new(Request { + inner: self.inner.request(fence)?, + watch: self.watch.clone(), + })) + } +} +struct Request { + inner: Arc, + watch: Arc, +} +impl RequestServices for Request { + fn directory_visit( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + offset: u64, + after: Option>, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .directory_visit(mount, serial, handle, offset, after) + } + fn publish_cookies( + &self, + offer: KernelCookieOffer, + names: Vec>, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.publish_cookies(offer, names) + } + fn close_directory( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_directory(mount, serial, handle) + } + fn read_visit( + &self, + mount: KernelMount, + serial: u64, + handle: Option, + offset: u64, + length: u32, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner.read_visit(mount, serial, handle, offset, length) + } + fn close_file( + &self, + mount: KernelMount, + serial: u64, + handle: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.close_file(mount, serial, handle) + } + fn provider_failure(&self) -> Result, ServiceError> { + self.inner.provider_failure() + } + fn failed_base_read(&self, step: ServiceError) -> ServiceError { + self.inner.failed_base_read(step) + } + fn observe_visit( + &self, + mount: KernelMount, + request: u64, + serial: u64, + handle: Option, + operation: KernelReadOperation, + facts: Arc, + ) -> ServiceFuture<'_, ServiceReply>> { + self.inner + .observe_visit(mount, request, serial, handle, operation, facts) + } + fn base(&self) -> ServiceFuture<'_, BaseView> { + self.inner.base() + } + fn mutate_visit( + &self, + request: KernelVisitRequest, + ) -> ServiceFuture<'_, ServiceReply>> { + let watch = self.watch.clone(); + let future = self.inner.mutate_visit(request); + Box::pin(async move { + let reply = future.await?; + if let Ok(layerfs_workspace::JobOutcome::Applied { publication, .. }) = + &reply.get().result + { + watch.published.fetch_add(1, Ordering::AcqRel); + let sent = watch.wire.counts.lock().unwrap().0; + watch.owed.lock().unwrap().push((*publication, sent)); + } + Ok(reply) + }) + } + fn recognize(&self, serial: u64, offset: u64, data: WriteData) -> ServiceFuture<'_, WriteData> { + self.inner.recognize(serial, offset, data) + } + fn forget( + &self, + mount: KernelMount, + serial: u64, + count: u64, + ) -> ServiceFuture<'_, ServiceReply<()>> { + self.inner.forget(mount, serial, count) + } + fn reserve_serial(&self) -> Result, ServiceError> { + let refuse = self + .watch + .refuse_serials + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |left| { + left.checked_sub(1) + }) + .is_ok(); + if refuse { + self.watch.refused.fetch_add(1, Ordering::AcqRel); + return Ok(None); + } + self.inner.reserve_serial() + } + fn reply_attempted(&self, publication: Publication) -> ServiceFuture<'_, ServiceReply<()>> { + let wire = &self.watch.wire; + let owed = { + let mut owed = self.watch.owed.lock().unwrap(); + let at = owed.iter().position(|(made, _)| *made == publication); + at.map_or(u64::MAX, |at| owed.remove(at).1) + }; + let counts = wire.counts.lock().unwrap(); + // The reply of the call that published this ticket must reach the + // client before the release, whatever the client sends next. + let (counts, timeout) = wire + .changed + .wait_timeout_while(counts, Duration::from_secs(5), |(_, received)| { + *received < owed + }) + .unwrap(); + let (sent, received) = *counts; + drop(counts); + if timeout.timed_out() { + self.watch.released_before.fetch_add(1, Ordering::AcqRel); + self.watch.log.lock().unwrap().push(format!( + "released before reply: sent {sent} received {received}" + )); + } else { + self.watch.released_after.fetch_add(1, Ordering::AcqRel); + } + self.inner.reply_attempted(publication) + } +} diff --git a/core/crates/layerfs-nfs/tests/support/tree.rs b/core/crates/layerfs-nfs/tests/support/tree.rs new file mode 100644 index 0000000000..e1bb648bc8 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/support/tree.rs @@ -0,0 +1,136 @@ +//! A host tree as an oracle, and a protocol walk that checks an export +//! against it: every path, kind, mode, byte, link target, listing, `..`, +//! directory link count and the identity behind hard links. +use super::client::Client; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +/// What the host tree holds, read before the export runs. +#[derive(Debug)] +pub enum Node { + File { mode: u32, bytes: Vec, ino: u64 }, + Directory { mode: u32, names: BTreeSet> }, + Link { target: Vec }, +} +pub fn oracle(root: &Path) -> BTreeMap { + use std::os::unix::ffi::OsStrExt; + let mut nodes = BTreeMap::new(); + let mut pending = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + let path = root.join(&relative); + let meta = fs::symlink_metadata(&path).unwrap(); + let node = if meta.file_type().is_symlink() { + Node::Link { + target: fs::read_link(&path) + .unwrap() + .as_os_str() + .as_bytes() + .to_vec(), + } + } else if meta.is_dir() { + let mut names = BTreeSet::new(); + for entry in fs::read_dir(&path).unwrap() { + let name = entry.unwrap().file_name(); + pending.push(relative.join(&name)); + names.insert(name.as_bytes().to_vec()); + } + Node::Directory { + mode: meta.mode() & 0o7777, + names, + } + } else { + Node::File { + mode: meta.mode() & 0o7777, + bytes: fs::read(&path).unwrap(), + ino: meta.ino(), + } + }; + nodes.insert(relative, node); + } + nodes +} + +/// Walks the export from `handle` and checks every node against the oracle. +/// Returns (paths, regular bytes read) and the fileid of every host inode. +pub fn walk( + client: &mut Client, + oracle: &BTreeMap, + budget: u32, + owner: (u32, u32), +) -> (usize, usize, BTreeMap>) { + use std::os::unix::ffi::OsStrExt; + let root = client.mnt("/"); + let root_attr = client.getattr(&root).unwrap(); + let mut identities: BTreeMap> = BTreeMap::new(); + let (mut paths, mut bytes) = (0, 0); + let mut pending = vec![(PathBuf::new(), root, root_attr.clone(), root_attr.fileid)]; + while let Some((relative, handle, attr, parent)) = pending.pop() { + paths += 1; + assert_eq!((attr.uid, attr.gid), owner, "{relative:?}"); + match &oracle[&relative] { + Node::File { + mode, + bytes: expected, + ino, + } => { + assert_eq!(attr.kind, 1, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + assert_eq!(attr.size, expected.len() as u64, "{relative:?} size"); + let actual = client.read_all(&handle, 128 * 1024); + assert_eq!(&actual, expected, "{relative:?} bytes"); + bytes += actual.len(); + identities.entry(*ino).or_default().insert(attr.fileid); + } + Node::Link { target } => { + assert_eq!(attr.kind, 5, "{relative:?}"); + assert_eq!(&client.readlink(&handle).unwrap(), target, "{relative:?}"); + } + Node::Directory { mode, names } => { + assert_eq!(attr.kind, 2, "{relative:?}"); + assert_eq!(attr.mode, *mode, "{relative:?} mode"); + let (entries, _) = client.list(&handle, budget, false); + let mut listed = BTreeSet::new(); + for entry in &entries { + assert!(listed.insert(entry.name.clone()), "{relative:?} duplicate"); + } + let dot = entries.iter().find(|entry| entry.name == b".").unwrap(); + let dotdot = entries.iter().find(|entry| entry.name == b".."); + assert_eq!(dot.fileid, attr.fileid, "{relative:?} ."); + if let Some(dotdot) = dotdot { + assert_eq!(dotdot.fileid, parent, "{relative:?} .."); + } + listed.remove(&b"."[..]); + listed.remove(&b".."[..]); + assert_eq!(&listed, names, "{relative:?} complete listing"); + let (up, up_attr) = client.lookup(&handle, b"..").unwrap(); + assert_eq!(up_attr.fileid, parent, "{relative:?} LOOKUP .."); + drop(up); + // Subdirectory link counts: two plus child directories. + let subdirectories = names + .iter() + .filter(|name| { + matches!( + oracle.get(&relative.join(std::ffi::OsStr::from_bytes(name))), + Some(Node::Directory { .. }) + ) + }) + .count(); + assert_eq!( + attr.nlink as usize, + subdirectories + 2, + "{relative:?} nlink" + ); + for name in names { + let (child, child_attr) = client.lookup(&handle, name).unwrap(); + let child_path = relative.join(std::ffi::OsStr::from_bytes(name)); + pending.push((child_path, child, child_attr, attr.fileid)); + } + } + } + } + (paths, bytes, identities) +} diff --git a/core/crates/layerfs-nfs/tests/write.rs b/core/crates/layerfs-nfs/tests/write.rs new file mode 100644 index 0000000000..4c48aae822 --- /dev/null +++ b/core/crates/layerfs-nfs/tests/write.rs @@ -0,0 +1,590 @@ +//! A writable Workspace served over loopback NFSv3: every changing +//! procedure and its error statuses, the publication-ticket contract (each +//! ticket released only after its reply was received, contended serial +//! reservations), the export's own handling of client silly renames and +//! AppleDouble names, and a product Commit whose tree equals what the same +//! operations made of a host directory. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/spy.rs"] +mod spy; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{ + Client, How, Set, ACCES, EXIST, INVAL, ISDIR, JUKEBOX, NAMETOOLONG, NOENT, NOTDIR, NOTEMPTY, + NOTSUPP, NOT_SYNC, OK, PERM, STALE, +}; +use layerfs_daemon::Mounted; +use std::{ + fs, + net::TcpListener, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; +const FILE_SYNC: u32 = 2; +const LINUX_SILLY: &[u8] = b".nfs00000000000012ab00000001"; +const MAC_SILLY: &[u8] = b".nfs.20051025.b896"; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-w{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("src/nested")).unwrap(); + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write(root.join("src/lib.rs"), b"pub fn answer() -> u32 { 42 }\n").unwrap(); + fs::write(root.join("src/nested/deep.txt"), b"deep\n").unwrap(); + fs::write(root.join("docs/guide.md"), b"guide\n").unwrap(); + fs::hard_link(root.join("src/lib.rs"), root.join("src/alias.rs")).unwrap(); + std::os::unix::fs::symlink("src/lib.rs", root.join("link")).unwrap(); + fs::write(root.join("run.sh"), b"#!/bin/sh\n").unwrap(); + fs::set_permissions(root.join("run.sh"), fs::Permissions::from_mode(0o755)).unwrap(); +} +fn pattern(seed: u32, length: usize) -> Vec { + (0..length) + .map(|n| (n as u32).wrapping_mul(31).wrapping_add(seed * 97) as u8) + .collect() +} + +#[test] +fn every_changing_procedure_and_its_statuses() { + let base = temp("procedures"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + + // CREATE: UNCHECKED, GUARDED over an existing name, UNCHECKED again. + let made = c.touch(&root, b"new.txt", 0o640).unwrap(); + assert_eq!( + (made.attr.kind, made.attr.mode, made.attr.size), + (1, 0o640, 0) + ); + assert_eq!((made.attr.uid, made.attr.gid), (UID, GID)); + assert!(made.directory.before.is_some() && made.directory.after.is_some()); + let guarded = How::Guarded(Set::default()); + assert_eq!(c.create(&root, b"new.txt", guarded).map(|_| ()), Err(EXIST)); + let again = c.touch(&root, b"new.txt", 0o600).unwrap(); + assert_eq!(again.attr.fileid, made.attr.fileid); + assert_eq!( + again.attr.mode, 0o640, + "UNCHECKED keeps an existing file's mode" + ); + let file = made.handle.clone(); + + // WRITE: FILE_SYNC with this export's verifier; a hole reads as zeros; + // a call larger than one engine window is split and fully written. + let (count, committed, verifier, wcc) = c.write(&file, 0, b"hello").unwrap(); + assert_eq!( + (count, committed, verifier), + (5, FILE_SYNC, server.verifier()) + ); + assert_eq!(wcc.after.unwrap().size, 5); + let big = pattern(1, 300_000); + let (count, _, _, wcc) = c.write(&file, 10, &big).unwrap(); + assert_eq!(count as usize, big.len()); + assert_eq!(wcc.after.unwrap().size, 300_010); + let mut expected = b"hello\0\0\0\0\0".to_vec(); + expected.extend_from_slice(&big); + assert_eq!(c.read_all(&file, 128 * 1024), expected); + assert_eq!(c.commit(&file), Ok(server.verifier())); + + // SETATTR: truncate, mode, client mtime, owner no-op and refusal, guard. + let set = |size, mode| Set { + size, + mode, + ..Set::default() + }; + let after = c + .setattr(&file, set(Some(3), None), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.size, 3); + assert_eq!(c.read_all(&file, 4096), b"hel"); + let after = c + .setattr(&file, set(None, Some(0o600)), None) + .unwrap() + .after + .unwrap(); + assert_eq!(after.mode, 0o600); + let stamp = Set { + mtime: Some(Some((1_000_000_000, 5))), + ..Set::default() + }; + let after = c.setattr(&file, stamp, None).unwrap().after.unwrap(); + assert_eq!(after.mtime, (1_000_000_000, 5)); + let same = Set { + uid: Some(UID), + gid: Some(GID), + ..Set::default() + }; + assert!( + c.setattr(&file, same, None).is_ok(), + "chown to the same identity" + ); + let other = Set { + uid: Some(0), + ..Set::default() + }; + assert_eq!( + c.setattr(&file, other, None).map(|_| ()).unwrap_err().0, + PERM + ); + let wrong = c.setattr(&file, set(Some(1), None), Some((7, 7))); + assert_eq!(wrong.map(|_| ()).unwrap_err().0, NOT_SYNC); + assert_eq!( + c.getattr(&file).unwrap().size, + 3, + "a failed guard changes nothing" + ); + let right = c.setattr(&file, set(Some(1), None), Some(after.mtime)); + assert_eq!(right.unwrap().after.unwrap().size, 1); + + // MKDIR, SYMLINK, MKNOD. + let dir = c.mkdir(&root, b"d", 0o750).unwrap(); + assert_eq!( + (dir.attr.kind, dir.attr.mode, dir.attr.nlink), + (2, 0o750, 2) + ); + assert_eq!(c.mkdir(&root, b"d", 0o750).map(|_| ()), Err(EXIST)); + let (_, up) = c.lookup(&dir.handle, b"..").unwrap(); + assert_eq!(up.fileid, c.getattr(&root).unwrap().fileid); + let link = c.symlink(&root, b"s", b"new.txt").unwrap(); + assert_eq!(link.attr.kind, 5); + assert_eq!(c.readlink(&link.handle).unwrap(), b"new.txt"); + let long = vec![b'x'; 4096]; + assert_eq!( + c.symlink(&root, b"long", &long).map(|_| ()), + Err(NAMETOOLONG) + ); + assert_eq!(c.mknod_fifo(&root, b"fifo"), NOTSUPP); + assert_eq!(c.lookup(&root, b"fifo").map(|_| ()), Err(NOENT)); + + // LINK: one more name and link count; directories and taken names refused. + let linked = c.link(&file, &dir.handle, b"alias").unwrap(); + assert_eq!((linked.fileid, linked.nlink), (made.attr.fileid, 2)); + assert_ne!(c.link(&dir.handle, &root, b"dirlink").map(|_| ()), Ok(())); + assert_eq!(c.link(&file, &root, b"README.md").map(|_| ()), Err(EXIST)); + + // RENAME: across directories, of a directory, replacing, into itself. + c.rename(&root, b"new.txt", &dir.handle, b"moved").unwrap(); + assert_eq!(c.lookup(&root, b"new.txt").map(|_| ()), Err(NOENT)); + let (_, moved) = c.lookup(&dir.handle, b"moved").unwrap(); + assert_eq!(moved.fileid, made.attr.fileid); + let sub = c.mkdir(&dir.handle, b"sub", 0o755).unwrap(); + c.rename(&dir.handle, b"sub", &root, b"top").unwrap(); + let (_, up) = c.lookup(&sub.handle, b"..").unwrap(); + assert_eq!( + up.fileid, + c.getattr(&root).unwrap().fileid, + ".. follows a move" + ); + assert_eq!(c.getattr(&dir.handle).unwrap().nlink, 2); + assert_ne!( + c.rename(&root, b"d", &dir.handle, b"inside").map(|_| ()), + Ok(()) + ); + let (readme, readme_attr) = c.lookup(&root, b"README.md").unwrap(); + c.rename(&root, b"run.sh", &root, b"README.md").unwrap(); + assert_eq!(c.getattr(&readme), Err(STALE), "the replaced inode is gone"); + let (_, replaced) = c.lookup(&root, b"README.md").unwrap(); + assert_ne!(replaced.fileid, readme_attr.fileid); + assert_eq!(replaced.mode, 0o755); + let missing = c.rename(&root, b"absent", &root, b"x").map(|_| ()); + assert_eq!(missing, Err(NOENT)); + + // REMOVE and RMDIR. + c.remove(&dir.handle, b"alias").unwrap(); + assert_eq!( + c.getattr(&file).unwrap().nlink, + 1, + "the other name keeps it" + ); + assert_eq!(c.remove(&root, b"absent").map(|_| ()), Err(NOENT)); + assert_eq!(c.remove(&root, b"d").map(|_| ()), Err(ISDIR)); + assert_eq!(c.rmdir(&root, b"d").map(|_| ()), Err(NOTEMPTY)); + assert_eq!(c.rmdir(&root, b"README.md").map(|_| ()), Err(NOTDIR)); + assert_eq!(c.rmdir(&root, b".").map(|_| ()), Err(INVAL)); + c.remove(&dir.handle, b"moved").unwrap(); + assert_eq!(c.getattr(&file), Err(STALE), "its last name is gone"); + c.rmdir(&root, b"d").unwrap(); + assert_eq!(c.getattr(&dir.handle), Err(STALE)); + + // Names the export refuses or the protocol forbids. + assert_eq!(c.touch(&root, b"._new.txt", 0o644).map(|_| ()), Err(ACCES)); + assert_eq!(c.mkdir(&root, b"._d", 0o755).map(|_| ()), Err(ACCES)); + let refused = c.rename(&root, b"s", &root, b"._s").map(|_| ()); + assert_eq!(refused, Err(ACCES)); + assert_eq!(c.touch(&root, b".", 0o644).map(|_| ()), Err(EXIST)); + let long_name = vec![b'n'; 256]; + assert_eq!( + c.touch(&root, &long_name, 0o644).map(|_| ()), + Err(NAMETOOLONG) + ); + + // EXCLUSIVE: a retransmission finds its own file; another verifier does not. + let first = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + let again = c + .create(&root, b"excl", How::Exclusive(*b"verifier")) + .unwrap(); + assert_eq!(first.attr.fileid, again.attr.fileid); + let other = c.create(&root, b"excl", How::Exclusive(*b"another!")); + assert_eq!(other.map(|_| ()), Err(EXIST)); + + // ACCESS on a writable export grants changes by mode. + let (guide, _) = c.walk_to(&root, "docs/guide.md").unwrap(); + assert_eq!(c.access(&guide, 0x3f), Ok(0x01 | 0x04 | 0x08)); + let (docs, _) = c.lookup(&root, b"docs").unwrap(); + assert_eq!(c.access(&docs, 0x3f), Ok(0x01 | 0x02 | 0x04 | 0x08 | 0x10)); + + // Every serial handed out holds one reference: removed ones gave theirs back. + let held = server.held(); + assert_eq!((held.silly, held.retained, held.names_refused), (0, 0, 3)); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn silly_renames_stay_out_of_the_workspace() { + let base = temp("silly"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + for (silly, name) in [(LINUX_SILLY, &b"open.bin"[..]), (MAC_SILLY, b"open2.bin")] { + let made = c.touch(&root, name, 0o644).unwrap(); + c.write(&made.handle, 0, b"still open").unwrap(); + assert_eq!(c.lookup(&root, silly).map(|_| ()), Err(NOENT)); + c.rename(&root, name, &root, silly).unwrap(); + assert_eq!(c.lookup(&root, name).map(|_| ()), Err(NOENT)); + let (handle, attr) = c.lookup(&root, silly).unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 10); + // The open file stays readable, writable and truncatable. + c.write(&handle, 10, b"!").unwrap(); + assert_eq!(c.read_all(&handle, 4096), b"still open!"); + let truncate = Set { + size: Some(5), + ..Set::default() + }; + assert_eq!( + c.setattr(&handle, truncate, None) + .unwrap() + .after + .unwrap() + .size, + 5 + ); + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| e.name != silly && e.name != name)); + assert_eq!(server.held().silly, 1); + c.remove(&root, silly).unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().silly, 0); + } + // A user's own names that only resemble the patterns are ordinary. + let made = c.touch(&root, b".nfsrc", 0o644).unwrap(); + c.rename(&root, b".nfsrc", &root, b".nfs-old").unwrap(); + assert!(c.lookup(&root, b".nfs-old").is_ok()); + drop(made); + // A silly name still held at Commit is not captured; neither is the + // name it replaced. + let kept = c.touch(&root, b"kept.bin", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + c.rename(&root, b"kept.bin", &root, LINUX_SILLY).unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([76; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b".nfs-old".to_vec())); + assert!(!names + .iter() + .any(|n| n.starts_with(b".nfs0") || n == b"kept.bin")); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + // Release gives the silly inode back. + drop(c); + let released = server.stop(); + assert_eq!((released.silly, released.silly_renamed), (1, 3)); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn tickets_are_released_only_after_the_reply() { + let base = temp("tickets"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let watch = Arc::new(spy::Watch::default()); + let services = Arc::new(spy::Spy { + inner: Arc::new(Mounted(rig.bound.clone())), + watch: watch.clone(), + }); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let server = rig.export_services(listener, services, UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + c.wire = Some(watch.wire.clone()); + let root = c.mnt("/"); + + let made = c.touch(&root, b"f", 0o644).unwrap(); + // 300 000 bytes are three engine windows: three publications, three + // tickets, all released after the one reply. + c.write(&made.handle, 0, &pattern(2, 300_000)).unwrap(); + let truncate = Set { + size: Some(10), + ..Set::default() + }; + c.setattr(&made.handle, truncate, None).unwrap(); + c.mkdir(&root, b"dir", 0o755).unwrap(); + c.rename(&root, b"f", &root, b"g").unwrap(); + c.remove(&root, b"g").unwrap(); + let published = watch.published.load(Ordering::Acquire); + assert_eq!(published, 1 + 3 + 1 + 1 + 1 + 1); + // The last call's tickets are released just after its reply arrived. + let settled = std::time::Instant::now() + Duration::from_secs(5); + while watch.released_after.load(Ordering::Acquire) < published + && std::time::Instant::now() < settled + { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(watch.released_after.load(Ordering::Acquire), published); + assert_eq!( + watch.released_before.load(Ordering::Acquire), + 0, + "{:?}", + watch.log.lock().unwrap() + ); + + // A contended allocator: nothing is reserved and nothing created; the + // client is told to retry later, and the retry succeeds. + watch.refuse_serials.store(1, Ordering::Release); + assert_eq!(c.touch(&root, b"later", 0o644).map(|_| ()), Err(JUKEBOX)); + assert_eq!(watch.refused.load(Ordering::Acquire), 1); + assert_eq!(c.lookup(&root, b"later").map(|_| ()), Err(NOENT)); + assert_eq!(watch.published.load(Ordering::Acquire), published); + assert!(c.touch(&root, b"later", 0o644).is_ok()); + + // No ticket is left pending: the Workspace captures and commits. + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + assert_eq!(server.held().retained, 0); + drop(c); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + +/// The same operations, over NFS and on a host directory. +struct Twin<'a> { + client: &'a mut Client, + root: Vec, + host: PathBuf, +} +impl Twin<'_> { + fn handle(&mut self, path: &str) -> Vec { + self.client.walk_to(&self.root, path).unwrap().0 + } + fn split(path: &str) -> (&str, &str) { + path.rsplit_once('/').unwrap_or(("", path)) + } + fn mkdir(&mut self, path: &str, mode: u32) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client.mkdir(&parent, name.as_bytes(), mode).unwrap(); + fs::create_dir(self.host.join(path)).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn write(&mut self, path: &str, offset: u64, data: &[u8]) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let handle = match self.client.lookup(&parent, name.as_bytes()) { + Ok((handle, _)) => handle, + Err(_) => { + let made = self.client.touch(&parent, name.as_bytes(), 0o644).unwrap(); + fs::write(self.host.join(path), b"").unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(0o644)) + .unwrap(); + made.handle + } + }; + self.client.write_all(&handle, offset, data, 64 * 1024); + use std::os::unix::fs::FileExt; + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.write_all_at(data, offset).unwrap(); + } + fn truncate(&mut self, path: &str, size: u64) { + let handle = self.handle(path); + let set = Set { + size: Some(size), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + let file = fs::OpenOptions::new() + .write(true) + .open(self.host.join(path)) + .unwrap(); + file.set_len(size).unwrap(); + } + fn chmod(&mut self, path: &str, mode: u32) { + let handle = self.handle(path); + let set = Set { + mode: Some(mode), + ..Set::default() + }; + self.client.setattr(&handle, set, None).unwrap(); + fs::set_permissions(self.host.join(path), fs::Permissions::from_mode(mode)).unwrap(); + } + fn rename(&mut self, from: &str, to: &str) { + let (a, an) = Self::split(from); + let (b, bn) = Self::split(to); + let (a, b) = (self.handle(a), self.handle(b)); + self.client + .rename(&a, an.as_bytes(), &b, bn.as_bytes()) + .unwrap(); + fs::rename(self.host.join(from), self.host.join(to)).unwrap(); + } + fn link(&mut self, existing: &str, new: &str) { + let file = self.handle(existing); + let (parent, name) = Self::split(new); + let parent = self.handle(parent); + self.client.link(&file, &parent, name.as_bytes()).unwrap(); + fs::hard_link(self.host.join(existing), self.host.join(new)).unwrap(); + } + fn symlink(&mut self, path: &str, target: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .symlink(&parent, name.as_bytes(), target.as_bytes()) + .unwrap(); + std::os::unix::fs::symlink(target, self.host.join(path)).unwrap(); + } + fn remove(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + let directory = fs::symlink_metadata(self.host.join(path)).unwrap().is_dir(); + self.client + .unlink(&parent, name.as_bytes(), directory) + .unwrap(); + if directory { + fs::remove_dir(self.host.join(path)).unwrap(); + } else { + fs::remove_file(self.host.join(path)).unwrap(); + } + } + /// A file the client still has open when it is unlinked: renamed to a + /// silly name over NFS, simply gone on the host. + fn unlink_open(&mut self, path: &str) { + let (parent, name) = Self::split(path); + let parent = self.handle(parent); + self.client + .rename(&parent, name.as_bytes(), &parent, LINUX_SILLY) + .unwrap(); + fs::remove_file(self.host.join(path)).unwrap(); + } +} + +#[test] +fn a_commit_holds_what_the_same_operations_make_of_a_host_tree() { + let base = temp("commit"); + let tree = base.join("tree"); + source(&tree); + let host = base.join("host"); + fs::create_dir_all(&host).unwrap(); + assert!(std::process::Command::new("cp") + .arg("-a") + .arg(format!("{}/.", tree.display())) + .arg(&host) + .status() + .unwrap() + .success()); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut client = Client::connect(server.local_addr(), UID, GID); + let root = client.mnt("/"); + let mut t = Twin { + client: &mut client, + root, + host: host.clone(), + }; + t.mkdir("a", 0o755); + t.mkdir("a/b", 0o700); + t.mkdir("a/b/c", 0o755); + t.write("a/b/c/big.bin", 0, &pattern(3, 400_000)); + t.write("a/b/c/big.bin", 123_456, &pattern(4, 70_000)); + t.truncate("a/b/c/big.bin", 350_001); + t.write("a/empty", 0, b""); + t.write("a/hello.txt", 0, b"hello\n"); + t.chmod("a/hello.txt", 0o755); + t.write("README.md", 0, b"# rewritten\n"); + t.truncate("src/nested/deep.txt", 0); + t.rename("a/hello.txt", "a/b/hello.txt"); + t.rename("a/b/c", "c-moved"); + t.link("a/b/hello.txt", "hello-alias"); + t.symlink("a/to-hello", "b/hello.txt"); + t.remove("src/alias.rs"); + t.remove("docs/guide.md"); + t.remove("docs"); + t.write("tmp-old", 0, b"old\n"); + t.write("tmp-new", 0, b"new\n"); + t.rename("tmp-new", "tmp-old"); + t.write("unicod\u{e9}.txt", 0, "caf\u{e9}\n".as_bytes()); + t.write("open-at-commit", 0, b"open\n"); + t.unlink_open("open-at-commit"); + drop(t); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + + // An independent Workspace bound to the committed Branch, read over NFS, + // equals the host tree path for path and byte for byte. + let expected = tree::oracle(&host); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let (paths, _, identities) = tree::walk(&mut v, &expected, 4096, (UID, GID)); + assert_eq!(paths, expected.len()); + let hello = fs::metadata(host.join("hello-alias")).unwrap(); + use std::os::unix::fs::MetadataExt; + assert_eq!(identities[&hello.ino()].len(), 1, "both names, one inode"); + drop(v); + viewer.stop(); + rig.close_view(view); + assert_eq!(OK, 0); + drop(client); + let released = server.stop(); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index 7b761f3ce9..b927b00c53 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -23,7 +23,7 @@ "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", } -UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") +UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", "layerfs-content": "#![forbid(unsafe_code)]", @@ -34,6 +34,7 @@ "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", + "layerfs-nfs": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", @@ -53,6 +54,8 @@ "layerfs-overlay": set(), "layerfs-workspace": {"layerfs-content", "layerfs-overlay", "layerfs-telemetry"}, "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, + # The NFS export reuses the kernel ports and operations; never daemon. + "layerfs-nfs": {"layerfs-content", "layerfs-overlay", "layerfs-workspace", "layerfs-fuse"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. "layerfs-store-service": {"layerfs-overlay", "layerfs-workspace", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, @@ -66,7 +69,7 @@ DOMAIN_CRATES = {"layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-project"} ENGINES_AND_CLUSTER2 = { "layerfs-s3", "layerfs-metadata", "layerfs-persistence", "layerfs-overlay", "layerfs-workspace", - "layerfs-store-service", "layerfs-fuse", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", + "layerfs-store-service", "layerfs-fuse", "layerfs-nfs", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", "layerfs-server", "layerfs-sdk", "layerfs-api", "layerfs-api-core", } From 757e4480cc995d98411c2495a96a358b06e3e471 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Fri, 23 Oct 2026 00:30:00 +0800 Subject: [PATCH 42/64] feat(daemon): NFS as a kernel mount entry beside FUSE (LAYERFS_KERNEL=nfs) With LAYERFS_KERNEL=nfs (always on macOS) the daemon serves each attached Workspace as a layerfs-nfs export on 127.0.0.1:0 and runs a mount helper to mount it on the Workspace's directory, instead of opening /dev/fuse. Control requests, their replies and records, and every engine and Store step are the same as on the FUSE route; the bridge wire format does not change (an NFS receipt carries abi_major 0 and the export port as mount_id). - control/export.rs: Attach (export, helper mount, then Ready), Commit flush before capture (helper, default sync(2)), Unmount with reversible busy, ForceUnmount, retained custody (ExportKept), status and diagnostics. - Helpers are LAYERFS_NFS_{MOUNT,UNMOUNT,FORCE,FLUSH} with {port}/{dir} placeholders. macOS default: /sbin/mount_nfs -o vers=3,tcp,port={port}, mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072, soft,intr,timeo=20,retrans=6,nobrowse 127.0.0.1:/ {dir}; umount / umount -f. - After a Commit or merge installs another root into an NFS-served Workspace, the daemon makes the macOS client drop its caches with a mode-preserving chmod of each stale inode through the mount (the EdenFS method; the export's refresh mark keeps it from writing). LAYERFS_NFS_REFRESH=0 turns it off. - Kernel::Attaching is no longer Linux-only; settle moves from detach.rs to operations.rs so it compiles on macOS. - nix gains the "resource" feature of the existing pinned crate; no new dependency. Boundary guard: the daemon may depend on layerfs-nfs. The Linux default is unchanged: without LAYERFS_KERNEL=nfs no export is built, installed_export is false and the FUSE invalidation runs as before. Adds layerfs-nfs tests/daemon.rs and examples/daemon_nfs.rs. No existing test changed or deleted. Production LOC: 113,293 -> 114,316 (delta +1023) Scope: combined core+sdk+dsh, tools/production_loc.py, parent e63af4c10. --- core/Cargo.lock | 1 + core/crates/layerfs-daemon/Cargo.toml | 4 +- .../layerfs-daemon/src/application/config.rs | 19 +- .../src/application/filesystem.rs | 82 +- .../layerfs-daemon/src/application/owner.rs | 13 + .../layerfs-daemon/src/application/serve.rs | 2 + .../layerfs-daemon/src/control/attach.rs | 5 + .../layerfs-daemon/src/control/detach.rs | 17 +- .../layerfs-daemon/src/control/export.rs | 966 ++++++++++++++++++ .../layerfs-daemon/src/control/force.rs | 21 +- .../layerfs-daemon/src/control/invalidate.rs | 6 + .../layerfs-daemon/src/control/kernel.rs | 11 +- core/crates/layerfs-daemon/src/control/mod.rs | 2 + .../layerfs-daemon/src/control/observation.rs | 6 +- .../layerfs-daemon/src/control/operations.rs | 31 + .../layerfs-daemon/src/control/registry.rs | 22 + .../layerfs-daemon/src/control/status.rs | 2 +- .../layerfs-daemon/src/control/types.rs | 1 - .../layerfs-daemon/src/control/unmount.rs | 20 +- .../crates/layerfs-nfs/examples/daemon_nfs.rs | 172 ++++ core/crates/layerfs-nfs/tests/daemon.rs | 292 ++++++ core/tools/check_product_boundary.py | 2 +- 22 files changed, 1661 insertions(+), 36 deletions(-) create mode 100644 core/crates/layerfs-daemon/src/control/export.rs create mode 100644 core/crates/layerfs-nfs/examples/daemon_nfs.rs create mode 100644 core/crates/layerfs-nfs/tests/daemon.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index b9c1005ad3..06a58e735f 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -588,6 +588,7 @@ dependencies = [ "layerfs-content", "layerfs-fuse", "layerfs-history", + "layerfs-nfs", "layerfs-overlay", "layerfs-persistence", "layerfs-project", diff --git a/core/crates/layerfs-daemon/Cargo.toml b/core/crates/layerfs-daemon/Cargo.toml index 592a889261..aecf6afd5c 100644 --- a/core/crates/layerfs-daemon/Cargo.toml +++ b/core/crates/layerfs-daemon/Cargo.toml @@ -11,6 +11,8 @@ layerfs-overlay = { path = "../layerfs-overlay" } layerfs-store-service = { path = "../layerfs-store-service" } layerfs-workspace = { path = "../layerfs-workspace" } layerfs-fuse = { path = "../layerfs-fuse" } +# NFS kernel serving: the native route's mount entry beside FUSE. +layerfs-nfs = { path = "../layerfs-nfs" } layerfs-bridge = { path = "../layerfs-bridge" } layerfs-content = { path = "../layerfs-content" } layerfs-storage = { path = "../layerfs-storage" } @@ -19,7 +21,7 @@ layerfs-telemetry = { path = "../layerfs-telemetry" } # Concrete Store opening stays in application bootstrap, outside store/. layerfs-persistence = { path = "../layerfs-persistence" } -nix = { version = "=0.31.3", features = ["fs", "user", "process"] } +nix = { version = "=0.31.3", features = ["fs", "user", "process", "resource"] } [dev-dependencies] blake3 = { version = "=1.8.5", default-features = false } diff --git a/core/crates/layerfs-daemon/src/application/config.rs b/core/crates/layerfs-daemon/src/application/config.rs index e3c404c8a0..a2269fba99 100644 --- a/core/crates/layerfs-daemon/src/application/config.rs +++ b/core/crates/layerfs-daemon/src/application/config.rs @@ -29,7 +29,10 @@ pub(super) fn read(path: &Path) -> io::Result { return Err(invalid("private config record size")); } let config = DaemonSetup::decode(&bytes).map_err(|_| invalid("private config fields"))?; - if config.command_uid == uid { + // The NFS route opens no FUSE device and the native route runs its + // commands as the daemon's own user, unconfined (no isolation); every + // other route keeps the two identities apart. + if config.command_uid == uid && !super::filesystem::nfs_route() { return Err(invalid("command and daemon identities must differ")); } // One early reservation restores any value below the refill window; a @@ -108,11 +111,21 @@ fn secure_process() -> io::Result<()> { } Ok(()) } -#[cfg(not(target_os = "linux"))] +/// macOS (the native NFS route): files the daemon creates are its own +/// alone, and it never writes a core file holding its keys or Store pages. +/// There is no dumpable flag; a debugger of the same user is not refused. +#[cfg(target_os = "macos")] +fn secure_process() -> io::Result<()> { + use nix::sys::resource::{setrlimit, Resource}; + nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o077)); + setrlimit(Resource::RLIMIT_CORE, 0, 0).map_err(io::Error::from)?; + Ok(()) +} +#[cfg(not(any(target_os = "linux", target_os = "macos")))] fn secure_process() -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Unsupported, - "production daemon requires Linux process protection", + "production daemon requires Linux or macOS process protection", )) } fn invalid(message: &'static str) -> io::Error { diff --git a/core/crates/layerfs-daemon/src/application/filesystem.rs b/core/crates/layerfs-daemon/src/application/filesystem.rs index 095cae93bb..d428c363cf 100644 --- a/core/crates/layerfs-daemon/src/application/filesystem.rs +++ b/core/crates/layerfs-daemon/src/application/filesystem.rs @@ -1,10 +1,72 @@ //! Kernel serving assembly over the daemon's one Store and one overlay owner. use crate::{ - control::{KernelServing, Service}, + control::{ExportServing, ExportServingConfig, Helper, KernelServing, Mounter, Service}, store::Store, Owner, }; -use std::sync::Arc; +use layerfs_bridge::daemon_setup::DaemonSetup; +use std::{path::Path, sync::Arc}; + +/// Selects the kernel route: `nfs` serves every mount over loopback NFSv3 +/// (the native route); anything else, or unset, is FUSE. +pub const KERNEL_ROUTE: &str = "LAYERFS_KERNEL"; +/// Optional helper command lines (`{port}`, `{dir}` replaced) overriding the +/// host client's defaults for the NFS route. +pub const NFS_MOUNT: &str = "LAYERFS_NFS_MOUNT"; +pub const NFS_UNMOUNT: &str = "LAYERFS_NFS_UNMOUNT"; +pub const NFS_FORCE: &str = "LAYERFS_NFS_FORCE"; +pub const NFS_FLUSH: &str = "LAYERFS_NFS_FLUSH"; + +/// Whether this daemon serves its mounts over NFS. +/// macOS has no other route: there it is always NFS. +pub(super) fn nfs_route() -> bool { + cfg!(target_os = "macos") || std::env::var(KERNEL_ROUTE).is_ok_and(|route| route == "nfs") +} +/// NFS serving for the native route, from the protected setup and the +/// declared helper overrides. Nothing is mounted until an explicit Attach. +pub(super) fn assemble_export(setup: &DaemonSetup) -> std::io::Result>> { + if !nfs_route() { + return Ok(None); + } + let mounts = Path::new(&setup.mounts); + if !mounts.is_absolute() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "mount home must be absolute", + )); + } + if !mounts.is_dir() { + std::fs::create_dir(mounts)?; + } + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper(NFS_MOUNT) { + mounter.mount = mount; + } + if let Some(unmount) = helper(NFS_UNMOUNT) { + mounter.unmount = unmount; + } + if let Some(force) = helper(NFS_FORCE) { + mounter.force = force; + } + mounter.flush = helper(NFS_FLUSH); + let serving = ExportServing::start(ExportServingConfig::new( + mounts.to_owned(), + setup.command_uid, + setup.command_gid, + mounter, + )); + // A daemon of this mount home that ended without unmounting left its + // client mounts; they are forced off before this one serves. + for line in serving.reclaim() { + eprintln!("LAYERFS_NFS_RECLAIM {line}"); + } + Ok(Some(Arc::new(serving))) +} /// Control and kernel serving share the same Store handle and overlay owner; /// the Service's registry stays the sole authority for every mount. The @@ -13,19 +75,19 @@ pub(super) fn service( store: Arc, owner: &Owner, kernel: Option<&Arc>, + export: Option<&Arc>, serial_low_water: u64, ) -> Arc { store.set_serial_low_water(serial_low_water); - Arc::new(match kernel { - Some(kernel) => Service::with_kernel(store, owner, kernel.clone()), - None => Service::new(store, owner), + Arc::new(match (export, kernel) { + (Some(export), _) => Service::with_export(store, owner, export.clone()), + (None, Some(kernel)) => Service::with_kernel(store, owner, kernel.clone()), + (None, None) => Service::new(store, owner), }) } /// Kernel serving is a Linux capability; elsewhere control alone is assembled. #[cfg(not(target_os = "linux"))] -pub(super) fn assemble( - _setup: &layerfs_bridge::daemon_setup::DaemonSetup, -) -> std::io::Result>> { +pub(super) fn assemble(_setup: &DaemonSetup) -> std::io::Result>> { Ok(None) } #[cfg(target_os = "linux")] @@ -58,6 +120,10 @@ mod linux { pub(in crate::application) fn assemble( setup: &DaemonSetup, ) -> io::Result>> { + // The NFS route opens no FUSE device. + if super::nfs_route() { + return Ok(None); + } let mounts = Path::new(&setup.mounts); device()?; home(mounts)?; diff --git a/core/crates/layerfs-daemon/src/application/owner.rs b/core/crates/layerfs-daemon/src/application/owner.rs index 9206dee1ac..c15d6833d8 100644 --- a/core/crates/layerfs-daemon/src/application/owner.rs +++ b/core/crates/layerfs-daemon/src/application/owner.rs @@ -44,6 +44,8 @@ pub struct Application { pub(super) owner: Owner, /// Fixed shared kernel workers, entered before control readiness. pub(super) kernel: Option>, + /// NFS serving of the native route, when selected. + pub(super) export: Option>, pub(super) state: Mutex, pub(super) changed: Condvar, pub(super) workers: Mutex>>>, @@ -120,11 +122,21 @@ impl Application { }) } }; + let export = match super::filesystem::assemble_export(&setup) { + Ok(export) => export, + Err(cause) => { + return Err(ApplicationError::AfterOverlay { + owner: Box::new(owner), + cause: Box::new(ApplicationError::Io(cause)), + }) + } + }; let service = existing.as_ref().map(|opened| { super::filesystem::service( opened.store.clone(), &owner, kernel.as_ref(), + export.as_ref(), l.serial_low_water, ) }); @@ -139,6 +151,7 @@ impl Application { instance, owner, kernel, + export, state: Mutex::new(State { startup: Startup { phase, diff --git a/core/crates/layerfs-daemon/src/application/serve.rs b/core/crates/layerfs-daemon/src/application/serve.rs index f88db6a0e6..645571dd32 100644 --- a/core/crates/layerfs-daemon/src/application/serve.rs +++ b/core/crates/layerfs-daemon/src/application/serve.rs @@ -96,6 +96,7 @@ impl Application { installed.opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.installed = Some(installed); @@ -119,6 +120,7 @@ impl Application { opened.store.clone(), &self.owner, self.kernel.as_ref(), + self.export.as_ref(), l.serial_low_water, )); state.startup.phase = DaemonPhase::ControlReady; diff --git a/core/crates/layerfs-daemon/src/control/attach.rs b/core/crates/layerfs-daemon/src/control/attach.rs index 2c300bd5b4..3e095b1480 100644 --- a/core/crates/layerfs-daemon/src/control/attach.rs +++ b/core/crates/layerfs-daemon/src/control/attach.rs @@ -35,6 +35,11 @@ impl Service { token: WorkspaceToken, timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return crate::telemetry::phase(timing, "kernel.attach", || { + self.attach_export(&export, token) + }); + } let Some(serving) = self.kernel.clone() else { return Err(Failure::Rejected( ControlCode::Invalid, diff --git a/core/crates/layerfs-daemon/src/control/detach.rs b/core/crates/layerfs-daemon/src/control/detach.rs index 2e400e039b..89f97f942d 100644 --- a/core/crates/layerfs-daemon/src/control/detach.rs +++ b/core/crates/layerfs-daemon/src/control/detach.rs @@ -29,7 +29,12 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + Kernel::Unattached + | Kernel::Attaching + | Kernel::Retained(_) + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => { unreachable!("handled by terminal admission") } } @@ -135,16 +140,6 @@ impl Service { } } } - /// Poison recovery stores this exact owner only; a poisoned registry never - /// authorizes another product operation. - pub(super) fn settle(&self, token: WorkspaceToken, kernel: Kernel, activity: Activity) { - let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); - if let Ok(binding) = bound_mut(&mut entries, token) { - binding.kernel = kernel; - binding.activity = activity; - binding.epoch = binding.epoch.saturating_add(1); - } - } /// Stops a terminal operation after effects, keeping its owner in the entry. /// A forced teardown's facts are stored with it, as they stood at the stop. pub(super) fn retain( diff --git a/core/crates/layerfs-daemon/src/control/export.rs b/core/crates/layerfs-daemon/src/control/export.rs new file mode 100644 index 0000000000..adbd75714f --- /dev/null +++ b/core/crates/layerfs-daemon/src/control/export.rs @@ -0,0 +1,966 @@ +//! NFS kernel serving: the native route's mount entry, beside FUSE. +//! +//! Each Attach serves the Workspace's engine mount as one loopback NFSv3 +//! export (`layerfs-nfs`) on a fresh port, then has the host's own NFS client +//! mount it on the Workspace's mount directory with the configured helper +//! (`mount_nfs` on macOS). The registry keeps the export as the entry's +//! kernel half; the Workspace API, its control records and every engine and +//! Store step are the ones the FUSE route uses: +//! +//! - Ready carries the same receipt record; an NFS receipt is recognized by +//! `abi_major == 0`, and its `mount_id` is the export's TCP port. +//! - Commit first has the client write back what it buffered (the flush +//! helper, `sync(2)` by default) and closes the export's implicit file +//! descriptors, then runs the unchanged Commit. A Commit that installed +//! another root closes the export's descriptors again; the client +//! revalidates attributes on its own schedule (close-to-open, `actimeo`). +//! - Unmount asks the helper to unmount (a busy answer is reversible and the +//! export keeps serving), then stops the export (every connection joined, +//! every reference and descriptor given back), revokes the engine mount +//! and runs the one logical Close. A step that is not established keeps +//! the export and its engine mount in the entry, as FUSE custody is kept. +//! - ForceUnmount runs the forced helper once, then the same teardown. +use super::{ + kernel::{bounded, Kernel}, + registry::{bound_mut, idle}, + Failure, Service, Success, +}; +use crate::store::BoundWorkspace; +use layerfs_bridge::control::{ + AbortDisposition, Activity, ControlCode, DetachDisposition, ForceUnmounted, ForcedCleanup, + ForcedFacts, ForcedOutcome, MountPhase, MountReceipt, MountStatus, MountWork, ReadyMount, + Reply, TeardownCustody, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, +}; +use layerfs_nfs::{ExportConfig, Released, Server}; +use layerfs_overlay::{CleanupState, KernelMount}; +use std::{ + fmt, + io::Read, + net::{IpAddr, Ipv4Addr, TcpListener}, + path::PathBuf, + process::{Command, Stdio}, + sync::Arc, + time::{Duration, Instant}, +}; + +/// Largest write and read the export negotiates (FSINFO wtmax/rtmax). +const WINDOW: u32 = 128 * 1024; +/// How long one helper run may take before it is killed and reported. +pub const HELPER_WAIT: Duration = Duration::from_secs(60); +/// Bytes of a helper's output kept for a refusal. +const OUTPUT: usize = 2048; + +/// Stale inodes one install refreshes through the mount point at most. +pub const REFRESHED: usize = 4096; +/// `0` turns the post-install refresh off (see [`Service::installed_export`]). +pub const REFRESH: &str = "LAYERFS_NFS_REFRESH"; + +/// One mode-preserving chmod of each path below `root`, through the host's +/// NFS client; a symbolic link (chmod would follow it) and a path that is +/// gone are skipped. The export answers each without a change. +fn refresh(root: &std::path::Path, paths: &[PathBuf]) { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + for path in paths { + let path = root.join(path); + let Ok(metadata) = std::fs::symlink_metadata(&path) else { + continue; + }; + if !metadata.file_type().is_symlink() { + let mode = std::fs::Permissions::from_mode(metadata.mode() & 0o7777); + let _ = std::fs::set_permissions(&path, mode); + } + } +} +/// One helper command; `{port}` and `{dir}` in its words are replaced. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Helper(pub Vec); +impl Helper { + /// Splits a configured command line on whitespace. + pub fn words(line: &str) -> Option { + let words: Vec = line.split_whitespace().map(str::to_owned).collect(); + (!words.is_empty()).then_some(Self(words)) + } + fn command(&self, port: u16, directory: &str) -> Option { + let mut words = self.0.iter().map(|word| { + word.replace("{port}", &port.to_string()) + .replace("{dir}", directory) + }); + let mut command = Command::new(words.next()?); + command.args(words); + Some(command) + } +} +/// How the host's NFS client mounts, flushes and unmounts an export. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Mounter { + pub mount: Helper, + pub unmount: Helper, + pub force: Helper, + /// Writes back what the client buffered before a Commit; `sync(2)` of + /// the daemon itself when absent. + pub flush: Option, +} +impl Mounter { + /// The host's own client: `mount_nfs` on macOS (no root needed), the + /// Linux NFS client elsewhere (root, as the daemon is in its container). + /// The macOS mount is soft (an export that ended fails calls after the + /// retries instead of hanging them) and hidden from Finder (`nobrowse`). + pub fn host() -> Self { + let helper = |words: &[&str]| Helper(words.iter().map(|w| (*w).to_owned()).collect()); + if cfg!(target_os = "macos") { + Self { + mount: helper(&[ + "/sbin/mount_nfs", + "-o", + "vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,timeo=20,retrans=6,nobrowse", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["/sbin/umount", "{dir}"]), + force: helper(&["/sbin/umount", "-f", "{dir}"]), + flush: None, + } + } else { + Self { + mount: helper(&[ + "mount", + "-t", + "nfs", + "-o", + "vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60", + "127.0.0.1:/", + "{dir}", + ]), + unmount: helper(&["umount", "{dir}"]), + force: helper(&["umount", "-f", "-l", "{dir}"]), + flush: None, + } + } + } +} +/// Declared NFS serving inputs, assembled once by the application. +#[derive(Clone, Debug)] +pub struct ExportServingConfig { + /// Existing daemon-owned directory; each mount is a new child of it. + pub mounts: PathBuf, + /// Runtime command identity presented as every inode's owner. + pub command_uid: u32, + pub command_gid: u32, + /// Address each export listens on, port chosen by the system. + pub listen: IpAddr, + pub mounter: Mounter, + /// Refuse creating macOS AppleDouble `._*` names. + pub refuse_apple_double: bool, + /// Lookup references one export keeps before collecting. + pub lookups_held: usize, +} +impl ExportServingConfig { + pub fn new(mounts: PathBuf, command_uid: u32, command_gid: u32, mounter: Mounter) -> Self { + Self { + mounts, + command_uid, + command_gid, + listen: IpAddr::V4(Ipv4Addr::LOCALHOST), + mounter, + refuse_apple_double: true, + lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + } + } +} +/// Process-wide NFS serving: configuration only; each Attach starts its own +/// export and each terminal operation stops it. +#[derive(Debug)] +pub struct ExportServing { + config: ExportServingConfig, +} +impl ExportServing { + pub fn start(config: ExportServingConfig) -> Self { + Self { config } + } + pub fn config(&self) -> &ExportServingConfig { + &self.config + } + fn directory(&self, token: WorkspaceToken) -> PathBuf { + self.config.mounts.join(token.namespace.to_string()) + } + /// At startup, before any Attach: every mount the host still has at or + /// under the mount home is one a daemon that ended without unmounting + /// left (its export is gone, so the client would hang on it). Each is + /// forced off once with the force helper; one line per mount says what + /// happened. Nothing is retried. + pub fn reclaim(&self) -> Vec { + let points = match mounted_under(&self.config.mounts) { + Ok(points) => points, + Err(error) => return vec![format!("mount table unreadable: {error}")], + }; + (points.into_iter()) + .map(|point| { + let text = point.display().to_string(); + match run(&self.config.mounter.force, 0, &text) { + Ran::Done => format!("forced off {text}"), + Ran::Busy(detail) | Ran::Failed(detail) => { + format!("left {text}: {detail}") + } + } + }) + .collect() + } +} +/// Mount points at or under `home`, deepest first, from the host's table: +/// `mount`'s ` on ()` lines on macOS, +/// `/proc/self/mountinfo` (field 5, octal escapes) on Linux. +fn mounted_under(home: &std::path::Path) -> std::io::Result> { + #[cfg(target_os = "macos")] + let points: Vec = { + let out = Command::new("/sbin/mount") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + (String::from_utf8_lossy(&out.stdout).lines()) + .filter_map(|line| { + Some(PathBuf::from( + line.split_once(" on ")?.1.rsplit_once(" (")?.0, + )) + }) + .collect() + }; + #[cfg(not(target_os = "macos"))] + let points: Vec = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) + .filter_map(|line| line.split(' ').nth(4)) + .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) + .collect(); + let mut found: Vec = (points.into_iter()) + .filter(|point| point.starts_with(home)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} +/// A serving export and the mount the host client holds of it. +pub(super) struct Exported { + pub server: Arc, + pub mount: KernelMount, + pub ready: ReadyMount, +} +/// A terminal operation owns the export; status still observes it. +pub(super) struct ExportLeaving { + pub ready: ReadyMount, + pub phase: MountPhase, + pub work: MountWork, +} +/// Export custody a terminal or attach step could not finish disposing: +/// kept exactly, never retried or settled by a later observation. +pub(super) struct ExportKept { + pub ready: Option, + pub stage: TeardownStage, + pub detail: String, + pub work: Option, + pub evidence: Box, + pub forced: Option, +} +impl ExportKept { + pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { + TeardownCustody { + token, + stage: self.stage, + detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), + work: self.work, + detail: bounded(&self.detail), + forced: self.forced.clone(), + } + } +} +impl fmt::Debug for ExportKept { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExportKept") + .field("ready", &self.ready) + .field("stage", &self.stage) + .field("detail", &self.detail) + .field("work", &self.work) + .field("evidence", &self.evidence) + .field("forced", &self.forced) + .finish() + } +} +impl fmt::Debug for Exported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Exported") + .field("address", &self.server.local_addr()) + .field("mount", &self.mount) + .field("held", &self.server.held()) + .finish() + } +} +/// The maintained counters of a serving export. +pub(super) fn work(server: &Server) -> MountWork { + let held = server.held(); + let served = server.counters().iter().map(|count| count.calls).sum(); + MountWork { + loops_configured: 1, + loops_entered: 1, + loops_exited: 0, + loops_joined: 0, + active: 0, + retained: u32::try_from(held.retained).unwrap_or(u32::MAX), + completed: served, + served, + inline: 0, + refused: 0, + terminal: 0, + forget_units: held.collected, + } +} +/// The status block of an export-owning entry. +pub(super) fn block(kernel: &Kernel) -> Option { + Some(match kernel { + Kernel::Exported(exported) => MountStatus { + phase: MountPhase::Ready, + ready: Some(exported.ready.clone()), + detached: false, + work: Some(work(&exported.server)), + }, + Kernel::ExportLeaving(leaving) => MountStatus { + phase: leaving.phase, + ready: Some(leaving.ready.clone()), + detached: leaving.phase == MountPhase::Draining, + work: Some(leaving.work), + }, + Kernel::ExportKept(kept) => MountStatus { + phase: MountPhase::Retained, + ready: kept.ready.clone(), + detached: matches!(kept.stage, TeardownStage::Revoke | TeardownStage::Close), + work: kept.work, + }, + _ => return None, + }) +} +/// What one helper run answered. +enum Ran { + Done, + /// The client reported the mount busy; nothing changed. + Busy(String), + Failed(String), +} +fn run(helper: &Helper, port: u16, directory: &str) -> Ran { + let Some(mut command) = helper.command(port, directory) else { + return Ran::Failed("empty helper command".into()); + }; + let child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + let mut child = match child { + Ok(child) => child, + Err(error) => return Ran::Failed(format!("{:?}: {error}", helper.0.first())), + }; + let deadline = Instant::now() + HELPER_WAIT; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(5)), + Ok(None) => { + drop(child.kill()); + drop(child.wait()); + break Err(format!("helper did not finish in {HELPER_WAIT:?}; killed")); + } + Err(error) => break Err(format!("helper wait: {error}")), + } + }; + let mut output = String::new(); + for pipe in [ + child.stderr.take().map(|p| Box::new(p) as Box), + child.stdout.take().map(|p| Box::new(p) as Box), + ] + .into_iter() + .flatten() + { + let mut bytes = Vec::new(); + drop(pipe.take(OUTPUT as u64).read_to_end(&mut bytes)); + output.push_str(&String::from_utf8_lossy(&bytes)); + } + match status { + Ok(status) if status.success() => Ran::Done, + Ok(status) => { + let detail = format!("{status}: {}", output.trim()); + if output.to_ascii_lowercase().contains("busy") { + Ran::Busy(detail) + } else { + Ran::Failed(detail) + } + } + Err(detail) => Ran::Failed(format!("{detail}: {}", output.trim())), + } +} +fn receipt(token: WorkspaceToken, directory: String, mount: KernelMount, port: u16) -> ReadyMount { + ReadyMount { + token, + directory, + receipt: MountReceipt { + mount: mount.owner_id(), + root: mount.root_serial(), + mount_id: u64::from(port), + device_major: 0, + device_minor: 0, + abi_major: 0, + abi_minor: 0, + offered: 0, + selected: 0, + max_write: WINDOW, + max_readahead: WINDOW, + max_background: 0, + congestion_threshold: 0, + page_size: 4096, + loops: 1, + abort_bound: true, + }, + } +} +fn kernel_failure(code: ControlCode, phase: &'static str, detail: impl Into) -> Failure { + Failure::Kernel(Box::new(super::types::KernelFailure { + code, + phase, + detail: detail.into(), + evidence: None, + })) +} +/// Where a failed export Attach leaves the entry. +enum Stop { + Unattached(Failure), + Unknown(Failure), + Retained(Box), +} +impl Service { + /// Attach over NFS: Ready is acknowledged only after the export serves + /// and the host client's mount helper reported the mount. + pub(super) fn attach_export( + &self, + serving: &ExportServing, + token: WorkspaceToken, + ) -> Result { + let workspace = self.admit_export(token)?; + match self.export_admitted(serving, token, &workspace) { + Ok(exported) => { + let reply = Reply::Ready(Box::new(exported.ready.clone())); + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + Ok(Success::reply(reply)) + } + Err(Stop::Unattached(failure)) => { + self.settle(token, Kernel::Unattached, Activity::Idle); + Err(failure) + } + Err(Stop::Unknown(failure)) => { + self.settle(token, Kernel::Attaching, Activity::Uncertain); + Err(failure) + } + Err(Stop::Retained(kept)) => { + let custody = kept.custody(token); + self.settle(token, Kernel::ExportKept(kept), Activity::Uncertain); + Err(Failure::Retained(Box::new(custody))) + } + } + } + fn admit_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } + idle(value)?; + if !matches!(value.kernel, Kernel::Unattached) { + return Err(kernel_failure( + ControlCode::Invalid, + "attach:admission", + "Workspace already owns a kernel connection", + )); + } + value.kernel = Kernel::Attaching; + value.activity = Activity::Attaching; + value.epoch = value.epoch.saturating_add(1); + Ok(value.workspace.clone()) + } + fn export_admitted( + &self, + serving: &ExportServing, + token: WorkspaceToken, + workspace: &Arc, + ) -> Result { + let config = &serving.config; + let directory = serving.directory(token); + let Some(text) = directory + .to_str() + .filter(|text| text.len() <= MOUNT_DIRECTORY_LIMIT) + .map(str::to_owned) + else { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Invalid, + "attach:directory", + "mount directory exceeds the control record bound", + ))); + }; + let unknown = |failure: Failure| { + if failure.uncertain() { + Stop::Unknown(failure) + } else { + Stop::Unattached(failure) + } + }; + // The same preparation as the FUSE Attach: the root serial, then the + // initial authoritative serial range before any request can create. + let operation = workspace + .operation() + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + let root = operation + .workspace() + .base() + .map(|base| base.root().root_inode().serial()) + .map_err(|error| Stop::Unattached(Failure::Workspace(error)))?; + if let Err(error) = operation.workspace().prepare_serials(operation.ports()) { + let failure = match operation.ports().failure() { + Ok(Some(cause)) => match cause.as_ref() { + crate::store::PortError::History(error) => Failure::History(error.clone()), + _ => Failure::Workspace(error), + }, + Ok(None) | Err(_) => Failure::Workspace(error), + }; + return Err(unknown(failure)); + } + drop(operation); + if let Err(error) = std::fs::create_dir(&directory) { + return Err(Stop::Unattached(kernel_failure( + ControlCode::Failed, + "attach:directory", + format!("mount directory {text}: {error}"), + ))); + } + let mount = match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_mount(route, root) + }) { + Ok(mount) => mount, + Err(failure) => { + drop(std::fs::remove_dir(&directory)); + return Err(unknown(failure)); + } + }; + let export = ExportConfig { + path: "/".into(), + uid: config.command_uid, + gid: config.command_gid, + read_only: false, + refuse_apple_double: config.refuse_apple_double, + lookups_held: config.lookups_held, + }; + let started = TcpListener::bind((config.listen, 0)).and_then(|listener| { + Server::start( + listener, + Arc::new(crate::service::Mounted(workspace.clone())), + mount, + export, + ) + }); + let server = match started { + Ok(server) => server, + Err(error) => { + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + None, + kernel_failure(ControlCode::Failed, "attach:export", error.to_string()), + )); + } + }; + let port = server.local_addr().port(); + match run(&config.mounter.mount, port, &text) { + Ran::Done => {} + Ran::Busy(detail) | Ran::Failed(detail) => { + let released = server.stop(); + drop(std::fs::remove_dir(&directory)); + return Err(self.unwind_export( + workspace, + mount, + Some(released), + kernel_failure(ControlCode::Failed, "attach:mount", detail), + )); + } + } + Ok(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }) + } + /// No client mount and no export of this attempt remains: revoke its + /// engine mount so the entry returns to Unattached. + fn unwind_export( + &self, + workspace: &BoundWorkspace, + mount: KernelMount, + released: Option, + failure: Failure, + ) -> Stop { + match self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + Ok(()) => Stop::Unattached(failure), + Err(revoke) => Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Revoke, + detail: revoke.wire().detail, + work: None, + evidence: Box::new((failure, mount, released, revoke)), + forced: None, + })), + } + } + /// Before a Commit: the client writes back what it buffered, and the + /// export closes its implicit file descriptors. Runs outside the + /// registry lock; a Workspace without an export has nothing to flush. + pub(super) fn flush_export(&self, token: WorkspaceToken) -> Result<(), Failure> { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(()), + } + }; + let Some(serving) = self.export.as_ref() else { + return Ok(()); + }; + match &serving.config.mounter.flush { + None => nix::unistd::sync(), + Some(helper) => match run(helper, server.local_addr().port(), &directory) { + Ran::Done => {} + Ran::Busy(detail) | Ran::Failed(detail) => { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) + } + }, + } + server.close_files(); + Ok(()) + } + /// After a Commit installed another root: the export's descriptors of the + /// replaced tree are closed, and the inodes the install made stale that + /// the client may cache are refreshed through the mount point (see + /// `layerfs_nfs::refresh`): a mode-preserving chmod of each, at most + /// [`REFRESHED`], on a thread of its own after this returns, so the + /// Commit reply never waits on the client. False without an export. + pub(super) fn installed_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + shown: layerfs_content::ObjectId, + installed: layerfs_content::ObjectId, + ) -> Result { + let (server, directory) = { + let entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + match &super::registry::bound(&entries, token)?.kernel { + Kernel::Exported(exported) => { + (exported.server.clone(), exported.ready.directory.clone()) + } + _ => return Ok(false), + } + }; + server.installed(); + // `LAYERFS_NFS_REFRESH=0` leaves the client to its own revalidation + // (`actimeo`, close-to-open): the comparison without the refresh. + if std::env::var(REFRESH).is_ok_and(|value| value == "0") { + return Ok(true); + } + let mut stale = std::collections::BTreeSet::new(); + // A walk that stops (at the bound, or failing) refreshes what it found; + // the rest is revalidated on the client's own schedule. + let _ = workspace.stale_after_install(shown, installed, |found| { + stale.insert(match found { + crate::store::Stale::Inode(serial) => serial, + crate::store::Stale::Entry { parent, .. } => parent, + }); + if stale.len() >= REFRESHED { + return Err(std::io::Error::other("refresh bound")); + } + Ok(()) + }); + let paths = server.refresh(stale); + if !paths.is_empty() { + let root = PathBuf::from(directory); + let _ = std::thread::Builder::new() + .name("layerfs-nfs-refresh".into()) + .spawn(move || refresh(&root, &paths)); + } + Ok(true) + } + /// Normal terminal unmount of an export-owning entry, admitted by + /// [`take`]. A busy client mount is the reversible answer. + pub(super) fn unmount_export( + &self, + token: WorkspaceToken, + workspace: Arc, + exported: Exported, + ) -> Result { + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.unmount.clone()); + let port = exported.server.local_addr().port(); + let ran = match &helper { + Some(helper) => run(helper, port, &exported.ready.directory), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => {} + Ran::Busy(detail) => { + self.settle(token, Kernel::Exported(Box::new(exported)), Activity::Idle); + return Err(kernel_failure( + ControlCode::Busy, + "unmount:kernel", + format!("client reported the mount busy; no terminal effect: {detail}"), + )); + } + Ran::Failed(detail) => { + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("client unmount: {detail}"), + exported, + None, + )) + } + } + self.teardown_export(token, &workspace, exported, None)?; + self.close(token, &workspace, None) + } + /// After the client mount is gone: stop the export, revoke the engine + /// mount, remove the directory and leave the entry Unattached for Close. + fn teardown_export( + &self, + token: WorkspaceToken, + workspace: &BoundWorkspace, + exported: Exported, + forced: Option, + ) -> Result<(), Failure> { + self.export_phase(token, MountPhase::Draining, Some(work(&exported.server))); + let Exported { + server, + mount, + ready, + } = exported; + let released = match Arc::try_unwrap(server) { + Ok(server) => server.stop(), + Err(server) => { + return Err(self.keep_export( + token, + TeardownStage::Requests, + "export still shared by a running control step".into(), + Exported { + server, + mount, + ready, + }, + forced, + )) + } + }; + if let Err(failure) = self.job(workspace.route(), |client: &crate::OwnerClient, route| { + client.kernel_revoke(route, mount) + }) { + let custody = ExportKept { + ready: Some(ready), + stage: TeardownStage::Revoke, + detail: failure.wire().detail, + work: None, + evidence: Box::new((failure, mount, released)), + forced, + }; + let reply = custody.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(custody)), + Activity::Uncertain, + ); + return Err(Failure::Retained(Box::new(reply))); + } + drop(std::fs::remove_dir(&ready.directory)); + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = Kernel::Unattached; + } + } + self.released + .lock() + .map(|mut last| *last = Some((token, released))) + .ok(); + Ok(()) + } + fn keep_export( + &self, + token: WorkspaceToken, + stage: TeardownStage, + detail: String, + exported: Exported, + forced: Option, + ) -> Failure { + let kept = ExportKept { + ready: Some(exported.ready.clone()), + stage, + detail, + work: Some(work(&exported.server)), + evidence: Box::new(exported), + forced, + }; + let custody = kept.custody(token); + self.settle( + token, + Kernel::ExportKept(Box::new(kept)), + Activity::Uncertain, + ); + Failure::Retained(Box::new(custody)) + } + fn export_phase(&self, token: WorkspaceToken, phase: MountPhase, work: Option) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + if let Kernel::ExportLeaving(leaving) = &mut binding.kernel { + leaving.phase = phase; + if let Some(work) = work { + leaving.work = work; + } + } + } + } + } + /// Describes retained export custody for an operator. Observation only. + pub fn retained_export(&self, token: WorkspaceToken) -> Result, Failure> { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + Ok(match &bound_mut(&mut entries, token)?.kernel { + Kernel::ExportKept(kept) => Some(format!("{kept:?}")), + _ => None, + }) + } + /// What the last stopped export of `token` gave back (diagnostics). + pub fn export_released(&self, token: WorkspaceToken) -> Option { + let last = self.released.lock().ok()?; + last.filter(|(seen, _)| *seen == token) + .map(|(_, released)| released) + } + /// The current holdings of a serving export (diagnostics). + pub fn export_held(&self, token: WorkspaceToken) -> Option { + let entries = self.entries.lock().ok()?; + match &super::registry::bound(&entries, token).ok()?.kernel { + Kernel::Exported(exported) => Some(exported.server.held()), + _ => None, + } + } + /// Forced terminal unmount of an export-owning entry: the same + /// before-effect guard as FUSE, one forced client unmount, then the + /// normal teardown and Close. + pub(super) fn force_export( + &self, + token: WorkspaceToken, + relinquish_unknown: bool, + ) -> Result { + let (workspace, exported, commit) = { + let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; + let value = bound_mut(&mut entries, token)?; + let commit = super::force::guard(value, token, relinquish_unknown)?; + let exported = take(&mut value.kernel, MountPhase::Stopping)?; + value.activity = Activity::Closing; + value.epoch = value.epoch.saturating_add(1); + (value.workspace.clone(), exported, commit) + }; + let mut facts = ForcedFacts { + abort: AbortDisposition::Written, + detach: DetachDisposition::NotAttempted, + commit, + fenced: 0, + }; + let helper = self + .export + .as_ref() + .map(|serving| serving.config.mounter.force.clone()); + let port = exported.server.local_addr().port(); + let ran = match &helper { + Some(helper) => run(helper, port, &exported.ready.directory), + None => Ran::Failed("export serving not assembled".into()), + }; + match ran { + Ran::Done => facts.detach = DetachDisposition::Detached, + Ran::Busy(detail) | Ran::Failed(detail) => { + facts.detach = DetachDisposition::Failed; + return Err(self.keep_export( + token, + TeardownStage::Detach, + format!("forced client unmount: {detail}"), + exported, + Some(facts), + )); + } + } + let work = work(&exported.server); + self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; + let kernel = self.closed(token, &workspace, None, Some(facts.clone()))?; + let (cleanup, observation_failure) = match self + .owner + .observe_cleanup(None, token.namespace, token.workspace.to_bytes()) + .map_err(Failure::Owner) + { + Ok(CleanupState::Held) => (ForcedCleanup::Held, None), + Ok(CleanupState::Queued) => (ForcedCleanup::Queued, None), + Ok(CleanupState::Gone) => (ForcedCleanup::Gone, None), + Ok(CleanupState::Live) => ( + ForcedCleanup::Unobserved, + Some(Box::new(Failure::Rejected( + ControlCode::Unknown, + "closed Workspace remains live", + ))), + ), + Err(error) => (ForcedCleanup::Unobserved, Some(Box::new(error))), + }; + self.depart( + token, + Success { + reply: Reply::ForceUnmounted(Box::new(ForceUnmounted { + token, + outcome: ForcedOutcome { + facts, + work, + cleanup, + }, + })), + commit: None, + observation_failure, + kernel, + }, + ) + } +} +/// Terminal admission over an export-owning entry; the entry observes the +/// export as leaving until the operation settles it. +pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { + match kernel { + Kernel::Exported(_) => {} + Kernel::ExportLeaving(_) => { + return Err(kernel_failure( + ControlCode::Busy, + "unmount:admission", + "terminal unmount already admitted", + )) + } + _ => { + return Err(kernel_failure( + ControlCode::Invalid, + "unmount:admission", + "Workspace owns no export", + )) + } + } + let Kernel::Exported(exported) = std::mem::replace(kernel, Kernel::Unattached) else { + unreachable!("checked export") + }; + *kernel = Kernel::ExportLeaving(Box::new(ExportLeaving { + ready: exported.ready.clone(), + phase, + work: work(&exported.server), + })); + Ok(*exported) +} diff --git a/core/crates/layerfs-daemon/src/control/force.rs b/core/crates/layerfs-daemon/src/control/force.rs index c7baa82537..ac37ee96bc 100644 --- a/core/crates/layerfs-daemon/src/control/force.rs +++ b/core/crates/layerfs-daemon/src/control/force.rs @@ -43,18 +43,19 @@ fn refused(code: ControlCode, phase: &'static str, detail: &'static str) -> Fail /// The whole before-effect decision, made under the registry lock and in this /// order. A refusal changes nothing. Admitted, it returns the Commit knowledge /// the receipt will carry: copied here, never settled by a later read. -fn guard( +pub(super) fn guard( value: &Binding, token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { // Stored custody answers first: no second abort or detach ever follows it. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { return Err(Failure::Retained(Box::new(kept.custody(token)))); } - #[cfg(not(target_os = "linux"))] - let _ = token; if matches!( value.activity, Activity::Committing | Activity::Attaching | Activity::Closing @@ -106,7 +107,6 @@ fn connection(kernel: &Kernel) -> Result { ADMISSION, "Workspace owns no kernel connection", )), - #[cfg(target_os = "linux")] Kernel::Attaching => Err(refused( ControlCode::Unknown, ADMISSION, @@ -122,6 +122,13 @@ fn connection(kernel: &Kernel) -> Result { )), #[cfg(target_os = "linux")] Kernel::Ready(attached) => Ok(attached.session.abort_bound()), + // The forced client unmount is the export's abort. + Kernel::Exported(_) => Ok(true), + Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => Err(refused( + ControlCode::Busy, + ADMISSION, + "terminal unmount already admitted", + )), } } #[cfg(not(target_os = "linux"))] @@ -132,6 +139,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; guard(bound_mut(&mut entries, token)?, token, relinquish_unknown)?; Err(refused( @@ -167,6 +177,9 @@ impl Service { token: WorkspaceToken, relinquish_unknown: bool, ) -> Result { + if self.export.is_some() { + return self.force_export(token, relinquish_unknown); + } let admitted = self.admit_force(token, relinquish_unknown)?; self.force_admitted(token, admitted) } diff --git a/core/crates/layerfs-daemon/src/control/invalidate.rs b/core/crates/layerfs-daemon/src/control/invalidate.rs index 7a7b0dc8b5..19f668df8a 100644 --- a/core/crates/layerfs-daemon/src/control/invalidate.rs +++ b/core/crates/layerfs-daemon/src/control/invalidate.rs @@ -17,6 +17,12 @@ impl Service { shown: ObjectId, installed: ObjectId, ) -> Result<(), Failure> { + // An NFS client holds no invalidation channel: the export closes its + // descriptors of the replaced tree and refreshes the stale inodes the + // client may cache through the mount point. + if self.installed_export(token, workspace, shown, installed)? { + return Ok(()); + } #[cfg(target_os = "linux")] { use super::{kernel::Kernel, registry::bound}; diff --git a/core/crates/layerfs-daemon/src/control/kernel.rs b/core/crates/layerfs-daemon/src/control/kernel.rs index 14d266041f..91252d6110 100644 --- a/core/crates/layerfs-daemon/src/control/kernel.rs +++ b/core/crates/layerfs-daemon/src/control/kernel.rs @@ -29,7 +29,6 @@ pub use super::serving::KernelServing; pub(super) enum Kernel { Unattached, /// One Attach is admitted, or its engine outcome is unknown. - #[cfg(target_os = "linux")] Attaching, #[cfg(target_os = "linux")] Ready(Box), @@ -38,6 +37,12 @@ pub(super) enum Kernel { Leaving(Box), #[cfg(target_os = "linux")] Retained(Box), + /// NFS route: a serving export the host client has mounted. + Exported(Box), + /// NFS route: a terminal operation owns the export. + ExportLeaving(Box), + /// NFS route: export custody a teardown could not finish disposing. + ExportKept(Box), } impl Kernel { /// Maintained observations only: no syscall, SQL job or kernel request. @@ -50,8 +55,10 @@ impl Kernel { }; match self { Self::Unattached => plain(MountPhase::Unattached), - #[cfg(target_os = "linux")] Self::Attaching => plain(MountPhase::Attaching), + Self::Exported(_) | Self::ExportLeaving(_) | Self::ExportKept(_) => { + super::export::block(self).unwrap_or_else(|| plain(MountPhase::Unattached)) + } #[cfg(target_os = "linux")] Self::Ready(attached) => { let facts = attached.session.facts(); diff --git a/core/crates/layerfs-daemon/src/control/mod.rs b/core/crates/layerfs-daemon/src/control/mod.rs index 9d42da651e..87c3bd6073 100644 --- a/core/crates/layerfs-daemon/src/control/mod.rs +++ b/core/crates/layerfs-daemon/src/control/mod.rs @@ -5,6 +5,7 @@ mod cleanup; #[cfg(target_os = "linux")] mod detach; mod dirty; +mod export; mod failure; mod force; mod inspect; @@ -19,6 +20,7 @@ mod serving; mod status; mod types; mod unmount; +pub use export::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; pub use kernel::{KernelConfig, KernelServing}; pub(crate) use observation::{reply_token, request_token}; pub use registry::Service; diff --git a/core/crates/layerfs-daemon/src/control/observation.rs b/core/crates/layerfs-daemon/src/control/observation.rs index b8f0edda08..e403c58c59 100644 --- a/core/crates/layerfs-daemon/src/control/observation.rs +++ b/core/crates/layerfs-daemon/src/control/observation.rs @@ -57,7 +57,11 @@ impl Service { Kernel::Ready(attached) => Some(attached.session.observer()), Kernel::Leaving(leaving) => Some(leaving.observer.clone()), Kernel::Retained(kept) => kept.observer.clone(), - Kernel::Unattached | Kernel::Attaching => None, + Kernel::Unattached + | Kernel::Attaching + | Kernel::Exported(_) + | Kernel::ExportLeaving(_) + | Kernel::ExportKept(_) => None, }) } } diff --git a/core/crates/layerfs-daemon/src/control/operations.rs b/core/crates/layerfs-daemon/src/control/operations.rs index 2d551544ff..b027bcd16c 100644 --- a/core/crates/layerfs-daemon/src/control/operations.rs +++ b/core/crates/layerfs-daemon/src/control/operations.rs @@ -217,6 +217,12 @@ impl Service { run: impl FnOnce(&BoundWorkspace) -> Result>, ) -> Result { let workspace = self.admit(token, Activity::Committing)?; + // NFS route: what the client buffered is written back first, as a + // FUSE write has already reached the Workspace when it returned. + if let Err(failure) = self.flush_export(token) { + self.settle_activity(token, Activity::Idle); + return Err(failure); + } let result = run(&workspace); // An install that replaced what the Workspace showed is invalidated // in the kernel before the answer. @@ -313,6 +319,31 @@ impl Service { }, } } + /// Poison recovery stores this exact owner only; a poisoned registry never + /// authorizes another product operation. + pub(super) fn settle( + &self, + token: WorkspaceToken, + kernel: super::kernel::Kernel, + activity: Activity, + ) { + let mut entries = self.entries.lock().unwrap_or_else(|e| e.into_inner()); + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.kernel = kernel; + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + /// Returns an admitted entry to `activity` when its operation stopped + /// before any effect. + fn settle_activity(&self, token: WorkspaceToken, activity: Activity) { + if let Ok(mut entries) = self.entries.lock() { + if let Ok(binding) = bound_mut(&mut entries, token) { + binding.activity = activity; + binding.epoch = binding.epoch.saturating_add(1); + } + } + } pub(super) fn job( &self, route: layerfs_workspace::Route, diff --git a/core/crates/layerfs-daemon/src/control/registry.rs b/core/crates/layerfs-daemon/src/control/registry.rs index f21131caa2..2bf5e2e910 100644 --- a/core/crates/layerfs-daemon/src/control/registry.rs +++ b/core/crates/layerfs-daemon/src/control/registry.rs @@ -33,6 +33,10 @@ pub struct Service { pub(super) entries: Mutex>, /// Absent for a control-only Service: Attach is then refused before effects. pub(super) kernel: Option>, + /// NFS serving (native route); when present, Attach serves NFS. + pub(super) export: Option>, + /// What the last stopped export gave back, for diagnostics. + pub(super) released: Mutex>, } impl Service { /// Binds control to the already initialized owner and its configured capacity. @@ -43,8 +47,26 @@ impl Service { capacity: owner.configuration().namespaces, entries: Mutex::new(BTreeMap::new()), kernel: None, + export: None, + released: Mutex::new(None), } } + /// Binds control and NFS serving (the native route's mount entry) to the + /// same Store and overlay owner. + pub fn with_export( + store: Arc, + owner: &Owner, + export: Arc, + ) -> Self { + Self { + export: Some(export), + ..Self::new(store, owner) + } + } + /// Whether Attach serves NFS rather than FUSE. + pub fn serves_nfs(&self) -> bool { + self.export.is_some() + } /// Binds control and kernel serving to the same Store and overlay owner. pub fn with_kernel(store: Arc, owner: &Owner, kernel: Arc) -> Self { Self { diff --git a/core/crates/layerfs-daemon/src/control/status.rs b/core/crates/layerfs-daemon/src/control/status.rs index 9b86d52495..43a4a1a30c 100644 --- a/core/crates/layerfs-daemon/src/control/status.rs +++ b/core/crates/layerfs-daemon/src/control/status.rs @@ -44,7 +44,7 @@ impl Service { value.epoch, value.published.clone(), // Copied with the control fields: maintained counters only. - self.kernel.as_ref().map(|_| value.kernel.block()), + (self.kernel.is_some() || self.export.is_some()).then(|| value.kernel.block()), ) }; let observed = self.job( diff --git a/core/crates/layerfs-daemon/src/control/types.rs b/core/crates/layerfs-daemon/src/control/types.rs index 320b574fd1..8d4df91f7e 100644 --- a/core/crates/layerfs-daemon/src/control/types.rs +++ b/core/crates/layerfs-daemon/src/control/types.rs @@ -65,7 +65,6 @@ impl Success { } } impl Failure { - #[cfg(target_os = "linux")] pub(super) fn kernel( code: ControlCode, phase: &'static str, diff --git a/core/crates/layerfs-daemon/src/control/unmount.rs b/core/crates/layerfs-daemon/src/control/unmount.rs index ca1d959a65..dd39b65df0 100644 --- a/core/crates/layerfs-daemon/src/control/unmount.rs +++ b/core/crates/layerfs-daemon/src/control/unmount.rs @@ -17,6 +17,8 @@ pub(super) enum Departure { Unattached, #[cfg(target_os = "linux")] Session(Box), + /// NFS route: the export and the client mount of it. + Export(Box), } impl Service { /// Refuses uncommitted changes with their paths unless `discard`. @@ -30,6 +32,7 @@ impl Service { let (workspace, departure) = self.admit_unmount(token)?; match departure { Departure::Unattached => {} + Departure::Export(exported) => return self.unmount_export(token, workspace, *exported), #[cfg(target_os = "linux")] Departure::Session(attached) => { return self.unmount_kernel(token, workspace, *attached) @@ -46,6 +49,9 @@ impl Service { let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; let value = bound_mut(&mut entries, token)?; // Retained custody answers before ordinary activity admission. + if let Kernel::ExportKept(kept) = &value.kernel { + return Err(Failure::Retained(Box::new(kept.custody(token)))); + } #[cfg(target_os = "linux")] if let Kernel::Retained(kept) = &value.kernel { return Err(Failure::Retained(Box::new(kept.custody(token)))); @@ -53,7 +59,12 @@ impl Service { idle(value)?; let departure = match value.kernel { Kernel::Unattached => Departure::Unattached, - #[cfg(target_os = "linux")] + Kernel::Exported(_) | Kernel::ExportLeaving(_) | Kernel::ExportKept(_) => { + Departure::Export(Box::new(super::export::take( + &mut value.kernel, + layerfs_bridge::control::MountPhase::Probing, + )?)) + } Kernel::Attaching => { return Err(Failure::kernel( layerfs_bridge::control::ControlCode::Unknown, @@ -170,13 +181,16 @@ impl Service { } failure } - /// Attach is a Linux capability; elsewhere it is refused before effects. + /// FUSE Attach is a Linux capability; elsewhere only NFS serving attaches. #[cfg(not(target_os = "linux"))] pub(super) fn attach( &self, - _token: WorkspaceToken, + token: WorkspaceToken, _timing: &layerfs_telemetry::timer::TimingScope<'_, layerfs_telemetry::timer::Active>, ) -> Result { + if let Some(export) = self.export.clone() { + return self.attach_export(&export, token); + } Err(Failure::Rejected( layerfs_bridge::control::ControlCode::Invalid, "kernel serving requires Linux", diff --git a/core/crates/layerfs-nfs/examples/daemon_nfs.rs b/core/crates/layerfs-nfs/examples/daemon_nfs.rs new file mode 100644 index 0000000000..2ce0ea9dd1 --- /dev/null +++ b/core/crates/layerfs-nfs/examples/daemon_nfs.rs @@ -0,0 +1,172 @@ +//! The daemon's control Service with NFS serving, driven by control requests +//! (A3 kernel-client runs). +//! +//! usage: daemon_nfs +//! +//! Initializes a Store from `` (Branch `main`), then runs the +//! Workspace control requests the daemon's control connection answers: +//! Mount and Attach (`BOUND`, `READY `); the client mount, flush +//! and unmount are the helpers named by `LAYERFS_NFS_MOUNT`, +//! `LAYERFS_NFS_FLUSH`, `LAYERFS_NFS_UNMOUNT` and `LAYERFS_NFS_FORCE` +//! (`{port}`, `{dir}` replaced), as in the daemon. `LAYERFS_NFS_LOOKUPS` sets +//! the export's reference bound. Then, whenever a trigger file appears: +//! `.status` prints the Status kernel block and the export's +//! holdings (`STATUS …`); `.commit` runs a control Commit +//! (`COMMITTED …`) and serves an independent view of the committed Branch +//! read-only on `` (`VIEW …`); `.unmount` runs a +//! control Unmount without discard (`UNMOUNT …`), and on success stops. +#[allow(dead_code)] +#[path = "../tests/support/rig.rs"] +mod rig; +use layerfs_bridge::control::{CommitControl, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter}; +use layerfs_history::WorkspaceId; +use std::{net::TcpListener, path::PathBuf, sync::Arc, time::Duration}; + +fn main() { + let arguments: Vec = std::env::args().collect(); + if arguments.len() != 7 { + eprintln!("usage: daemon_nfs "); + std::process::exit(2); + } + let source = PathBuf::from(&arguments[1]); + let work = PathBuf::from(&arguments[2]); + let mounts = PathBuf::from(&arguments[3]); + let view_port: u16 = arguments[4].parse().expect("view port"); + let uid: u32 = arguments[5].parse().expect("uid"); + let gid: u32 = arguments[6].parse().expect("gid"); + let trigger = |name: &str| PathBuf::from(format!("{}.{name}", work.display())); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(600)); + println!("INIT entries={}", rig.entries); + let mut mounter = Mounter::host(); + let helper = |name: &str| { + std::env::var(name) + .ok() + .and_then(|line| Helper::words(&line)) + }; + if let Some(mount) = helper("LAYERFS_NFS_MOUNT") { + mounter.mount = mount; + } + if let Some(unmount) = helper("LAYERFS_NFS_UNMOUNT") { + mounter.unmount = unmount; + } + if let Some(force) = helper("LAYERFS_NFS_FORCE") { + mounter.force = force; + } + mounter.flush = helper("LAYERFS_NFS_FLUSH"); + let mut config = ExportServingConfig::new(mounts, uid, gid, mounter); + if let Some(bound) = std::env::var("LAYERFS_NFS_LOOKUPS") + .ok() + .and_then(|value| value.parse().ok()) + { + config.lookups_held = bound; + } + let service = rig.service(Arc::new(ExportServing::start(config))); + let workspace = WorkspaceId::from_authority([41; 32]).unwrap(); + let token = match service + .execute_control(&Request::Mount { + workspace, + branch: rig.branch, + }) + .expect("Mount") + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + println!("BOUND {token:?}"); + let ready = match service.execute_control(&Request::Attach(token)) { + Ok(success) => match success.reply { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }, + Err(failure) => { + println!("ATTACH_FAILED {failure:?}"); + std::process::exit(1); + } + }; + println!("READY {} {}", ready.receipt.mount_id, ready.directory); + let mut views = Vec::new(); + loop { + if trigger("status").exists() { + match service + .execute_control(&Request::Status(token)) + .map(|s| s.reply) + { + Ok(Reply::Status(status)) => println!( + "STATUS phase={:?} work={:?} dirty_inodes={:?} held={:?}", + status.kernel.as_ref().map(|k| k.phase), + status.kernel.as_ref().and_then(|k| k.work), + status.local.as_ref().map(|l| l.dirty_inodes), + service.export_held(token) + ), + other => println!("STATUS {other:?}"), + } + drop(std::fs::remove_file(trigger("status"))); + } + if trigger("commit").exists() { + let at = std::time::Instant::now(); + let done = service.execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("a3 over nfs".into()), + })); + let ms = at.elapsed().as_millis(); + match done { + Ok(success) => { + let kind = match &success.reply { + Reply::Committed(_) => "Committed", + Reply::MergeCommitted(_) => "MergeCommitted", + Reply::Conflict(_) => "Conflict", + _ => "Other", + }; + println!("COMMITTED {kind} ms={ms}"); + let mut authority = [90; 32]; + authority[0] = views.len() as u8; + let view = rig.view(authority); + let port = view_port + views.len() as u16; + let viewer = rig.serve_view( + TcpListener::bind(("127.0.0.1", port)).expect("bind view"), + &view, + uid, + gid, + ); + println!("VIEW {}", viewer.local_addr()); + views.push((viewer, view)); + } + Err(failure) => println!("COMMIT_FAILED ms={ms} {failure:?}"), + } + drop(std::fs::remove_file(trigger("commit"))); + } + if trigger("unmount").exists() { + drop(std::fs::remove_file(trigger("unmount"))); + let held = service.export_held(token); + match service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: false, + })) { + Ok(success) => { + println!("UNMOUNT {:?} held_before={held:?}", success.reply); + if matches!(success.reply, Reply::Unmounted(_)) { + break; + } + } + Err(failure) => println!("UNMOUNT_FAILED {failure:?}"), + } + } + std::thread::sleep(Duration::from_millis(200)); + } + let released = service.export_released(token).unwrap_or_default(); + println!("RELEASED {released:?}"); + println!( + "DIRECTORY_GONE {}", + !std::path::Path::new(&ready.directory).exists() + ); + for (viewer, view) in views { + viewer.stop(); + rig.close_view(view); + } + drop(service); + let (_, outstanding) = rig.stop(released); + println!("STOPPED outstanding_readers={outstanding}"); +} diff --git a/core/crates/layerfs-nfs/tests/daemon.rs b/core/crates/layerfs-nfs/tests/daemon.rs new file mode 100644 index 0000000000..eacaf9e35b --- /dev/null +++ b/core/crates/layerfs-nfs/tests/daemon.rs @@ -0,0 +1,292 @@ +//! A3: the daemon's control Service with NFS serving (the native route's +//! mount entry). Every step is a control request through `execute_control`, +//! the function the daemon's control connection answers with: Mount (bind), +//! Attach (export + client mount helper), Status, Commit (client flush, then +//! the unchanged product Commit), Unmount (dirty refusal, reversible busy, +//! terminal) and ForceUnmount. The client mount itself is a recording helper +//! here (this host's kernel has no NFSv3 client); the protocol client of the +//! tests plays the kernel. The kernel-client run is in the A3 VM scripts. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::{Client, NOENT}; +use layerfs_bridge::control::{ + CommitControl, MountPhase, ReadyMount, Reply, Request, UnmountControl, WorkspaceToken, +}; +use layerfs_daemon::control::{ + ExportServing, ExportServingConfig, Failure, Helper, Mounter, Service, +}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + net::{Ipv4Addr, SocketAddr}, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-d{}-{label}", std::process::id())) +} +/// A helper that appends ` ` to `log`; it answers "busy" +/// (exit 32) while `busy` exists. +fn recording(what: &str, log: &Path, busy: Option<&Path>) -> Helper { + let refuse = busy + .map(|flag| { + format!( + "if [ -e {} ]; then echo 'umount: target is busy' >&2; exit 32; fi; ", + flag.display() + ) + }) + .unwrap_or_default(); + Helper(vec![ + "sh".into(), + "-c".into(), + format!("{refuse}echo {what} {{port}} {{dir}} >> {}", log.display()), + ]) +} +fn bind(service: &Service, tag: u8, branch: layerfs_history::BranchId) -> WorkspaceToken { + let workspace = WorkspaceId::from_authority([tag; 32]).unwrap(); + match service + .execute_control(&Request::Mount { workspace, branch }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + } +} +fn attach(service: &Service, token: WorkspaceToken) -> ReadyMount { + match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + } +} +fn phase(service: &Service, token: WorkspaceToken) -> MountPhase { + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => status.kernel.expect("kernel block").phase, + other => panic!("{other:?}"), + } +} +fn commit(service: &Service, token: WorkspaceToken) -> Reply { + service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("over nfs".into()), + })) + .unwrap() + .reply +} +fn unmount(service: &Service, token: WorkspaceToken, discard: bool) -> Result { + service + .execute_control(&Request::Unmount(UnmountControl { token, discard })) + .map(|success| success.reply) +} +fn connect(ready: &ReadyMount) -> Client { + let port = u16::try_from(ready.receipt.mount_id).unwrap(); + Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), UID, GID) +} + +#[test] +fn workspace_control_over_the_nfs_route() { + let source = temp("src"); + let work = temp("work"); + let mounts = temp("mounts"); + let log = temp("helpers.log"); + let busy = temp("busy"); + for path in [&source, &work, &mounts] { + drop(fs::remove_dir_all(path)); + } + drop(fs::remove_file(&log)); + fs::create_dir_all(source.join("src")).unwrap(); + fs::write(source.join("README.md"), b"base readme\n").unwrap(); + fs::write(source.join("src/lib.rs"), b"pub fn base() {}\n").unwrap(); + fs::write(source.join("gone.txt"), b"to be removed\n").unwrap(); + fs::create_dir_all(&mounts).unwrap(); + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let mounter = Mounter { + mount: recording("mount", &log, None), + unmount: recording("unmount", &log, Some(&busy)), + force: recording("force", &log, None), + flush: Some(recording("flush", &log, None)), + }; + let export = Arc::new(ExportServing::start(ExportServingConfig::new( + mounts.clone(), + UID, + GID, + mounter, + ))); + let service = rig.service(export); + assert!(service.serves_nfs()); + + // Mount + Attach: Ready with an NFS receipt; the helper mounted it. + let token = bind(&service, 31, rig.branch); + assert_eq!(phase(&service, token), MountPhase::Unattached); + let ready = attach(&service, token); + assert_eq!(ready.receipt.abi_major, 0); + assert_eq!( + ready.directory, + mounts.join(token.namespace.to_string()).to_str().unwrap() + ); + assert!(Path::new(&ready.directory).is_dir()); + assert_eq!(phase(&service, token), MountPhase::Ready); + let port = ready.receipt.mount_id; + let logged = fs::read_to_string(&log).unwrap(); + assert_eq!(logged, format!("mount {port} {}\n", ready.directory)); + // A second Attach of the same Workspace is refused before effects. + assert!(service.execute_control(&Request::Attach(token)).is_err()); + + // Writes through the export. + let mut nfs = connect(&ready); + let root = nfs.mnt("/"); + let made = nfs.touch(&root, b"made.txt", 0o644).unwrap(); + nfs.write_all(&made.handle, 0, b"made over nfs\n", 8); + let dir = nfs.mkdir(&root, b"newdir", 0o755).unwrap(); + let inner = nfs.touch(&dir.handle, b"inner.bin", 0o600).unwrap(); + nfs.write_all(&inner.handle, 0, &vec![7u8; 300_000], 65536); + nfs.remove(&root, b"gone.txt").unwrap(); + let (src, _) = nfs.lookup(&root, b"src").unwrap(); + nfs.rename(&src, b"lib.rs", &dir.handle, b"lib.rs").unwrap(); + nfs.symlink(&root, b"link", b"newdir/lib.rs").unwrap(); + match service + .execute_control(&Request::Status(token)) + .unwrap() + .reply + { + Reply::Status(status) => { + let block = status.kernel.unwrap(); + assert!(block.work.unwrap().served > 10, "{block:?}"); + assert!(status.local.unwrap().dirty_inodes > 0); + } + other => panic!("{other:?}"), + } + // Unmount refuses uncommitted changes, naming them; nothing closes. + match unmount(&service, token, false).unwrap() { + Reply::Dirty(report) => { + assert!(report.paths.iter().any(|p| p == "/made.txt"), "{report:?}") + } + other => panic!("{other:?}"), + } + assert_eq!(phase(&service, token), MountPhase::Ready); + + // Commit: the flush helper ran first, then the product Commit. + match commit(&service, token) { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + assert!(fs::read_to_string(&log) + .unwrap() + .contains(&format!("flush {port} "))); + // The export keeps serving after the Commit. + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + + // An independent view of the committed Branch. + let view = rig.view([92; 32]); + let viewer = rig.serve_view( + std::net::TcpListener::bind("127.0.0.1:0").unwrap(), + &view, + UID, + GID, + ); + let mut reader = Client::connect(viewer.local_addr(), UID, GID); + let view_root = reader.mnt("/"); + let (handle, _) = reader.walk_to(&view_root, "made.txt").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"made over nfs\n"); + let (handle, attr) = reader.walk_to(&view_root, "newdir/inner.bin").unwrap(); + assert_eq!((attr.size, attr.mode & 0o777), (300_000, 0o600)); + assert_eq!(reader.read_all(&handle, 65536), vec![7u8; 300_000]); + let (handle, _) = reader.walk_to(&view_root, "newdir/lib.rs").unwrap(); + assert_eq!(reader.read_all(&handle, 4096), b"pub fn base() {}\n"); + assert_eq!( + reader.walk_to(&view_root, "gone.txt").map(|_| ()), + Err(NOENT) + ); + assert_eq!( + reader.walk_to(&view_root, "src/lib.rs").map(|_| ()), + Err(NOENT) + ); + let (handle, _) = reader.walk_to(&view_root, "link").unwrap(); + assert_eq!(reader.readlink(&handle).unwrap(), b"newdir/lib.rs"); + drop(reader); + viewer.stop(); + rig.close_view(view); + + // A busy client mount is the reversible answer: still Ready, serving. + fs::write(&busy, b"").unwrap(); + let refused = unmount(&service, token, true).unwrap_err(); + assert!(format!("{refused:?}").contains("busy"), "{refused:?}"); + assert_eq!(phase(&service, token), MountPhase::Ready); + assert_eq!(nfs.read_all(&made.handle, 4096), b"made over nfs\n"); + fs::remove_file(&busy).unwrap(); + + // Terminal unmount: client unmount, export stopped, engine mount + // revoked, directory removed, Workspace closed. + let held = service.export_held(token).unwrap(); + assert_eq!( + unmount(&service, token, true).unwrap(), + Reply::Unmounted(token) + ); + assert!(!Path::new(&ready.directory).exists()); + let released = service.export_released(token).unwrap(); + assert_eq!(released.lookups, held.lookups); + assert_eq!(released.retained, 0); + drop(nfs); + assert!(service.execute_control(&Request::Status(token)).is_err()); + + // ForceUnmount of a second Workspace with uncommitted changes. + let second = bind(&service, 32, rig.branch); + let ready2 = attach(&service, second); + let mut nfs = connect(&ready2); + let root = nfs.mnt("/"); + nfs.touch(&root, b"lost.txt", 0o644).unwrap(); + match service + .execute_control(&Request::ForceUnmount { + token: second, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => assert_eq!(done.token, second), + other => panic!("{other:?}"), + } + assert!(!Path::new(&ready2.directory).exists()); + drop(nfs); + let logged = fs::read_to_string(&log).unwrap(); + println!("HELPERS\n{logged}"); + assert_eq!( + logged.lines().filter(|l| l.starts_with("mount ")).count(), + 2 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("unmount ")).count(), + 1 + ); + assert_eq!( + logged.lines().filter(|l| l.starts_with("force ")).count(), + 1 + ); + + drop(service); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + for path in [&source, &mounts] { + fs::remove_dir_all(path).unwrap(); + } + fs::remove_file(&log).unwrap(); +} diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index b927b00c53..5f4747f486 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -61,7 +61,7 @@ "layerfs-store-service": {"layerfs-overlay", "layerfs-workspace", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, # Concrete Store open/install belongs to application composition. The # provider-independent daemon store/ subtree has a narrower source boundary. - "layerfs-daemon": {"layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, + "layerfs-daemon": {"layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-nfs", "layerfs-bridge", "layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry"}, "layerfs-api": {"layerfs-content", "layerfs-storage", "layerfs-history", "layerfs-persistence", "layerfs-telemetry", "layerfs-bridge", "layerfs-project", "layerfs-sandbox", "layerfs-store-service"}, "layerfs-bridge": {"layerfs-content", "layerfs-history"}, "layerfs-sandbox": set(), From 6c8ca0e9684a4b3011e72c3762aa4f5b3948c7a9 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Fri, 23 Oct 2026 00:30:00 +0800 Subject: [PATCH 43/64] feat(sandbox): local Sandboxes on macOS and the SDK's native route (LAYERFS_SANDBOX=native) The local runtime (#402) reads process facts from /proc on Linux. On macOS backend/local/darwin.rs supplies them instead: - Process identity, group, state and start time from proc_pidinfo (PROC_PIDTBSDINFO) and the process list from proc_listallpids, both from libc (already a dependency). /bin/ps is setuid root, and Seatbelt refuses to exec it, so dsh's sandboxed bash could not start a Sandbox through it. - Boot identity from sysctl kern.bootsessionuuid (kern.boottime as the fallback), mounts from /sbin/mount, forced detach with /sbin/umount -f; fixed paths, C locale, one call each. - A native Sandbox runs its commands as the current user. layerfs-sandbox moves from #![forbid(unsafe_code)] to #![deny(unsafe_code)] with the two darwin.rs functions as the only #[allow(unsafe_code)], the same shape as layerfs-persistence's audited seal module. The boundary guard names src/backend/local/darwin.rs as the crate's one audited unsafe module, and its self-test checks the new root attribute and that a sibling module still may not use unsafe. This needs the owner's authorization; the alternative is the libproc crate already in Cargo.lock (optional dependency of telemetry), which needs bindgen and libclang at build time. SDK: LAYERFS_SANDBOX=native selects a local daemon whose Workspaces this host's NFS client mounts (LAYERFS_KERNEL=nfs), with commands run as this user: no root, no Docker. LAYERFS_NFS_* reach the daemon unchanged. sdk/js/src/client.js accepts "native" beside "docker" and "local". sdk/js/README.md gains a "Native Sandboxes (macOS)" section. tests/unit/runtime.rs adds the native cases; no existing assertion changed. On macOS a local daemon and its commands start with this process's own PATH: a native command is this user's own tool (node in ~/.local/bin, Homebrew in /opt/homebrew/bin) and no image supplies it, so the fixed container search path left `node` unfound (exit 127) in a native Workspace. Without a PATH, or with an empty one, it is the container search path plus /opt/homebrew/bin. Linux keeps the container's fixed search path, whatever this process has. tests/local_search_path.rs (macOS) and a local_runtime case (Linux) pin both. Production LOC: 114,316 -> 114,591 (delta +275) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 757e4480c. --- .../src/backend/local/create.rs | 59 ++++-- .../src/backend/local/darwin.rs | 194 ++++++++++++++++++ .../layerfs-sandbox/src/backend/local/exec.rs | 16 +- .../src/backend/local/groups.rs | 5 + .../layerfs-sandbox/src/backend/local/mod.rs | 15 +- .../src/backend/local/mounts.rs | 30 ++- .../src/backend/local/process.rs | 11 +- .../src/backend/local/search.rs | 26 +++ .../crates/layerfs-sandbox/src/backend/mod.rs | 2 +- core/crates/layerfs-sandbox/src/lib.rs | 4 +- .../layerfs-sandbox/tests/local_runtime.rs | 24 +++ .../tests/local_search_path.rs | 113 ++++++++++ core/tools/check_product_boundary.py | 8 +- core/tools/test_check_product_boundary.py | 6 +- sdk/js/README.md | 37 +++- sdk/js/native/src/discovery.rs | 51 +++-- sdk/js/native/src/lib.rs | 2 +- sdk/js/native/src/local.rs | 44 +++- sdk/js/native/src/runtime.rs | 68 +++--- sdk/js/native/src/sandbox.rs | 4 +- sdk/js/native/tests/unit/runtime.rs | 24 ++- sdk/js/src/client.js | 14 +- 22 files changed, 663 insertions(+), 94 deletions(-) create mode 100644 core/crates/layerfs-sandbox/src/backend/local/darwin.rs create mode 100644 core/crates/layerfs-sandbox/src/backend/local/search.rs create mode 100644 core/crates/layerfs-sandbox/tests/local_search_path.rs diff --git a/core/crates/layerfs-sandbox/src/backend/local/create.rs b/core/crates/layerfs-sandbox/src/backend/local/create.rs index b352dadcdc..6e930c9658 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/create.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/create.rs @@ -3,6 +3,7 @@ use super::{ process::LocalProcess, sandbox::{Sandbox, LOG}, + search::search_path, }; use crate::{ContainerId, RuntimeError}; use nix::fcntl::OFlag; @@ -23,14 +24,14 @@ use std::{ const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; /// Pause between two reads of a starting daemon's output. const PAUSE: Duration = Duration::from_millis(10); -/// The daemon's environment: a search path and nothing of this process's. -const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; /// A local runtime root: one directory per Sandbox under it. #[derive(Clone, Debug)] pub struct Local { root: PathBuf, name: Option, + /// Variables the daemon starts with besides `PATH`. + environment: Vec<(String, String)>, } /// Where one Sandbox keeps its parts, all under its directory. @@ -80,6 +81,30 @@ impl Local { /// another identity, so one closed directory on the way, such as a 0700 /// `/root`, would fail every command at spawn). pub fn new(root: impl AsRef) -> Result { + let local = Self::native(root.as_ref())?; + let root = root.as_ref(); + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } + Ok(local) + } + /// Selects `root` for the native route, whose commands run as this user + /// (no other identity): the root's own checks as in [`Self::new`], but + /// the directories above it need not be traversable by others (a macOS + /// home is 0750). + pub fn native(root: impl AsRef) -> Result { let root = root.as_ref(); if !root.is_absolute() || root @@ -98,26 +123,19 @@ impl Local { "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", )); } - // every directory on the path as given and on the path it resolves to - let resolved = fs::canonicalize(root)?; - for directory in root.ancestors().chain(resolved.ancestors()) { - let metadata = fs::metadata(directory)?; - if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { - return Err(RuntimeError::Io(io::Error::new( - io::ErrorKind::PermissionDenied, - format!( - "{} is not traversable by others (o+x); local Sandbox commands run as \ - another identity and must pass it to reach their Workspace", - directory.display() - ), - ))); - } - } Ok(Self { root: root.to_owned(), name: None, + environment: Vec::new(), }) } + /// Starts the next daemon with `environment` besides its search path (the + /// native route's `LAYERFS_KERNEL=nfs` and its mount helpers). Nothing + /// else of this process's environment reaches it. + pub fn environment(mut self, environment: Vec<(String, String)>) -> Self { + self.environment = environment; + self + } /// Names the next Sandbox's directory `name`, under the Docker name rule /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at /// most 128), so it is one plain path component. @@ -154,8 +172,8 @@ impl Local { /// Makes the named Sandbox's directory, writes `config` (the encoded /// daemon setup naming [`Self::paths`]) 0600, spawns `executable /// --config ` in its own process group with no inherited - /// environment, and waits up to `wait` for its listener line. Once - /// anything is made, a failure returns it. + /// environment but the search path, and waits up to `wait` for its + /// listener line. Once anything is made, a failure returns it. pub fn create( &self, executable: &Path, @@ -204,7 +222,8 @@ impl Local { .arg("--config") .arg(&paths.config) .env_clear() - .env("PATH", PATH) + .env("PATH", search_path()) + .envs(self.environment.iter().map(|(k, v)| (k, v))) .current_dir(&paths.directory) .stdin(Stdio::null()) .stdout(output) diff --git a/core/crates/layerfs-sandbox/src/backend/local/darwin.rs b/core/crates/layerfs-sandbox/src/backend/local/darwin.rs new file mode 100644 index 0000000000..151357084b --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/darwin.rs @@ -0,0 +1,194 @@ +//! macOS facts the local runtime reads from `/proc` on Linux: a process's +//! state, group and start time (`proc_pidinfo`, a system call: `ps` is setuid +//! root, which a Seatbelt-sandboxed caller may not execute), this boot's +//! identity (`sysctl`), the mount table (`mount`) and a forced unmount +//! (`umount -f`). Each tool is one run with a fixed path and the C locale, +//! read once; nothing is retried. +use super::process::Stat; +use nix::libc; +use std::{ + io, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::OnceLock, +}; + +const SYSCTL: &str = "/usr/sbin/sysctl"; +const MOUNT: &str = "/sbin/mount"; +const UMOUNT: &str = "/sbin/umount"; + +/// One tool's standard output, or its failure with what it said. +fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { + let out = Command::new(program) + .args(args) + .env_clear() + .env("LC_ALL", "C") + .env("PATH", "/usr/bin:/bin:/usr/sbin:/sbin") + .stdin(Stdio::null()) + .output()?; + Ok(( + out.status.success(), + String::from_utf8_lossy(&out.stdout).into_owned(), + )) +} + +fn invalid(what: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, what) +} + +/// State, process group and start time of `pid` from the kernel +/// (`proc_pidinfo`, `PROC_PIDTBSDINFO`); `NotFound` when no such process +/// exists. Not `ps`: it is setuid root, and a sandboxed caller (dsh's +/// Seatbelt profile for agents' bash) may not execute it. +#[allow(unsafe_code)] +pub(super) fn stat(pid: u32) -> io::Result { + let id = i32::try_from(pid).map_err(|_| invalid("process id width"))?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: the buffer is a proc_bsdinfo of exactly `size` bytes. + let got = unsafe { + libc::proc_pidinfo( + id, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if got <= 0 { + let error = io::Error::last_os_error(); + return Err(match error.raw_os_error() { + Some(libc::ESRCH) | Some(0) | None => io::Error::from(io::ErrorKind::NotFound), + _ => error, + }); + } + if got != size { + return Err(invalid("proc_bsdinfo size")); + } + // SAFETY: the kernel filled all `size` bytes. + let info = unsafe { info.assume_init() }; + if info.pbi_pid != pid { + return Err(invalid("proc_bsdinfo pid")); + } + Ok(from_info(&info)) +} + +/// `ps`'s state letters for `p_stat`, and the start time in microseconds: +/// equal for the same process, different for another under the same id. +fn from_info(info: &libc::proc_bsdinfo) -> Stat { + let state = match info.pbi_status { + 1 => b'I', + 2 => b'R', + 3 => b'S', + 4 => b'T', + libc::SZOMB => b'Z', + _ => b'?', + }; + Stat { + state, + group: info.pbi_pgid, + start: info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec, + } +} + +/// Every process now, with its stat (`proc_listallpids`); a process that +/// ends or is not readable between the listing and its read is left out. +#[allow(unsafe_code)] +pub(super) fn processes() -> io::Result> { + // SAFETY: a null buffer asks only for the count. + let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + let mut pids = vec![0 as libc::pid_t; count as usize + 64]; + let bytes = (pids.len() * std::mem::size_of::()) as libc::c_int; + // SAFETY: `pids` holds `bytes` bytes of pid_t. + let listed = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + if listed < 0 { + return Err(io::Error::last_os_error()); + } + pids.truncate((listed as usize).min(pids.len())); + Ok(pids + .into_iter() + .filter(|pid| *pid > 0) + .filter_map(|pid| { + let pid = pid as u32; + stat(pid).ok().map(|stat| (pid, stat)) + }) + .collect()) +} + +/// This boot's identity as a number, read once per process: +/// `kern.bootsessionuuid`, else `kern.boottime` (seconds and microseconds). +pub(super) fn boot() -> io::Result { + static BOOT: OnceLock = OnceLock::new(); + if let Some(boot) = BOOT.get() { + return Ok(*boot); + } + let uuid = output(SYSCTL, &["-n", "kern.bootsessionuuid"]) + .ok() + .and_then(|(ran, text)| if ran { uuid(&text) } else { None }); + let boot = match uuid { + Some(boot) => boot, + None => { + let (_, text) = output(SYSCTL, &["-n", "kern.boottime"])?; + boottime(&text).ok_or_else(|| invalid("kern.boottime"))? + } + }; + Ok(*BOOT.get_or_init(|| boot)) +} + +fn uuid(text: &str) -> Option { + let hex: String = text.trim().chars().filter(|c| *c != '-').collect(); + (hex.len() == 32) + .then(|| u128::from_str_radix(&hex, 16).ok()) + .flatten() +} + +/// `{ sec = 1729500000, usec = 123456 } Tue Oct 21 ...` as sec * 10^6 + usec. +fn boottime(text: &str) -> Option { + let field = |name: &str| -> Option { + let after = text.split(name).nth(1)?; + let digits: String = after + .trim_start_matches([' ', '=']) + .chars() + .take_while(char::is_ascii_digit) + .collect(); + digits.parse().ok() + }; + Some(field("sec")? * 1_000_000 + field("usec")?) +} + +/// Every mount point now, from `mount`'s ` on ()`. +pub(super) fn mount_points() -> io::Result> { + let (ran, text) = output(MOUNT, &[])?; + if !ran { + return Err(io::Error::other("mount listing failed")); + } + Ok(text.lines().filter_map(mount_point).collect()) +} + +fn mount_point(line: &str) -> Option { + let (_, rest) = line.split_once(" on ")?; + let (point, _) = rest.rsplit_once(" (")?; + Some(PathBuf::from(point)) +} + +/// Forces the mount at `point` off once; a point no longer mounted is done. +pub(super) fn force_unmount(point: &Path) -> io::Result<()> { + let out = Command::new(UMOUNT) + .arg("-f") + .arg(point) + .env_clear() + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + if out.status.success() || !mount_points()?.iter().any(|p| p == point) { + return Ok(()); + } + Err(io::Error::other(format!( + "umount -f {}: {}", + point.display(), + String::from_utf8_lossy(&out.stderr).trim() + ))) +} diff --git a/core/crates/layerfs-sandbox/src/backend/local/exec.rs b/core/crates/layerfs-sandbox/src/backend/local/exec.rs index ec099ddb89..1bd7e47a4c 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/exec.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/exec.rs @@ -1,9 +1,10 @@ //! One ordinary command to its end in a local Sandbox: the command identity, -//! the Workspace directory and a search path; nothing of this process's -//! environment, no shell of its own and no implicit timeout. Each command -//! leads a process group of its own, named in the Sandbox until it is empty, -//! so what it leaves running ends with the Sandbox. -use super::{groups, process, process::LocalProcess, sandbox::Sandbox}; +//! the Workspace directory and a search path (on macOS this process's own); +//! nothing else of this process's environment, no shell of its own and no +//! implicit timeout. Each command leads a process group of its own, named in +//! the Sandbox until it is empty, so what it leaves running ends with the +//! Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox, search::search_path}; use crate::{backend::docker::ExecRequest, RuntimeError, WireFailure}; use nix::sys::signal::{killpg, Signal}; use std::{ @@ -13,9 +14,6 @@ use std::{ process::{Command, Stdio}, }; -/// The search path a command starts with, before its own environment. -const PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; - /// One command's end: its exit code (128 + the signal for one a signal /// ended, as the Engine reports it), its process id (diagnostic only) and the /// bytes each stream delivered. @@ -45,7 +43,7 @@ impl Sandbox { command .args(&request.arguments[1..]) .env_clear() - .env("PATH", PATH) + .env("PATH", search_path()) .current_dir(Path::new(&request.directory)) // std drops supplementary groups (setgroups 0) when it sets a uid as root .uid(request.identity.uid) diff --git a/core/crates/layerfs-sandbox/src/backend/local/groups.rs b/core/crates/layerfs-sandbox/src/backend/local/groups.rs index e307f7e763..95a15fb706 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/groups.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/groups.rs @@ -124,6 +124,11 @@ fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { /// Every process of this kernel now, with its stat. One that ends while /// being read is left out. +#[cfg(target_os = "macos")] +fn processes() -> io::Result> { + super::darwin::processes() +} +#[cfg(target_os = "linux")] fn processes() -> io::Result> { let mut table = Vec::new(); for item in fs::read_dir("/proc")? { diff --git a/core/crates/layerfs-sandbox/src/backend/local/mod.rs b/core/crates/layerfs-sandbox/src/backend/local/mod.rs index 870b54c0fd..52696d4e5e 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/mod.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/mod.rs @@ -6,14 +6,25 @@ //! kernel's processes, network, users and filesystem with everything else on //! it. Commands still run as their own nonroot identity, and what they leave //! running ends with the Sandbox. One attempt per operation, as in the Docker -//! runtime; nothing is retried. Linux only: it reads `/proc` and detaches -//! mounts. +//! runtime; nothing is retried. Linux reads `/proc` and detaches mounts; +//! macOS (the native route, whose daemon serves NFS) reads the same facts +//! with `proc_pidinfo`, `sysctl` and `mount`, and forces mounts off with +//! `umount -f`. +//! +//! The daemon and its commands start with one search path and nothing else of +//! this process's environment: on Linux a container's, whatever this process +//! has; on macOS this process's own `PATH`, since a native command is this +//! user's own tool (Homebrew's `/opt/homebrew/bin`, `~/.local/bin`, a version +//! manager's `node`) and no image supplies it. mod create; +#[cfg(target_os = "macos")] +mod darwin; mod exec; mod groups; mod mounts; mod process; mod sandbox; +mod search; pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; pub use exec::LocalExit; pub use process::LocalProcess; diff --git a/core/crates/layerfs-sandbox/src/backend/local/mounts.rs b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs index 290d94beaf..ed5415f2d5 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/mounts.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/mounts.rs @@ -1,6 +1,8 @@ //! Kernel mounts left under a local Sandbox's directory once its daemon is //! gone. A container's mounts end with its namespace; a local daemon's stay -//! in this one ("transport endpoint is not connected") until detached. +//! in this one ("transport endpoint is not connected") until detached. On +//! macOS the table is `mount`'s output and a mount is forced off with +//! `umount -f`. use std::{ io, path::{Path, PathBuf}, @@ -8,6 +10,16 @@ use std::{ /// Mount points at or under `directory`, deepest first, from this /// process's mount table. +#[cfg(target_os = "macos")] +pub(super) fn under(directory: &Path) -> io::Result> { + let mut found: Vec = (super::darwin::mount_points()?.into_iter()) + .filter(|point| point.starts_with(directory)) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} +#[cfg(target_os = "linux")] pub(super) fn under(directory: &Path) -> io::Result> { let table = std::fs::read_to_string("/proc/self/mountinfo")?; let mut found = Vec::new(); @@ -26,6 +38,7 @@ pub(super) fn under(directory: &Path) -> io::Result> { Ok(found) } +#[cfg(target_os = "linux")] fn unescape(text: &str) -> String { let bytes = text.as_bytes(); let mut out = Vec::with_capacity(bytes.len()); @@ -53,9 +66,24 @@ fn unescape(text: &str) -> String { String::from_utf8_lossy(&out).into_owned() } +/// Unmounts every mount under `directory` once with `umount -f` (a dead +/// NFS export's mount is forced off; macOS has no lazy detach), then checks +/// that none is left. +#[cfg(target_os = "macos")] +pub(super) fn detach(directory: &Path) -> io::Result<()> { + for point in under(directory)? { + super::darwin::force_unmount(&point)?; + } + if under(directory)?.is_empty() { + Ok(()) + } else { + Err(io::Error::other("a mount under the local Sandbox remains")) + } +} /// Detaches every mount under `directory` once (lazy unmount: a process /// still inside keeps its view; new lookups no longer reach it), then checks /// that none is left. +#[cfg(target_os = "linux")] pub(super) fn detach(directory: &Path) -> io::Result<()> { use nix::mount::{umount2, MntFlags}; for point in under(directory)? { diff --git a/core/crates/layerfs-sandbox/src/backend/local/process.rs b/core/crates/layerfs-sandbox/src/backend/local/process.rs index d56fee6882..88ea62d448 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/process.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/process.rs @@ -1,8 +1,13 @@ //! A local process named by its id, kernel start time and boot, so a reused //! id, or the same id and start time after a reboot or snapshot restore, is -//! never taken for the process that was started. +//! never taken for the process that was started. On macOS the same three +//! facts come from `proc_pidinfo` and `sysctl` (see `darwin.rs`). +#[cfg(target_os = "macos")] +pub(super) use super::darwin::{boot, stat}; use nix::{sys::signal::Signal, unistd::Pid}; -use std::{io, sync::OnceLock}; +use std::io; +#[cfg(target_os = "linux")] +use std::sync::OnceLock; /// One started process: its id, its start time in clock ticks since boot /// (`/proc//stat` field 22) and the boot it started in @@ -61,6 +66,7 @@ impl LocalProcess { /// This boot's id (`/proc/sys/kernel/random/boot_id`, a UUID) as a number, /// read once per process. +#[cfg(target_os = "linux")] pub(super) fn boot() -> io::Result { static BOOT: OnceLock = OnceLock::new(); if let Some(boot) = BOOT.get() { @@ -92,6 +98,7 @@ impl Stat { /// State, process group and start time of `pid`. The command name may hold /// spaces and parentheses, so fields are counted after its last `)`. +#[cfg(target_os = "linux")] pub(super) fn stat(pid: u32) -> io::Result { let text = std::fs::read_to_string(format!("/proc/{pid}/stat"))?; let invalid = || io::Error::new(io::ErrorKind::InvalidData, "process stat fields"); diff --git a/core/crates/layerfs-sandbox/src/backend/local/search.rs b/core/crates/layerfs-sandbox/src/backend/local/search.rs new file mode 100644 index 0000000000..ef61ffbd6a --- /dev/null +++ b/core/crates/layerfs-sandbox/src/backend/local/search.rs @@ -0,0 +1,26 @@ +//! The search path a local daemon and its commands start with: a container's +//! on Linux, this process's own on macOS (the native route). +use std::ffi::OsString; + +/// A container's search path: what a local daemon and its commands start with +/// on Linux, where the local runtime stands in for a container. +const CONTAINER_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on Linux [`CONTAINER_PATH`], whatever this +/// process has. +#[cfg(not(target_os = "macos"))] +pub(super) fn search_path() -> OsString { + OsString::from(CONTAINER_PATH) +} + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on macOS this process's own `PATH`; without +/// one, or an empty one, [`CONTAINER_PATH`] and Homebrew's `/opt/homebrew/bin`. +#[cfg(target_os = "macos")] +pub(super) fn search_path() -> OsString { + match std::env::var_os("PATH") { + Some(path) if !path.is_empty() => path, + _ => OsString::from(format!("{CONTAINER_PATH}:/opt/homebrew/bin")), + } +} diff --git a/core/crates/layerfs-sandbox/src/backend/mod.rs b/core/crates/layerfs-sandbox/src/backend/mod.rs index 1512ff5338..352c5c3f60 100644 --- a/core/crates/layerfs-sandbox/src/backend/mod.rs +++ b/core/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,4 +1,4 @@ //! Concrete ordinary runtime providers. pub mod docker; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] pub mod local; diff --git a/core/crates/layerfs-sandbox/src/lib.rs b/core/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..49e25fef96 100644 --- a/core/crates/layerfs-sandbox/src/lib.rs +++ b/core/crates/layerfs-sandbox/src/lib.rs @@ -1,5 +1,7 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. -#![forbid(unsafe_code)] +// unsafe only in backend/local/darwin.rs: macOS process facts come from +// proc_pidinfo, a system call (`ps` is setuid, which Seatbelt refuses) +#![deny(unsafe_code)] #[cfg(unix)] pub mod backend; mod types; diff --git a/core/crates/layerfs-sandbox/tests/local_runtime.rs b/core/crates/layerfs-sandbox/tests/local_runtime.rs index c948c7ff5e..e569fb1a99 100644 --- a/core/crates/layerfs-sandbox/tests/local_runtime.rs +++ b/core/crates/layerfs-sandbox/tests/local_runtime.rs @@ -285,6 +285,30 @@ fn a_command_runs_as_its_identity_in_its_directory_with_only_its_environment() { sandbox.delete().unwrap(); } +#[test] +fn the_daemon_and_its_commands_start_with_a_containers_search_path() { + // on Linux, whatever this process's PATH is (macOS: tests/local_search_path.rs) + const CONTAINER: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + let (root, daemon) = fixture(LISTENS); + let mut sandbox = started(&root, &daemon, "layerfs-shop-path"); + let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); + assert!( + log.lines().any(|line| line == format!("PATH={CONTAINER}")), + "{log}" + ); + let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "printf %s \"$PATH\""], &[]); + assert_eq!((code, out.as_str()), (0, CONTAINER)); + // a command's own PATH still wins + let (code, out, _) = run( + &sandbox, + &["/bin/sh", "-c", "printf %s \"$PATH\""], + &["PATH=/opt/tools/bin:/usr/bin:/bin"], + ); + assert_eq!((code, out.as_str()), (0, "/opt/tools/bin:/usr/bin:/bin")); + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); +} + #[test] fn a_daemon_that_ends_is_the_stopped_observation_and_only_its_delete_remains() { let (root, daemon) = fixture(LISTENS); diff --git a/core/crates/layerfs-sandbox/tests/local_search_path.rs b/core/crates/layerfs-sandbox/tests/local_search_path.rs new file mode 100644 index 0000000000..f90919c56f --- /dev/null +++ b/core/crates/layerfs-sandbox/tests/local_search_path.rs @@ -0,0 +1,113 @@ +//! A native (macOS) local Sandbox's daemon and commands start with this +//! process's own PATH, so this user's tools (`node` in `~/.local/bin`, +//! Homebrew's `/opt/homebrew/bin`) are found as in the caller's shell; with +//! no PATH, or an empty one, a container's search path and Homebrew's. +//! A small shell script stands in for the daemon. One test: it changes this +//! process's PATH, which no other test in this binary reads. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::{ + docker::ExecRequest, + local::{Local, Sandbox}, + }, + CommandIdentity, +}; +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + time::Duration, +}; + +const WAIT: Duration = Duration::from_secs(10); +const FALLBACK: &str = + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/homebrew/bin"; + +/// A fresh 0755 runtime root and a stand-in daemon that prints its +/// environment, then its listener line, then waits. +fn fixture() -> (PathBuf, PathBuf) { + let base = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("layerfs-search-path-{}", std::process::id())); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(base.join("root")).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); + let daemon = base.join("daemon.sh"); + fs::write( + &daemon, + "#!/bin/sh\n[ \"$1\" = --config ] && [ -f \"$2\" ] || exit 3\n/usr/bin/env >&2\necho \"LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\"\nexec /bin/sleep 60\n", + ) + .unwrap(); + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); + (base.join("root"), daemon) +} + +fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { + Local::native(root) + .unwrap() + .naming(name) + .unwrap() + .create(daemon, b"setup bytes", WAIT, Vec::new()) + .unwrap() +} + +/// The daemon's PATH line, from its log. +fn daemon_path(sandbox: &Sandbox) -> String { + let log = String::from_utf8(sandbox.log_tail(200, 16384).unwrap()).unwrap(); + log.lines() + .find_map(|line| line.strip_prefix("PATH=")) + .unwrap_or_else(|| panic!("no PATH line: {log}")) + .to_owned() +} + +/// A command's PATH: `printf` by its absolute path, so a lookup is not needed. +fn command_path(sandbox: &Sandbox, environment: &[&str]) -> String { + let request = ExecRequest { + identity: CommandIdentity { + uid: nix::unistd::Uid::effective().as_raw(), + gid: nix::unistd::Gid::effective().as_raw(), + }, + arguments: vec!["/bin/sh".into(), "-c".into(), "printf %s \"$PATH\"".into()], + environment: environment.iter().map(|e| e.to_string()).collect(), + directory: sandbox.directory().display().to_string(), + stdin: false, + }; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); + assert_eq!(exit.code, 0, "{}", String::from_utf8_lossy(&err)); + String::from_utf8(out).unwrap() +} + +#[test] +fn a_native_daemon_and_its_commands_start_with_this_processs_path() { + let (root, daemon) = fixture(); + let own = "/Users/someone/.local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; + let saved = std::env::var_os("PATH"); + std::env::set_var("PATH", own); + let mut sandbox = started(&root, &daemon, "layerfs-path-1"); + assert_eq!(daemon_path(&sandbox), own); + assert_eq!(command_path(&sandbox, &[]), own); + // a command's own PATH still wins + assert_eq!( + command_path(&sandbox, &["PATH=/opt/x/bin:/bin"]), + "/opt/x/bin:/bin" + ); + // no PATH, then an empty one: a container's search path and Homebrew's + std::env::remove_var("PATH"); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + std::env::set_var("PATH", ""); + assert_eq!(command_path(&sandbox, &[]), FALLBACK); + let mut second = started(&root, &daemon, "layerfs-path-2"); + assert_eq!(daemon_path(&second), FALLBACK); + match saved { + Some(path) => std::env::set_var("PATH", path), + None => std::env::remove_var("PATH"), + } + for sandbox in [&mut sandbox, &mut second] { + sandbox.stop(1).unwrap(); + sandbox.delete().unwrap(); + } + let _ = fs::remove_dir_all(root.parent().unwrap()); +} diff --git a/core/tools/check_product_boundary.py b/core/tools/check_product_boundary.py index 5f4747f486..ba8d0b74d0 100644 --- a/core/tools/check_product_boundary.py +++ b/core/tools/check_product_boundary.py @@ -16,14 +16,16 @@ # Crates whose `unsafe` surface is bounded. layerfs-storage keeps one audited # FFI module (encoding/codec.rs); Persistence has the owner-authorized macOS -# seal file-control module. Their siblings stay unsafe-free. The +# seal file-control module; Sandbox reads macOS process facts through +# proc_pidinfo in backend/local/darwin.rs. Their siblings stay unsafe-free. The # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-sandbox": "src/backend/local/darwin.rs", } -UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", "layerfs-sandbox") +UNSAFE_FREE_CRATES = ("layerfs-content", "layerfs-telemetry", "layerfs-project", "layerfs-overlay", "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge") UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", "layerfs-content": "#![forbid(unsafe_code)]", @@ -38,7 +40,7 @@ "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", - "layerfs-sandbox": "#![forbid(unsafe_code)]", + "layerfs-sandbox": "#![deny(unsafe_code)]", } diff --git a/core/tools/test_check_product_boundary.py b/core/tools/test_check_product_boundary.py index 9769a3178f..234c9dfb42 100644 --- a/core/tools/test_check_product_boundary.py +++ b/core/tools/test_check_product_boundary.py @@ -86,7 +86,11 @@ def test_sdk_source_edges_and_unsafe_boundary(self): def test_sandbox_active_edges_and_unsafe_boundary(self): path=Path("core/crates/layerfs-sandbox/src/backend/docker/http.rs") self.assertTrue(unsafe_violations(path,"unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("core/crates/layerfs-sandbox/src/lib.rs"),"#![forbid(unsafe_code)]")) + self.assertFalse(unsafe_violations(Path("core/crates/layerfs-sandbox/src/lib.rs"),"#![deny(unsafe_code)]")) + self.assertTrue(unsafe_violations(Path("core/crates/layerfs-sandbox/src/lib.rs"),"#![forbid(unsafe_code)]")) + darwin=Path("core/crates/layerfs-sandbox/src/backend/local/darwin.rs") + self.assertFalse(unsafe_violations(darwin,"fn f() { unsafe { libc::proc_listallpids(); } }")) + self.assertTrue(unsafe_violations(Path("core/crates/layerfs-sandbox/src/backend/local/process.rs"),"fn f() { unsafe { call(); } }")) prefix='[package]\nname="layerfs-sandbox"\n[dependencies]\n' for dependency in ("layerfs-api-core","layerfs-server","layerfs-daemon","layerfs-persistence","layerfs-api"): self.assertTrue(dependency_violations(prefix+f'{dependency}={{path="../{dependency}"}}')) diff --git a/sdk/js/README.md b/sdk/js/README.md index 9c43a8fe50..9430f34d8f 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -75,7 +75,7 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open in this process; the client option `storePath` wins. It is `global/store.sqlite` in the project's volume, `/layerfs-store/global/store.sqlite` where that volume is mounted at `/layerfs-store`. Linux only, on a local filesystem the daemons share; elsewhere Store calls fail `Unsupported`. Unset, they fail `InvalidInput`. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional; the client option `daemonHost` wins. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | -| `LAYERFS_SANDBOX` | Optional; the client option `sandbox` wins. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `local`: a Sandbox is a daemon process on this machine, with no container and no Docker setting read (see [Local Sandboxes](#local-sandboxes-no-docker)). Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_SANDBOX` | Optional; the client option `sandbox` wins. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `local`: a Sandbox is a daemon process on this machine, with no container and no Docker setting read (see [Local Sandboxes](#local-sandboxes-no-docker)). `native`: the same on macOS without root: Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Any other value is refused as `InvalidInput` before anything is made. | | `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=local`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each local Sandbox gets its own directory under it. | | `LAYERFS_TRACE` | Optional per-call trace file (see [Call trace](#call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | @@ -288,6 +288,41 @@ observation: Workspace calls to it are refused, and the next mount or left before deleting its directory. Nothing outside that directory is removed; the Store is never touched. +## Native Sandboxes (macOS) + +`LAYERFS_SANDBOX=native` is the local runtime for macOS, with no root, no +Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon +runs as a process of this user and serves its Workspaces over NFSv3 on +127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them +(`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands +run as this user, in this machine's namespaces: like `local`, **it isolates +nothing** beyond what the caller's own sandbox (for instance Seatbelt) +imposes. A command starts with this process's `PATH` (so `node`, Homebrew's +tools and the like are found as in the caller's own shell) and its own +environment, nothing else of this process's; a `local` Sandbox on Linux keeps +a container's fixed search path. + +It needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned +by this user, 0755; the directories above it need not be traversable by +others) and the client's `storePath` on a local apfs or hfs volume; a Store on +any other filesystem is refused `Unsupported`. Directory layout, records, idle +close and reclaim are those of local Sandboxes; a record's `runtime` is +`native`, and native, local and Docker Sandboxes of one project are never +mixed. Before a Commit the daemon writes back what the client buffered; after +a Commit or merge installs another root, it makes the client drop the replaced +files' cached attributes at once. + +`LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's +helper commands (`{port}` and `{dir}` are filled in). The default mount is +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,timeo=20,retrans=6,nobrowse 127.0.0.1:/ {dir}`. +`LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client +then revalidates within `actimeo`, 60 s). + +Limits: background processes (`operator.process.*`) still read `/proc` and are +not available on macOS; many small writes cost about six times APFS's time; +checked on Apple silicon with macOS 27.2. On Linux the Store install needs +root, which a native Sandbox's commands may not be: use `local` there. + ## Internal operator ```js diff --git a/sdk/js/native/src/discovery.rs b/sdk/js/native/src/discovery.rs index c2725160f0..b417ed797a 100644 --- a/sdk/js/native/src/discovery.rs +++ b/sdk/js/native/src/discovery.rs @@ -10,7 +10,7 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use layerfs_sandbox::backend::local::{self, LocalProcess}; use layerfs_sandbox::{ backend::docker::{Docker, Sandbox, SandboxRequest}, @@ -29,6 +29,7 @@ use std::{ }; extern "C" { fn geteuid() -> u32; + fn getegid() -> u32; fn flock(fd: i32, operation: i32) -> i32; } const LOCK_EX: i32 = 2; @@ -73,10 +74,15 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } /// Whether this process runs as root, which the local runtime needs: its /// daemon mounts FUSE and its commands run as another uid/gid. -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] pub(crate) fn root() -> bool { unsafe { geteuid() == 0 } } +/// This process's effective uid and gid: a native Sandbox's command +/// identity (its commands run as this user). +pub(crate) fn user() -> (u32, u32) { + unsafe { (geteuid(), getegid()) } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -376,7 +382,7 @@ pub(crate) fn save_sandbox( ("volume", SdkValue::text(&runtime.request().store_volume)), ("labels", labels(&runtime.request().labels)), ]), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Runtime::Local(runtime) => { let process = runtime .process() @@ -385,7 +391,16 @@ pub(crate) fn save_sandbox( .boot .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; fields.extend([ - ("runtime", SdkValue::text(crate::runtime::LOCAL)), + ( + "runtime", + SdkValue::text(if crate::runtime::native(deploy) { + crate::runtime::NATIVE + } else { + crate::runtime::LOCAL + }), + ), + ("uid", SdkValue::Int(i64::from(state.identity.uid))), + ("gid", SdkValue::Int(i64::from(state.identity.gid))), ( "directory", SdkValue::text(runtime.directory().display().to_string()), @@ -477,7 +492,7 @@ fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { /// a local daemon process, its directory and port. pub(crate) enum Recorded { Docker(Docker, ContainerId, SandboxRequest), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Local { directory: PathBuf, id: ContainerId, @@ -488,7 +503,10 @@ pub(crate) enum Recorded { } /// Reads Sandbox `id`'s record's runtime. pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { - if value.get_text("runtime") != Some(crate::runtime::LOCAL) { + if !matches!( + value.get_text("runtime"), + Some(crate::runtime::LOCAL | crate::runtime::NATIVE) + ) { let (docker, container, request) = runtime(value)?; return Ok(Recorded::Docker(docker, container, request)); } @@ -500,7 +518,7 @@ pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result Result { let directory = PathBuf::from(registry::text(value, "directory")?); let expected = crate::local::directory(args, id)?; @@ -550,7 +568,7 @@ fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result labels: labels(value)?, }) } -#[cfg(not(target_os = "linux"))] +#[cfg(not(any(target_os = "linux", target_os = "macos")))] fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { Err(refused( "Unsupported", @@ -564,7 +582,7 @@ impl Recorded { Self::Docker(docker, id, request) => { Sandbox::reopen(docker, id, request).map(Runtime::Docker) } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local { directory, id, @@ -580,7 +598,7 @@ impl Recorded { Self::Docker(docker, id, request) => { Runtime::Docker(Sandbox::stopped(docker, id, request)) } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local { directory, id, @@ -613,7 +631,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { // a local daemon listens on this kernel's loopback; no other host reaches it let host = match recorded { Recorded::Docker(..) => crate::pool::daemon_host(args), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Recorded::Local { .. } => None, }; let runtime = recorded.reopen().map_err(|failure| { @@ -638,7 +656,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { name: value.get_text("name").unwrap_or_default().to_owned(), directory, runtime, - identity: CommandIdentity { uid: 501, gid: 20 }, + identity: recorded_identity(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -646,6 +664,15 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { refresh(&state)?; Ok(state) } +/// The command identity a Sandbox record names: a native Sandbox's is the +/// user who started it (`uid`, `gid`); every other runs as 501:20. +fn recorded_identity(value: &SdkValue) -> CommandIdentity { + let id = |key: &str| value.get_int(key).and_then(|n| u32::try_from(n).ok()); + match (id("uid"), id("gid")) { + (Some(uid), Some(gid)) => CommandIdentity { uid, gid }, + _ => CommandIdentity { uid: 501, gid: 20 }, + } +} /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); diff --git a/sdk/js/native/src/lib.rs b/sdk/js/native/src/lib.rs index 94b70f5f88..b604af581c 100644 --- a/sdk/js/native/src/lib.rs +++ b/sdk/js/native/src/lib.rs @@ -5,7 +5,7 @@ mod discovery; mod dispatch; mod exec; mod job; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] mod local; mod output; mod pool; diff --git a/sdk/js/native/src/local.rs b/sdk/js/native/src/local.rs index 57acba623a..8fcf370764 100644 --- a/sdk/js/native/src/local.rs +++ b/sdk/js/native/src/local.rs @@ -10,8 +10,10 @@ //! where this process opens it (`storePath`), so the daemon setup and the //! Store's locator name that path instead of `/layerfs-store`. The daemon //! listens on 127.0.0.1 only. Commands run as the same nonroot identity as -//! in a container. Start, failure and reclaim follow the Docker route: one -//! attempt, a creation intent while the outcome is open, and a failed start's +//! in a container. A native Sandbox (`deploy.sandbox` `native`) needs no +//! root: its daemon serves Workspaces over loopback NFS (`LAYERFS_KERNEL=nfs`) +//! to this host's own client, and its commands run as this user. Start, +//! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's //! daemon stopped and its directory deleted, each once. use crate::job::{failed, refused}; use crate::registry::{lock, text, SandboxState, SANDBOXES}; @@ -37,8 +39,9 @@ pub(crate) fn start( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + let native = crate::runtime::native(deploy); // Inputs are read first, so a refused request leaves no creation intent. - if !crate::discovery::root() { + if !native && !crate::discovery::root() { return Err(refused( "Unsupported", "LAYERFS_SANDBOX=local needs root: its daemon mounts FUSE and its commands run as \ @@ -51,8 +54,12 @@ pub(crate) fn start( "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a local Sandbox needs it", ) })?; - let local = Local::new(root) - .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let local = if native { + Local::native(root).map(|local| local.environment(native_environment())) + } else { + Local::new(root) + } + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; let executable = PathBuf::from(text(deploy, "daemon")?); let store = match args.get_text("storePath") { Some(path) if Path::new(path).is_absolute() => path.to_owned(), @@ -74,7 +81,7 @@ pub(crate) fn start( &saved_directory, &id, project, - (local, executable, store), + (local, executable, store, native), ); if made.is_err() { let _ = std::fs::remove_dir(&saved_directory); @@ -90,7 +97,7 @@ fn start_claimed( saved_directory: &Path, id: &str, (project, installed): (&crate::registry::Project, StoreManifest), - (local, executable, store): (Local, PathBuf, String), + (local, executable, store, native): (Local, PathBuf, String, bool), ) -> Result, SdkRefusal> { let name = text(args, "project")?; let sandbox_name = sandbox_name(args, id)?; @@ -106,6 +113,10 @@ fn start_claimed( let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let mut setup = crate::sandbox::setup(daemon.private, controller.public); + if native { + // the native route's commands run as this user, unconfined + (setup.command_uid, setup.command_gid) = crate::discovery::user(); + } setup.listen = "127.0.0.1:0".into(); setup.store = store.clone(); setup.overlay = path_text(&paths.overlay)?; @@ -211,6 +222,25 @@ fn start_claimed( Ok(state) } +/// What a native Sandbox's daemon starts with besides `PATH`: the NFS +/// route, and the mount helper and reference overrides this process was +/// given (`LAYERFS_NFS_*`), so a test host can name its own client. +fn native_environment() -> Vec<(String, String)> { + let mut environment = vec![("LAYERFS_KERNEL".to_owned(), "nfs".to_owned())]; + for name in [ + "LAYERFS_NFS_MOUNT", + "LAYERFS_NFS_UNMOUNT", + "LAYERFS_NFS_FORCE", + "LAYERFS_NFS_FLUSH", + "LAYERFS_NFS_REFRESH", + ] { + if let Ok(value) = std::env::var(name) { + environment.push((name.to_owned(), value)); + } + } + environment +} + /// A local Sandbox's directory name: `-`, the name its /// container would have. fn sandbox_name(args: &SdkValue, id: &str) -> Result { diff --git a/sdk/js/native/src/runtime.rs b/sdk/js/native/src/runtime.rs index e3d460592a..5e1a152e28 100644 --- a/sdk/js/native/src/runtime.rs +++ b/sdk/js/native/src/runtime.rs @@ -2,13 +2,16 @@ //! `deploy.sandbox` `local` (LAYERFS_SANDBOX=local) a daemon process on this //! kernel, for a VM or CI box that is itself the isolation boundary. Both run //! the same daemon and answer the same calls; only how it is started, how -//! commands are launched and how it is observed differ. The local runtime -//! is Linux only; elsewhere selecting it is refused as Unsupported. +//! commands are launched and how it is observed differ. With `native` +//! (LAYERFS_SANDBOX=native) the local daemon serves its Workspaces over +//! loopback NFS to this host's own client and commands run as this user, +//! with no root (macOS). The local runtimes run on Linux and macOS; +//! elsewhere selecting one is refused as Unsupported. use crate::job::{failed, refused}; use crate::output::Spool; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_sandbox::backend::docker; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use layerfs_sandbox::backend::local; use layerfs_sandbox::{CommandIdentity, ContainerId, WireFailure}; use std::net::SocketAddr; @@ -16,33 +19,44 @@ use std::net::SocketAddr; /// A local Sandbox record's `runtime`, and the image it records (a Docker /// record has no `runtime`). pub(crate) const LOCAL: &str = "local"; +/// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local +/// daemon serving its Workspaces over loopback NFS, commands as this user. +pub(crate) const NATIVE: &str = "native"; /// One Sandbox's runtime owner. pub(crate) enum Runtime { Docker(docker::Sandbox), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Local(local::Sandbox), } -/// Whether the deployment selects the local runtime: `deploy.sandbox` unset -/// or `docker` is Docker, `local` is local; anything else is refused before -/// any effect. +/// Whether the deployment selects a local daemon: `deploy.sandbox` unset or +/// `docker` is Docker, `local` and `native` are a local daemon; anything +/// else is refused before any effect. pub(crate) fn local(deploy: &SdkValue) -> Result { match deploy.get_text("sandbox") { None | Some("docker") => Ok(false), - Some("local") => Ok(true), + Some(LOCAL) | Some(NATIVE) => Ok(true), Some(other) => Err(refused( "InvalidInput", - format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or local, not {other}"), + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or local (or native), not {other}"), )), } } +/// Whether the deployment selects the native route: a local daemon whose +/// Workspaces this host's own NFS client mounts (`LAYERFS_KERNEL=nfs`), and +/// whose commands run as this user, unconfined. No root, no Docker, no FUSE. +pub(crate) fn native(deploy: &SdkValue) -> bool { + deploy.get_text("sandbox") == Some(NATIVE) +} + /// What a new Workspace of a deployment is placed by: its runtime and, for /// Docker, its image (`deploy.image`). #[derive(Debug)] pub(crate) struct Placement { - local: bool, + /// The record `runtime` it takes: none for Docker. + runtime: Option<&'static str>, image: Option, } @@ -51,10 +65,11 @@ pub(crate) fn placement(args: &SdkValue) -> Result, SdkRefusal let Some(deploy) = args.get("deploy") else { return Ok(None); }; - let local = local(deploy)?; + let runtime = local(deploy)?.then(|| if native(deploy) { NATIVE } else { LOCAL }); Ok(Some(Placement { - local, - image: (!local) + runtime, + image: runtime + .is_none() .then(|| deploy.get_text("image").map(str::to_owned)) .flatten(), })) @@ -66,9 +81,11 @@ impl Placement { /// Docker, the same image. The image string alone does not tell them /// apart: `local` is also a valid Docker image name. pub(crate) fn admits(&self, record: &SdkValue) -> bool { - let local = record.get_text("runtime") == Some(LOCAL); - local == self.local - && (local || self.image.is_none() || record.get_text("image") == self.image.as_deref()) + let runtime = record.get_text("runtime"); + runtime == self.runtime + && (runtime.is_some() + || self.image.is_none() + || record.get_text("image") == self.image.as_deref()) } } @@ -77,7 +94,7 @@ impl Runtime { pub(crate) fn identity(&self) -> ContainerId { match self { Self::Docker(sandbox) => sandbox.identity(), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => sandbox.identity(), } } @@ -85,7 +102,7 @@ impl Runtime { pub(crate) fn image(&self) -> &str { match self { Self::Docker(sandbox) => &sandbox.request().image, - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(_) => LOCAL, } } @@ -95,7 +112,7 @@ impl Runtime { pub(crate) fn endpoint(&self) -> Result> { match self { Self::Docker(sandbox) => sandbox.endpoint(), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => sandbox.endpoint(), } } @@ -105,7 +122,7 @@ impl Runtime { pub(crate) fn scratch(&self) -> String { match self { Self::Docker(_) => "/tmp".to_owned(), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => sandbox.tmp().display().to_string(), } } @@ -113,18 +130,21 @@ impl Runtime { pub(crate) fn delete(&mut self) -> Result<(), Box> { match self { Self::Docker(sandbox) => sandbox.delete(), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => sandbox.delete(), } } /// Names a background process a command started in a session of its own /// (`pid`, started at `start`), so it ends with a local Sandbox as it /// ends with a container. - #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] + #[cfg_attr( + not(any(target_os = "linux", target_os = "macos")), + allow(unused_variables) + )] pub(crate) fn track(&self, pid: u32, start: &str) -> Result<(), SdkRefusal> { match self { Self::Docker(_) => Ok(()), - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => { let start = start.parse::().map_err(failed)?; sandbox.track(pid, start).map_err(failed) @@ -161,7 +181,7 @@ impl Runtime { .known_root_exit() .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))? } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] Self::Local(sandbox) => { let request = docker::ExecRequest { identity, diff --git a/sdk/js/native/src/sandbox.rs b/sdk/js/native/src/sandbox.rs index d8d69c7996..ceff568661 100644 --- a/sdk/js/native/src/sandbox.rs +++ b/sdk/js/native/src/sandbox.rs @@ -114,9 +114,9 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result SdkValue { @@ -81,6 +81,28 @@ fn placement_keeps_runtimes_apart_even_for_an_image_named_local() { ); } +#[test] +fn native_is_a_local_daemon_placed_apart_from_local_and_docker() { + let native_deploy = deploy(&[("sandbox", "native")]); + assert!(local(&native_deploy).unwrap()); + assert!(native(&native_deploy)); + assert!(!native(&deploy(&[("sandbox", "local")]))); + assert!(!native(&deploy(&[]))); + let native_record = placed(Some("native"), "local"); + let local_record = placed(Some("local"), "local"); + let docker_record = placed(None, &format!("sha256:{}", "a".repeat(64))); + let natively = SdkValue::map([("deploy", native_deploy)]); + let natively = placement(&natively).unwrap().unwrap(); + assert!(natively.admits(&native_record)); + assert!(!natively.admits(&local_record)); + assert!(!natively.admits(&docker_record)); + let locally = SdkValue::map([("deploy", deploy(&[("sandbox", "local")]))]); + let locally = placement(&locally).unwrap().unwrap(); + assert!(!locally.admits(&native_record)); + let docker = SdkValue::map([("deploy", deploy(&[]))]); + assert!(!placement(&docker).unwrap().unwrap().admits(&native_record)); +} + /// A call's fields for Sandbox 1 of project `p` deployed locally under `root`. #[cfg(target_os = "linux")] fn call(root: Option<&str>) -> SdkValue { diff --git a/sdk/js/src/client.js b/sdk/js/src/client.js index 266dbaeebb..538cde9936 100644 --- a/sdk/js/src/client.js +++ b/sdk/js/src/client.js @@ -43,14 +43,16 @@ export function workspaceArgs(ws) { * every one but the container prefix and daemon host. With `sandbox` * (LAYERFS_SANDBOX) `local`, a Sandbox is a daemon process on this machine * instead of a container: it needs the daemon and `localDir` - * (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is Docker; - * any other value is refused. */ + * (LAYERFS_LOCAL_DIR) and no Docker setting. `native` is a local daemon + * whose Workspaces this host's own NFS client mounts, with commands run as + * this user (no root, no Docker image). Unset or `docker` is Docker; any + * other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || process.env[`DSH_LAYERFS_${name}`] || undefined; const sandbox = options.sandbox ?? env("SANDBOX"); - if (sandbox === "local") return localDeployment(options, env); + if (sandbox === "local" || sandbox === "native") return localDeployment(options, env, sandbox); if (sandbox !== undefined && sandbox !== "docker") { - throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or local, not ${sandbox}`, null); + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or local (or native), not ${sandbox}`, null); } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", @@ -71,9 +73,9 @@ export function deployment(options = {}) { /** A local Sandbox's settings: the daemon, the directory its Sandboxes * live in, and the optional name prefix. */ -function localDeployment(options, env) { +function localDeployment(options, env, sandbox) { const deploy = { - sandbox: "local", + sandbox, daemon: options.daemon ?? env("DAEMON"), localDir: options.localDir ?? env("LOCAL_DIR"), containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), From 1deb780d123c26424b13aea73ca10b470dde10cf Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Fri, 23 Oct 2026 00:30:00 +0800 Subject: [PATCH 44/64] feat(store-service): open the Store on macOS for the native route; bundled SQLite on macOS On macOS the native route is the only route: the daemon, the SDK and every command run on one macOS kernel as the user who installed the Store. - install.rs: installer_user() admits the installer on macOS. The Linux check (the Store installer is the daemon's root, read from /proc/self) is unchanged and still compiled only off macOS; the daemon's own check that it opens only Store files it owns still holds on both. - kernel_guard.rs: on macOS a Store on apfs or hfs is admitted (statfs); any other filesystem is still Unsupported. Linux is unchanged and every other platform is still refused. This relaxes #352 F-020 ("Store calls are Linux only") for macOS and needs the owner's decision. tests/kernel_guard.rs: the macOS refusal case becomes an apfs/hfs admission case (refused() and its import compile off macOS only); the Linux and other-platform assertions are unchanged. - layerfs-overlay and layerfs-persistence use rusqlite's bundled SQLite on macOS as on Linux: macOS 27.2's system SQLite 3.54.0 is outside the overlay's qualified growth profiles ("unqualified SQLite physical growth profile", the same cause as #403's first finding). Cargo.lock is unchanged. Production LOC: 114,591 -> 114,619 (delta +28) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 6c8ca0e96. --- core/crates/layerfs-overlay/Cargo.toml | 2 +- core/crates/layerfs-persistence/Cargo.toml | 2 +- .../layerfs-store-service/src/install.rs | 9 +++++++ .../layerfs-store-service/src/kernel_guard.rs | 27 ++++++++++++++++++- .../tests/kernel_guard.rs | 21 +++++++++++---- 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/core/crates/layerfs-overlay/Cargo.toml b/core/crates/layerfs-overlay/Cargo.toml index a0de581508..842ac36348 100644 --- a/core/crates/layerfs-overlay/Cargo.toml +++ b/core/crates/layerfs-overlay/Cargo.toml @@ -11,5 +11,5 @@ blake3 = { version = "=1.8.5", default-features = false } nix = { version = "=0.31.3", default-features = false, features = ["fs"] } rusqlite = { version = "=0.40.2", features = ["cache", "limits", "blob"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/core/crates/layerfs-persistence/Cargo.toml b/core/crates/layerfs-persistence/Cargo.toml index 8a34236d24..ffa66bd76d 100644 --- a/core/crates/layerfs-persistence/Cargo.toml +++ b/core/crates/layerfs-persistence/Cargo.toml @@ -22,5 +22,5 @@ nix = { version = "=0.31.3", default-features = false, features = ["fs"] } [target.'cfg(target_os = "linux")'.dev-dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthread", "signal"] } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rusqlite = { version = "=0.40.2", features = ["bundled"] } diff --git a/core/crates/layerfs-store-service/src/install.rs b/core/crates/layerfs-store-service/src/install.rs index 850570a334..f150c5685c 100644 --- a/core/crates/layerfs-store-service/src/install.rs +++ b/core/crates/layerfs-store-service/src/install.rs @@ -21,6 +21,7 @@ use std::sync::Arc; /// The daemon's user inside its Sandbox (the runtime's `User` is `0:0`). A /// daemon opens only a Store file and directory it owns, so an installer /// must run as this user too. +#[cfg(not(target_os = "macos"))] const DAEMON_UID: u32 = 0; /// The destination and its exclusive temporary name. @@ -202,7 +203,15 @@ pub fn install_local( result.map_err(|cause: String| LocalInstallFailure { cause, ..left }) } +/// macOS has only the native route, whose daemon runs as the user who +/// installs its Store (there is no root daemon to match); the daemon's own +/// check that it owns the Store file still holds. +#[cfg(target_os = "macos")] +fn installer_user() -> Result<(), String> { + Ok(()) +} /// Refuses an installer that is not the daemons' user. +#[cfg(not(target_os = "macos"))] fn installer_user() -> Result<(), String> { use std::os::unix::fs::MetadataExt; // the owner of /proc/self is this process's effective user diff --git a/core/crates/layerfs-store-service/src/kernel_guard.rs b/core/crates/layerfs-store-service/src/kernel_guard.rs index 49489cac1e..e0e3945972 100644 --- a/core/crates/layerfs-store-service/src/kernel_guard.rs +++ b/core/crates/layerfs-store-service/src/kernel_guard.rs @@ -56,7 +56,32 @@ pub fn check(store: &Path) -> Result<&'static str, Unsupported> { )) }) } - #[cfg(not(target_os = "linux"))] + // macOS native (LAYERFS_SANDBOX=native): the daemon, the SDK and every + // command share this one macOS kernel, so the Store's locks coordinate + // exactly as on Linux; only a local filesystem may hold the Store. + #[cfg(target_os = "macos")] + { + let directory = store + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let stat = nix::sys::statfs::statfs(directory).map_err(|error| { + Unsupported(format!( + "the Store's directory {} cannot be examined: {error}", + directory.display() + )) + })?; + match stat.filesystem_type_name() { + "apfs" => Ok("apfs"), + "hfs" => Ok("hfs"), + other => Err(Unsupported(format!( + "filesystem {other} at {} does not carry SQLite locks between kernels; \ + direct Store access needs apfs or hfs", + directory.display() + ))), + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { let _ = store; Err(Unsupported( diff --git a/core/crates/layerfs-store-service/tests/kernel_guard.rs b/core/crates/layerfs-store-service/tests/kernel_guard.rs index d9050f370d..e99d46ea85 100644 --- a/core/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/core/crates/layerfs-store-service/tests/kernel_guard.rs @@ -1,11 +1,13 @@ //! Direct Store access is refused `Unsupported` where the Store's locks do -//! not reach every opener (#352 F-020): on macOS, and on Linux for a -//! filesystem outside the allowlist (here tmpfs). The refusal comes before any -//! file is opened: the Store file's bytes are unchanged and no overlay -//! template is created. +//! not reach every opener (#352 F-020): off Linux and macOS, and on Linux for +//! a filesystem outside the allowlist (here tmpfs). The refusal comes before +//! any file is opened: the Store file's bytes are unchanged and no overlay +//! template is created. On macOS a local apfs or hfs directory is allowed. +#[cfg(not(target_os = "macos"))] use layerfs_store_service::{OpenRefusal, StoreService}; use std::path::{Path, PathBuf}; +#[cfg(not(target_os = "macos"))] fn refused(directory: &Path) -> (String, Vec, Vec) { let store = directory.join("store.sqlite"); std::fs::write(&store, b"not opened").unwrap(); @@ -30,7 +32,16 @@ fn scratch(parent: &Path, label: &str) -> PathBuf { directory } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] +#[test] +fn direct_access_is_allowed_on_a_local_macos_filesystem() { + let directory = scratch(&std::env::temp_dir(), "apfs"); + let checked = layerfs_store_service::kernel_guard::check(&directory.join("store.sqlite")); + assert!(matches!(checked, Ok("apfs") | Ok("hfs")), "{checked:?}"); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] fn direct_access_is_unsupported_off_linux() { let directory = scratch(&std::env::temp_dir(), "host"); From 5169c307b6ef7d3967494ef659ea66dad2a8491f Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Fri, 23 Oct 2026 00:30:00 +0800 Subject: [PATCH 45/64] feat(dsh): the LayerFS Host on the native macOS route (sandbox: native, no Docker) With `sandbox: native` in the layerfs-host config (or LAYERFS_SANDBOX=native in dsh's environment) the Host and agents load the real SDK in process and use local Sandboxes over loopback NFS. No Store container, no Docker volume, no docker call. Without it every path is as before. - config.js: sandbox/localDir/stores/nfsMount settings, defaults under realpath(os.tmpdir())/dsh-layerfs (inside dsh's Seatbelt writable roots); native is never ported, and needs only `daemon`. - init.js: a Store folder / (0700) instead of a Docker volume; native checks only the home's owner. - layerfs-client.js, env.js, index.js: the native deployment's LAYERFS_* / DSH_LAYERFS_* names, three new declared bash keys and one prompt line. - remove.js: removeNative refuses with busy while a project's daemon runs (pgrep -lf/-af, no ps: it is setuid, which Seatbelt refuses) or one of its Workspaces is still mounted; it never removes a folder under a live NFS mount; when pgrep fails otherwise than "no match", or the mount table cannot be read, it refuses too and removes nothing. Otherwise it deletes the Sandbox, Store and home entries. - runtime.js, route.js: no Docker probe on native; status reports sandbox, localDir and stores. - sdk/js/src/client.js: on native, DSH_LAYERFS_NFS_* stand in for unset LAYERFS_NFS_* (dsh bash calls only receive DSH_* names). dsh/README.md: an "On macOS without Docker (sandbox: native)" paragraph; sdk/js/README.md: the DSH_LAYERFS_NFS_* stand-ins. Tests: tests/native.mjs (no Docker or Store) and tests/native_real.mjs (real SDK and native build, NATIVE_WORKSPACE=0 runs the Store half only). Existing host, bindings-shared, sim and runtime tests are unchanged and pass. Production LOC: 114,619 -> 114,693 (delta +74) Scope: combined core+sdk+dsh, tools/production_loc.py, parent 1deb780d1. --- dsh/README.md | 2 + dsh/packages/host/src/config.js | 40 +++++- dsh/packages/host/src/env.js | 11 +- dsh/packages/host/src/index.js | 8 +- dsh/packages/host/src/init.js | 17 ++- dsh/packages/host/src/layerfs-client.js | 13 ++ dsh/packages/host/src/remove.js | 38 ++++++ dsh/packages/host/src/route.js | 2 +- dsh/packages/host/src/runtime.js | 4 + dsh/packages/host/tests/native.mjs | 147 +++++++++++++++++++++ dsh/packages/host/tests/native_real.mjs | 165 ++++++++++++++++++++++++ sdk/js/README.md | 3 +- sdk/js/src/client.js | 11 ++ 13 files changed, 447 insertions(+), 14 deletions(-) create mode 100644 dsh/packages/host/tests/native.mjs create mode 100644 dsh/packages/host/tests/native_real.mjs diff --git a/dsh/README.md b/dsh/README.md index db8f573acb..9cf3f322fb 100644 --- a/dsh/README.md +++ b/dsh/README.md @@ -281,6 +281,8 @@ Engine. This does not require running a second dsh server in WSL. The [verification record](docs/windows-live-verification-20261022.md) pins the tested source, native bundle, outcomes and platform gaps. +**On macOS without Docker (`sandbox: native`).** With `sandbox: native` in `layerfs-host` (or `LAYERFS_SANDBOX=native` in dsh web's environment) the Host and agents' scripts load the SDK in their own process, as on Linux, and need only `daemon`: a macOS build's `layerfs-daemon`, with that build's `layerfs.node` in `sdk/js/native/` (agents' bash calls are not given `LAYERFS_NATIVE`). There is no image, bundle, Store container or Docker volume: each project's Store is a folder `/` on the Mac's APFS, and Sandboxes are local daemons under `localDir` whose Workspaces macOS's own NFS client mounts (`mount_nfs`, no root, no macFUSE), with commands run as you ([native Sandboxes](../sdk/js/README.md#native-sandboxes-macos)). `localDir` and `stores` default to `/dsh-layerfs/{local,stores}`, inside dsh's Workspace Write sandbox, so agents' bash calls can also work in a mounted Workspace's folder. `nfsMount` replaces the daemon's mount command when the default options fail. Removing a project is refused `busy` while one of its Sandbox daemons runs or one of its Workspaces is mounted (a Sandbox closes 2 minutes after its last unmount). On this route the All projects page shows no running-Sandbox cards (a project's Sandboxes tab does), and background processes (`&`) are not available yet. + **Running two dsh webs on one machine.** The default `home` and `traceDir` are the same for every profile, so two dsh webs would share projects and the Store container. Give each its own pair under the OS temp area, and its own `containerPrefix`, since container names are per Docker engine (the Store container of another home is refused, never taken over): ```yaml diff --git a/dsh/packages/host/src/config.js b/dsh/packages/host/src/config.js index 6d6801dab6..adba3a39b2 100644 --- a/dsh/packages/host/src/config.js +++ b/dsh/packages/host/src/config.js @@ -1,5 +1,5 @@ // Host plugin configuration and its resolution into absolute settings. -import { mkdirSync, realpathSync } from "node:fs"; +import { chmodSync, mkdirSync, realpathSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join, posix } from "node:path"; import { fileURLToPath } from "node:url"; @@ -27,6 +27,14 @@ import { windowsConfig } from "./windows/host.js"; * @property {string} [wslUser] Windows: Linux user owning Store volumes (default root). * @property {string} [wslHome] Windows: persistent Linux discovery directory (default derived from Windows home). * @property {string} [wslNode] Windows: Linux Node executable (default /usr/local/bin/node). + * @property {"docker"|"native"} [sandbox] Where Live projects run (default `docker`, or the dsh + * process's LAYERFS_SANDBOX): `native` runs every SDK call in this process and in agents' bash + * calls, keeps each Store in a local folder (`stores`) and runs Sandboxes as local daemons whose + * Workspaces this machine's own NFS client mounts (`localDir`), commands as this user. No Docker, + * image or bundle; `daemon` is a native build's layerfs-daemon. Meant for macOS (APFS). + * @property {string} [localDir] native: the folder local Sandboxes live in (default `/dsh-layerfs/local`, 0755). + * @property {string} [stores] native: the folder holding each project's Store folder (default `/dsh-layerfs/stores`). + * @property {string} [nfsMount] native: the daemon's NFS mount command (LAYERFS_NFS_MOUNT), when the default options fail. */ /** @@ -43,6 +51,10 @@ import { windowsConfig } from "./windows/host.js"; * @property {string} traceDir * @property {boolean} traceInWorkspace Windows default: traces use the shell's writable workspace. * @property {number} pollMs + * @property {boolean} native the native route (sandbox `native`): no Docker, no Store container + * @property {string|null} localDir native: local Sandboxes' folder + * @property {string|null} stores native: the Stores' folder (a project's Store is `//global/store.sqlite`) + * @property {string|null} nfsMount native: LAYERFS_NFS_MOUNT for the daemons, or null (their default) * @property {string} sdk file URL of the SDK entry Live projects use (on macOS, port/sdk.js) * @property {string} operator file URL of its operator entry * @property {string} simSdk file URL of the SDK on the simulator (sim/sdk.js), for simulated projects @@ -62,8 +74,13 @@ import { windowsConfig } from "./windows/host.js"; export function resolveConfig(config, profile) { const temp = join(realpathSync(tmpdir()), "dsh-layerfs"); const bundle = config.bundle || null, home = config.home || join(temp, "home"); - // macOS runs no Store call itself: a bundle's SDK calls run in the Store container (port/) - const ported = process.platform === "darwin" && bundle !== null; + // the dsh process's LAYERFS_SANDBOX selects only the native route (`local` is the SDK's, not the Host's) + const sandbox = config.sandbox || (process.env.LAYERFS_SANDBOX === "native" ? "native" : "docker"); + if (sandbox !== "docker" && sandbox !== "native") throw new Error(`layerfs-host sandbox is docker or native, not ${sandbox}`); + const native = sandbox === "native"; + // macOS runs no Store call itself: a bundle's SDK calls run in the Store container (port/); + // the native route runs them here, as Linux does + const ported = process.platform === "darwin" && bundle !== null && !native; const prefix = config.containerPrefix || "layerfs-"; const windows = process.platform === "win32" ? windowsConfig(config, home, prefix) : null; const spec = { @@ -79,6 +96,10 @@ export function resolveConfig(config, profile) { traceDir: config.traceDir || join(temp, "traces"), traceInWorkspace: process.platform === "win32" && !config.traceDir, pollMs: config.pollMs ?? 400, + native, + localDir: native ? config.localDir || join(temp, "local") : null, + stores: native ? config.stores || join(temp, "stores") : null, + nfsMount: native ? config.nfsMount || null : null, sdk: windows ? new URL("./windows/sdk.js", import.meta.url).href : ported ? new URL("./port/sdk.js", import.meta.url).href : import.meta.resolve("@layerfs/sdk"), operator: windows ? new URL("./windows/operator.js", import.meta.url).href : ported ? new URL("./port/operator.js", import.meta.url).href : import.meta.resolve("@layerfs/sdk/operator"), simSdk: new URL("./sim/sdk.js", import.meta.url).href, @@ -87,11 +108,20 @@ export function resolveConfig(config, profile) { if (windows) spec.windows = windows; for (const dir of [spec.stateDir, spec.traceDir]) mkdirSync(dir, { recursive: true }); mkdirSync(spec.home, { recursive: true, mode: 0o700 }); + if (native) { + // a local Sandbox root is this user's, not group/other writable and traversable (0755) + mkdirSync(spec.localDir, { recursive: true, mode: 0o755 }); + chmodSync(spec.localDir, 0o755); + mkdirSync(spec.stores, { recursive: true, mode: 0o700 }); + } return spec; } -/** Deployment settings Init and bash calls need that are not configured (macOS: a bundle too). */ -export const missingDeployment = (spec) => ["image", "daemon", ...(spec.windows || process.platform === "darwin" ? ["bundle"] : [])].filter((name) => !spec[name]); +/** Deployment settings Init and bash calls need that are not configured (macOS: a bundle too; native: the daemon only). */ +export const missingDeployment = (spec) => (spec.native ? ["daemon"] : ["image", "daemon", ...(spec.windows || process.platform === "darwin" ? ["bundle"] : [])]).filter((name) => !spec[name]); + +/** Where the native route keeps a project's Store: `/`, as a Docker volume's `_data`. */ +export const storeDirOf = (spec, volume) => join(spec.stores, volume); /** The SDK entry as a path, for Node imports in the Host. */ export const sdkPath = (spec) => fileURLToPath(spec.sdk); diff --git a/dsh/packages/host/src/env.js b/dsh/packages/host/src/env.js index 51050551bf..faaa016833 100644 --- a/dsh/packages/host/src/env.js +++ b/dsh/packages/host/src/env.js @@ -29,6 +29,9 @@ const VARIABLES = { DSH_LAYERFS_STORE_CONTAINER: "macOS: the Store container (Linux) the SDK's calls run in", DSH_LAYERFS_BUNDLE: "macOS: the LayerFS build whose SDK DSH_LAYERFS_SDK runs", DSH_LAYERFS_WSL_ENDPOINT: "Windows: private endpoint file of the WSL LayerFS worker", + DSH_LAYERFS_SANDBOX: "native: Sandboxes are local daemons on this machine (no Docker); Workspaces are NFS mounts under DSH_LAYERFS_LOCAL_DIR and commands run as you", + DSH_LAYERFS_LOCAL_DIR: "native: the folder local Sandboxes and their Workspace mounts live in", + DSH_LAYERFS_NFS_MOUNT: "native: the NFS mount command the Sandbox daemon uses, when the default is overridden", DSH_LAYERFS_TRACE: "File the SDK appends one line per call to", DSH_LAYERFS_TRACE_STEP: "Label of this bash call on the SDK's trace lines", DSH_LAYERFS_TRACE_START: "1: the SDK also writes a line when each call starts", @@ -41,13 +44,17 @@ const VARIABLES = { * bash call that uses the project still locks the choice. * @param {import("./binding.js").BindingStore} bindings * @param {import("./collector.js").Collector} collector + * @param {object} [spec] resolved Host settings (the native route adds one sentence) */ -export function projectContext(bindings, collector, session) { +export function projectContext(bindings, collector, session, spec) { if (!session) return ""; const agent = collector.agentOf(session); const project = bindings.projectFor(agent.root, agent.cwd); if (!project) return ""; - return `LayerFS project of this conversation: ${project.name} (${isSim(project) ? "simulated" : "live"}). Every bash call has it as DSH_LAYERFS_PROJECT; work on it through the LayerFS SDK.`; + const native = spec?.native && !isSim(project) + ? " Its Sandboxes run natively on this machine: a Workspace is an NFS mount, and the path workspace.mount returns may also be your bash commands' working directory (commit through the SDK)." + : ""; + return `LayerFS project of this conversation: ${project.name} (${isSim(project) ? "simulated" : "live"}). Every bash call has it as DSH_LAYERFS_PROJECT; work on it through the LayerFS SDK.${native}`; } /** diff --git a/dsh/packages/host/src/index.js b/dsh/packages/host/src/index.js index 6beaecadd3..fd6d91c9cf 100644 --- a/dsh/packages/host/src/index.js +++ b/dsh/packages/host/src/index.js @@ -37,6 +37,10 @@ export const Config = z.object({ home: z.string().description("LayerFS home for discovery records (default: /dsh-layerfs/home)"), stateDir: z.string().description("Project bindings and agent names (default: /dsh-layerfs)"), traceDir: z.string().description("Per-agent SDK traces (Windows default: /.dsh-layerfs-traces; other platforms: /dsh-layerfs/traces)"), + sandbox: z.union(["docker", "native"]).description("Where Live projects run: docker (default) or native (macOS: local daemons, loopback NFS, no Docker)"), + localDir: z.string().description("native: the folder local Sandboxes live in (default: /dsh-layerfs/local)"), + stores: z.string().description("native: the folder of each project's Store (default: /dsh-layerfs/stores)"), + nfsMount: z.string().description("native: LAYERFS_NFS_MOUNT for Sandbox daemons, when mount_nfs needs other options"), pollMs: z.natural().min(50).max(10_000).default(400).description("How often trace files are read, in milliseconds"), }); @@ -50,6 +54,8 @@ export function apply(ctx, config) { ensureWindows(spec).catch((error) => ctx.logger?.warn?.(`layerfs: WSL worker did not start: ${error.message}`)); ctx.effect(() => () => closeWindows(spec).catch((error) => ctx.logger?.warn?.(`layerfs: ${error.message}`)), "layerfs-host: WSL worker"); } + // native: a Sandbox this process starts reads the daemon's mount command from its environment + if (spec.nfsMount) process.env.LAYERFS_NFS_MOUNT = spec.nfsMount; if (spec.box) { // the Host's own clients use the ported SDK too (port/sdk.js) Object.assign(process.env, { DSH_LAYERFS_STORE_CONTAINER: spec.box, DSH_LAYERFS_BUNDLE: spec.bundle }); @@ -79,7 +85,7 @@ export function apply(ctx, config) { // right after the working directory, the other fact about where the agent works order: scope.systemPrompt.getContextOrder("WORKING_DIRECTORY") + 1, interpolate: false, - text: (context) => (context.agent ? projectContext(bindings, collector, context.agent.session) : ""), + text: (context) => (context.agent ? projectContext(bindings, collector, context.agent.session, spec) : ""), }); }); } diff --git a/dsh/packages/host/src/init.js b/dsh/packages/host/src/init.js index 355e217de7..d46ad997eb 100644 --- a/dsh/packages/host/src/init.js +++ b/dsh/packages/host/src/init.js @@ -3,7 +3,7 @@ // (backend "sim") is imported into the simulator (sim/) instead: no Docker, // no deployment settings. import { execFile } from "node:child_process"; -import { accessSync, appendFileSync, constants, existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { accessSync, appendFileSync, constants, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; import { lstat, opendir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, isAbsolute, join } from "node:path"; @@ -12,7 +12,7 @@ import { setTimeout as pause } from "node:timers/promises"; import { promisify } from "node:util"; import { ensureBox } from "./box.js"; import { ensureWindows, windowsDocker } from "./windows/host.js"; -import { missingDeployment } from "./config.js"; +import { missingDeployment, storeDirOf } from "./config.js"; import { clientFor, isSim, release } from "./layerfs-client.js"; const run = promisify(execFile); @@ -86,12 +86,18 @@ export function liveOnlyDemo(name, scenarios = SCENARIOS) { * @returns {string|null} */ export function hostAccess(spec, { platform = process.platform, uid = process.getuid?.() } = {}) { + if (spec.native && uid !== undefined) return homeOwner(spec, uid); if (platform !== "linux" || spec.box || uid === undefined) return null; try { accessSync(spec.volumes, constants.R_OK | constants.W_OK | constants.X_OK); } catch (error) { if (error.code !== "ENOENT") return `this Host runs as uid ${uid} and cannot write Docker's volumes directory ${spec.volumes} (${error.code}), where Init installs the Store; run dsh as a user that can, such as root`; } + return homeOwner(spec, uid); +} + +/** The native binding requires the LayerFS home to belong to this process's user (the native route: any OS). */ +function homeOwner(spec, uid) { const owner = existsSync(spec.home) ? statSync(spec.home).uid : uid; if (owner !== uid) return `the LayerFS home ${spec.home} belongs to uid ${owner}, not this Host's uid ${uid}; set layerfs-host home to a folder this user owns`; return null; @@ -153,7 +159,9 @@ async function initFrom(spec, bindings, { folder, project }, report) { report({ step: "volume", totals }); const docker = ["-H", `unix://${spec.docker}`]; if (spec.windows && !isSim(project)) await ensureWindows(spec); - if (!isSim(project)) await (spec.windows ? windowsDocker(spec, ["volume", "create", project.volume]) : run("docker", [...docker, "volume", "create", project.volume])); + // the native route's "volume" is a local folder /, made here as Docker makes a volume + if (!isSim(project) && spec.native) mkdirSync(storeDirOf(spec, project.volume), { mode: 0o700 }); + else if (!isSim(project)) await (spec.windows ? windowsDocker(spec, ["volume", "create", project.volume]) : run("docker", [...docker, "volume", "create", project.volume])); report({ step: "import" }); let client; try { @@ -170,7 +178,8 @@ async function initFrom(spec, bindings, { folder, project }, report) { } // Init made the project's entry unless another project holds the name if (!isSim(project) && error?.code !== "Exists") rmSync(join(spec.home, project.name), { recursive: true, force: true }); - if (!isSim(project)) await run("docker", [...docker, "volume", "rm", project.volume]).catch(() => {}); + if (!isSim(project) && spec.native) rmSync(storeDirOf(spec, project.volume), { recursive: true, force: true }); + else if (!isSim(project)) await run("docker", [...docker, "volume", "rm", project.volume]).catch(() => {}); throw error; } await release(client); diff --git a/dsh/packages/host/src/layerfs-client.js b/dsh/packages/host/src/layerfs-client.js index ca8688b819..57d1b864c4 100644 --- a/dsh/packages/host/src/layerfs-client.js +++ b/dsh/packages/host/src/layerfs-client.js @@ -7,6 +7,7 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { Mirror } from "@dsh-layerfs/model/mirror.js"; import { DAEMON_HOST, storePathOf } from "./box.js"; +import { storeDirOf } from "./config.js"; import { simClient } from "./sim/client.js"; export const isSim = (project) => project.backend === "sim"; @@ -16,6 +17,18 @@ export function deploymentEnv(spec, project) { // the SDK asks for every deployment setting and, for Store calls, a Store // path (#352 F-020); a simulated project uses none of them if (isSim(project)) return { HOME: spec.home, DOCKER: "sim", IMAGE: "sim", DAEMON: "sim", VOLUME: project.volume, STORE_PATH: "sim" }; + // the native route: Store calls in this process on /, Sandboxes as local + // daemons under localDir whose Workspaces this machine's NFS client mounts (SDK `native`) + if (spec.native) return { + HOME: spec.home, + SANDBOX: "native", + DAEMON: spec.daemon ?? "", + LOCAL_DIR: spec.localDir, + VOLUME: project.volume, + STORE_PATH: join(storeDirOf(spec, project.volume), "global", "store.sqlite"), + CONTAINER_PREFIX: spec.prefix, + ...(spec.nfsMount ? { NFS_MOUNT: spec.nfsMount } : {}), + }; return { HOME: spec.windows?.home ?? spec.home, DOCKER: spec.docker, diff --git a/dsh/packages/host/src/remove.js b/dsh/packages/host/src/remove.js index 2cfc6ed84b..da3f4f00dc 100644 --- a/dsh/packages/host/src/remove.js +++ b/dsh/packages/host/src/remove.js @@ -17,6 +17,7 @@ import { execFile } from "node:child_process"; import { existsSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { join, relative } from "node:path"; import { promisify } from "node:util"; +import { storeDirOf } from "./config.js"; import { isSim } from "./layerfs-client.js"; import { stopProcesses } from "./sim/binding.js"; import { removeWindowsHome, releaseWindowsStore, windowsDocker } from "./windows/host.js"; @@ -58,6 +59,7 @@ export async function removeProject(spec, bindings, trees, name, run = execute, bindings.remove(project.name); return { removed: project.name, volume: project.volume }; } + if (spec.native) return removeNative(spec, bindings, project, home, run); const docker = ["-H", `unix://${spec.docker}`]; if (spec.windows) run = (_file, args) => windowsDocker(spec, args.slice(2)); const { stdout } = await run("docker", [...docker, "ps", "-a", "--filter", `volume=${project.volume}`, "--format", "{{.ID}} {{.State}}"]); @@ -75,3 +77,39 @@ export async function removeProject(spec, bindings, trees, name, run = execute, bindings.remove(project.name); return { removed: project.name, volume: project.volume }; } + +/** pgrep's matching lines, "" when none matches (it exits 1). Any other failure (pgrep missing or + * refused) refuses the removal: a daemon that cannot be ruled out may still serve a mount there. */ +const daemons = (run, args) => Promise.resolve().then(() => run("pgrep", args)).then((r) => r.stdout, (error) => { + if (error?.code === 1) return ""; + throw refuse("busy", `cannot tell whether a Sandbox daemon still runs (pgrep: ${error?.message ?? error}); nothing was removed`); +}); +/** The mount table; when it cannot be read the removal is refused. */ +const mountTable = (run) => Promise.resolve().then(() => run("mount", [])).then((r) => r.stdout, (error) => { + throw refuse("busy", `cannot read the mount table (mount: ${error?.message ?? error}); nothing was removed`); +}); + +/** + * The native route's removal: the project's local Sandboxes are folders + * `/-` with a daemon process each. It refuses + * while one of their daemons runs or a Workspace of theirs is still in the + * mount table (a live NFS mount must never be deleted through), else + * deletes their folders, the Store folder and the home entry. + */ +async function removeNative(spec, bindings, project, home, run) { + const own = new RegExp(`^${escape(spec.prefix + project.name)}-[0-9]+$`); + const folders = existsSync(spec.localDir) ? readdirSync(spec.localDir).filter((entry) => own.test(entry)).map((entry) => join(spec.localDir, entry)) : []; + // each match with its command line in one pgrep (macOS -l, Linux -a), never ps: macOS's + // ps is setuid root, which a Seatbelt-sandboxed Host may not execute + const listed = (await daemons(run, [process.platform === "darwin" ? "-lf" : "-af", `layerfs-daemon --config ${spec.localDir}/`])).split("\n").filter(Boolean); + const running = listed.filter((line) => folders.some((folder) => line.includes(`${folder}/`))).map((line) => line.split(" ")[0]); + const mounts = (await mountTable(run)).split("\n").filter((line) => folders.some((folder) => line.includes(` on ${folder}/`))); + if (running.length || mounts.length) throw refuse("busy", `${running.length} Sandbox daemon${running.length === 1 ? "" : "s"} and ${mounts.length} Workspace mount${mounts.length === 1 ? "" : "s"} of ${project.name} are live; a Sandbox closes 2 minutes after its last Workspace is unmounted`); + for (const folder of folders) rmSync(folder, { recursive: true, force: true }); + rmSync(storeDirOf(spec, project.volume), { recursive: true, force: true }); + if (existsSync(home)) rmSync(home, { recursive: true, force: true }); + bindings.remove(project.name); + return { removed: project.name, volume: project.volume }; +} + +const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); diff --git a/dsh/packages/host/src/route.js b/dsh/packages/host/src/route.js index c60b880ab7..695afeff62 100644 --- a/dsh/packages/host/src/route.js +++ b/dsh/packages/host/src/route.js @@ -73,7 +73,7 @@ export function registerRoutes(ctx, { spec, bindings, collector, trees, jobs = n const url = new URL(String(req.url), "http://localhost"), parts = url.pathname.slice(PREFIX.length).split("/").filter(Boolean); try { if (req.method === "GET" && parts[0] === "status" && parts.length === 1) - return send(res, 200, { configured: missingDeployment(spec).length === 0, missing: missingDeployment(spec), sdk: spec.sdk, operator: spec.operator, box: spec.box, bundle: spec.bundle, traceDir: spec.traceDir, home: spec.home, docker: spec.docker, image: spec.image, daemon: spec.daemon, empty: true, migration: spec.migration ?? null, ...(spec.windows ? { windows: spec.windows } : {}) }); + return send(res, 200, { configured: missingDeployment(spec).length === 0, missing: missingDeployment(spec), sdk: spec.sdk, operator: spec.operator, box: spec.box, bundle: spec.bundle, sandbox: spec.native ? "native" : "docker", localDir: spec.localDir ?? null, stores: spec.stores ?? null, traceDir: spec.traceDir, home: spec.home, docker: spec.docker, image: spec.image, daemon: spec.daemon, empty: true, migration: spec.migration ?? null, ...(spec.windows ? { windows: spec.windows } : {}) }); if (req.method === "GET" && parts[0] === "workspace" && parts.length === 1) { const folder = url.searchParams.get("folder"); if (!folder) return send(res, 400, { code: "invalid-folder", message: "folder is required" }); diff --git a/dsh/packages/host/src/runtime.js b/dsh/packages/host/src/runtime.js index d8167a3f13..af0e59ccfd 100644 --- a/dsh/packages/host/src/runtime.js +++ b/dsh/packages/host/src/runtime.js @@ -195,12 +195,15 @@ export async function cleanLeftovers(spec, bindings, names) { } async function engineState(spec) { + // the native route runs no container: its Sandboxes are local daemons, read through the SDK + if (spec.native) return { level: "ok", version: "native (no Docker)", native: true }; try { return { level: "ok", version: await docker(spec, "version", "--format", "{{.Server.Version}}") }; } catch (error) { return { level: "down", error: error.stderr?.trim() || error.message }; } } /** Gives each Sandbox (`{ image }`, an exact digest; one project's may differ) its `imageName`: the * image's first tag, else null. One `docker image inspect` for all; an image Docker no longer has stays unnamed. */ export async function nameImages(spec, sandboxes) { + if (spec.native) { for (const s of sandboxes) s.imageName = null; return sandboxes; } const digests = [...new Set(sandboxes.map((s) => s.image).filter(Boolean))]; const names = new Map(); if (digests.length) { @@ -218,6 +221,7 @@ export async function nameImages(spec, sandboxes) { /** This home's Sandbox containers (named `-`), running or not. */ async function containersOf(spec) { + if (spec.native) return []; const out = await docker(spec, "ps", "-a", "--no-trunc", "--filter", "label=layerfs.role=sandbox", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}\t{{.Label \"layerfs.project\"}}\t{{.Label \"layerfs.sandbox\"}}\t{{.Image}}\t{{.Ports}}"); return out.split("\n").filter(Boolean).map((row) => { const [name, state, status, project, sandbox, image, ports] = row.split("\t"); diff --git a/dsh/packages/host/tests/native.mjs b/dsh/packages/host/tests/native.mjs new file mode 100644 index 0000000000..69cc72a09f --- /dev/null +++ b/dsh/packages/host/tests/native.mjs @@ -0,0 +1,147 @@ +// The native route's Host settings (sandbox: native), without dsh, Docker or a Store: +// node dsh/packages/host/tests/native.mjs +// Configuration and its defaults, the bash environment an agent gets, Init's +// checks and its Store folder instead of a Docker volume, removal of a +// project's local Sandboxes, and the overview that asks no Docker. The +// Docker route's defaults are checked unchanged beside it. +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { register } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const STAND_IN = "export function appendPluginRecord(session, type, data) { session.records.push({ type, data }); }"; +register(`data:text/javascript,${encodeURIComponent(` +export async function resolve(specifier, context, next) { + if (specifier === "@deepseek-ai/dsh-session") return { url: ${JSON.stringify(`data:text/javascript,${encodeURIComponent(STAND_IN)}`)}, shortCircuit: true }; + return next(specifier, context); +}`)}`); +const { resolveConfig, missingDeployment, storeDirOf } = await import("../src/config.js"); +const { BindingStore } = await import("../src/binding.js"); +const { Collector } = await import("../src/collector.js"); +const { contributor, projectContext } = await import("../src/env.js"); +const { deploymentEnv } = await import("../src/layerfs-client.js"); +const { hostAccess, prepareInit } = await import("../src/init.js"); +const { removeProject } = await import("../src/remove.js"); +const { overview } = await import("../src/runtime.js"); +const { deployment } = await import("../../../../sdk/js/src/client.js"); + +const dir = mkdtempSync(join(tmpdir(), "dsh-layerfs-native-")); +const FOLDER = "/work/website"; +const session = (id, cwd) => ({ header: { id, cwd }, records: [] }); +const saved = process.env.LAYERFS_SANDBOX; + +try { + // the Docker route is the default, unchanged: image and daemon (and a bundle on macOS) are required + delete process.env.LAYERFS_SANDBOX; + const plain = resolveConfig({ home: join(dir, "plain-home"), traceDir: join(dir, "t") }, { dir: join(dir, "p0") }); + assert.equal(plain.native, false); + assert.deepEqual([plain.localDir, plain.stores], [null, null]); + assert.deepEqual(missingDeployment(plain), ["image", "daemon", ...(process.platform === "darwin" ? ["bundle"] : [])]); + // LAYERFS_SANDBOX=local is the SDK's; it does not select anything here + process.env.LAYERFS_SANDBOX = "local"; + assert.equal(resolveConfig({ home: join(dir, "plain-home"), traceDir: join(dir, "t") }, { dir: join(dir, "p0") }).native, false); + assert.throws(() => resolveConfig({ sandbox: "vm", home: join(dir, "plain-home") }, { dir: join(dir, "p0") }), /docker or native, not vm/); + + // native: from the config, or the dsh process's LAYERFS_SANDBOX=native; a bundle never ports it + process.env.LAYERFS_SANDBOX = "native"; + const byEnv = resolveConfig({ home: join(dir, "env-home"), traceDir: join(dir, "t"), bundle: "/b", localDir: join(dir, "env-local"), stores: join(dir, "env-stores") }, { dir: join(dir, "p1") }); + assert.equal(byEnv.native, true); + assert.equal(byEnv.box, null, "the native route runs no Store container, even with a bundle on macOS"); + assert.match(byEnv.sdk, /\/sdk\/js\/src\/index\.js$/); + delete process.env.LAYERFS_SANDBOX; + + const spec = resolveConfig({ sandbox: "native", home: join(dir, "home"), traceDir: join(dir, "traces"), localDir: join(dir, "local"), stores: join(dir, "stores") }, { dir: join(dir, "profile") }); + assert.deepEqual(missingDeployment(spec), ["daemon"], "native needs the daemon only: no image, no bundle"); + assert.equal(statSync(spec.localDir).mode & 0o777, 0o755, "a local Sandbox root is 0755"); + assert.ok(existsSync(spec.stores)); + // defaults: everything under the OS temp area, which dsh's workspace-write sandbox may write + const defaults = resolveConfig({ sandbox: "native", daemon: "/d/layerfs-daemon", stateDir: join(dir, "state-defaults") }, undefined); + assert.ok(defaults.localDir.endsWith("/dsh-layerfs/local") && defaults.stores.endsWith("/dsh-layerfs/stores") && defaults.home.endsWith("/dsh-layerfs/home")); + assert.deepEqual(missingDeployment(defaults), []); + + const native = { ...spec, daemon: "/d/layerfs-daemon" }; + const project = { name: "website", source: FOLDER, volume: "layerfs-website-store-abc", created: 1 }; + const env = deploymentEnv(native, project); + assert.deepEqual(env, { + HOME: native.home, SANDBOX: "native", DAEMON: "/d/layerfs-daemon", LOCAL_DIR: native.localDir, VOLUME: project.volume, + STORE_PATH: join(native.stores, project.volume, "global", "store.sqlite"), CONTAINER_PREFIX: "layerfs-", + }); + assert.equal(storeDirOf(native, project.volume), join(native.stores, project.volume)); + assert.equal(deploymentEnv({ ...native, nfsMount: "/sbin/mount_nfs -o vers=3 127.0.0.1:/ {dir}" }, project).NFS_MOUNT, "/sbin/mount_nfs -o vers=3 127.0.0.1:/ {dir}"); + + // the SDK reads that environment as its native deployment, from DSH_LAYERFS_* alone (what a bash call has) + const keep = Object.fromEntries(Object.keys(process.env).filter((k) => k.startsWith("LAYERFS_") || k.startsWith("DSH_LAYERFS_")).map((k) => [k, process.env[k]])); + for (const k of Object.keys(keep)) delete process.env[k]; + for (const [k, v] of Object.entries({ ...env, NFS_MOUNT: "/sbin/mount_nfs -o x 127.0.0.1:/ {dir}" })) process.env[`DSH_LAYERFS_${k}`] = v; + assert.deepEqual(deployment(), { sandbox: "native", daemon: "/d/layerfs-daemon", localDir: native.localDir, containerPrefix: "layerfs-" }); + assert.equal(process.env.LAYERFS_NFS_MOUNT, "/sbin/mount_nfs -o x 127.0.0.1:/ {dir}", "DSH_LAYERFS_NFS_MOUNT stands in for LAYERFS_NFS_MOUNT"); + process.env.LAYERFS_NFS_REFRESH = "0"; process.env.DSH_LAYERFS_NFS_REFRESH = "1"; + deployment(); + assert.equal(process.env.LAYERFS_NFS_REFRESH, "0", "a set LAYERFS_NFS_* wins"); + for (const k of Object.keys(process.env)) if (k.startsWith("LAYERFS_") || k.startsWith("DSH_LAYERFS_")) delete process.env[k]; + Object.assign(process.env, keep); + + // a bash call of a session on a native project: every value declared, none of Docker's + const live = new Map(), ctx = { sessions: { get: (id) => live.get(id) } }; + const bindings = new BindingStore(native.stateDir), collector = new Collector(ctx, native, bindings); + const bash = contributor(ctx, native, bindings, collector); + bindings.addProject(FOLDER, project); + const root = session("s-root", FOLDER); live.set("s-root", root); + const vars = bash.resolve({ agent: { session: root }, callId: "c1" }); + for (const key of Object.keys(vars)) assert.ok(key in bash.variables, `${key} is declared`); + assert.equal(vars.DSH_LAYERFS_SANDBOX, "native"); + assert.equal(vars.DSH_LAYERFS_LOCAL_DIR, native.localDir); + assert.equal(vars.DSH_LAYERFS_STORE_PATH, env.STORE_PATH); + assert.ok(!("DSH_LAYERFS_DOCKER" in vars) && !("DSH_LAYERFS_IMAGE" in vars) && !("DSH_LAYERFS_STORE_CONTAINER" in vars)); + assert.match(vars.DSH_LAYERFS_SDK, /\/sdk\/js\/src\/index\.js$/); + assert.match(projectContext(bindings, collector, root, native), /run natively on this machine/); + assert.doesNotMatch(projectContext(bindings, collector, root, plain), /natively/); + + // Init: daemon required; the Store folder is made instead of a Docker volume; the home must be this user's + assert.throws(() => prepareInit(spec, bindings, { folder: dir }), { code: "not-configured", message: /set daemon for/ }); + const prepared = prepareInit(native, bindings, { folder: dir, name: "shop" }); + assert.match(prepared.project.volume, /^layerfs-shop-store-/); + const uid = process.getuid?.() ?? 1000; + assert.equal(hostAccess(native, { platform: "darwin", uid }), null); + assert.match(hostAccess(native, { platform: "darwin", uid: uid + 1 }), /belongs to uid \d+, not this Host's uid/); + + // removal: refused while a daemon of its Sandboxes runs or a Workspace is mounted; else its folders, Store folder and home entry go + const sb1 = join(native.localDir, "layerfs-website-1"), sb2 = join(native.localDir, "layerfs-website-2"), other = join(native.localDir, "layerfs-website-x-1"); + for (const d of [sb1, sb2, other]) mkdirSync(join(d, "workspaces"), { recursive: true }); + mkdirSync(join(native.stores, project.volume, "global"), { recursive: true }); + writeFileSync(join(native.stores, project.volume, "global", "store.sqlite"), "x"); + mkdirSync(join(native.home, "website"), { recursive: true }); + const fakeTrees = { forget: async () => {} }; + const runWith = ({ pids = "", commands = {}, mounts = "" }) => async (file, args) => { + if (file === "pgrep") { + assert.deepEqual(args.slice(0, 1), [process.platform === "darwin" ? "-lf" : "-af"]); + if (!pids) throw Object.assign(new Error("none"), { code: 1 }); + return { stdout: pids.split("\n").filter(Boolean).map((pid) => `${pid} ${commands[pid] ?? ""}\n`).join("") }; + } + if (file === "mount") return { stdout: mounts }; + throw new Error(`unexpected ${file}`); + }; + await assert.rejects(removeProject(native, bindings, fakeTrees, "website", runWith({ pids: "41\n42\n", commands: { 41: `layerfs-daemon --config ${sb1}/config/daemon.setup`, 42: `layerfs-daemon --config ${other}/config/daemon.setup` } })), { code: "busy", message: /1 Sandbox daemon and 0 Workspace mounts/ }); + await assert.rejects(removeProject(native, bindings, fakeTrees, "website", runWith({ mounts: `127.0.0.1:/ on ${sb2}/workspaces/w1 (nfs)\n` })), { code: "busy", message: /1 Workspace mount/ }); + // a check that cannot run refuses and removes nothing: pgrep failing otherwise than "no match", or no mount table + const failing = (fail) => async (file, args) => { + if (file === fail) throw Object.assign(new Error(`${file} refused`), { code: fail === "pgrep" ? 3 : "EPERM" }); + return runWith({})(file, args); + }; + await assert.rejects(removeProject(native, bindings, fakeTrees, "website", failing("pgrep")), { code: "busy", message: /cannot tell whether a Sandbox daemon still runs/ }); + await assert.rejects(removeProject(native, bindings, fakeTrees, "website", failing("mount")), { code: "busy", message: /cannot read the mount table/ }); + assert.ok(existsSync(sb1) && existsSync(sb2) && existsSync(join(native.stores, project.volume)), "nothing was removed"); + // another project's daemon and mount (website-x) do not hold website + assert.deepEqual(await removeProject(native, bindings, fakeTrees, "website", runWith({ pids: "42\n", commands: { 42: `layerfs-daemon --config ${other}/config/daemon.setup` }, mounts: `127.0.0.1:/ on ${other}/workspaces/w1 (nfs)\n` })), { removed: "website", volume: project.volume }); + assert.deepEqual([existsSync(sb1), existsSync(sb2), existsSync(other), existsSync(join(native.stores, project.volume)), existsSync(join(native.home, "website")), bindings.byName("website")], [false, false, true, false, false, null]); + + // the overview asks no Docker: the engine is native, no containers, no Store container + const view = await overview(native, bindings, { spec: native }, { runningCalls: () => 0 }); + assert.deepEqual([view.engine.native, view.store, view.leftovers, view.level], [true, null, [], "ok"]); + + console.log("native: ok"); +} finally { + if (saved === undefined) delete process.env.LAYERFS_SANDBOX; else process.env.LAYERFS_SANDBOX = saved; + rmSync(dir, { recursive: true, force: true }); +} diff --git a/dsh/packages/host/tests/native_real.mjs b/dsh/packages/host/tests/native_real.mjs new file mode 100644 index 0000000000..b55256d994 --- /dev/null +++ b/dsh/packages/host/tests/native_real.mjs @@ -0,0 +1,165 @@ +// The native route end to end, with the real SDK and a native LayerFS build, without dsh: +// node dsh/packages/host/tests/native_real.mjs +// env: LAYERFS_NATIVE (the addon, else sdk/js/native/layerfs.node) +// NATIVE_WORKSPACE=0 skips the Workspace part (a machine whose user cannot NFS-mount, as a Linux CI box) +// NATIVE_NFS_MOUNT: the layerfs-host nfsMount setting (reaches agents as DSH_LAYERFS_NFS_MOUNT) +// NATIVE_KEEP=1 keeps the test folder; NATIVE_IDLE_WAIT= waits that long for idle Sandboxes before removal (default 150) +// The Host side as dsh web runs it: settings (sandbox native), Init (which +// imports a folder into /, starting no Sandbox), the bash +// environment of a session on the project, then agent bash calls as +// separate processes with that environment alone: the lfs CLI (status, +// fork, write, run in a Workspace), a script that mounts a Workspace, writes +// into its mount path from a shell whose working directory it is, commits +// and unmounts, and a merge and publish. The Host then reads History (its +// shared client) and main's files, the trace lines are counted, and the +// project is removed once its Sandbox closed. No Docker is asked anywhere. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { register } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const STAND_IN = "export function appendPluginRecord(session, type, data) { session.records.push({ type, data }); }"; +register(`data:text/javascript,${encodeURIComponent(` +export async function resolve(specifier, context, next) { + if (specifier === "@deepseek-ai/dsh-session") return { url: ${JSON.stringify(`data:text/javascript,${encodeURIComponent(STAND_IN)}`)}, shortCircuit: true }; + return next(specifier, context); +}`)}`); +const { resolveConfig, missingDeployment } = await import("../src/config.js"); +const { BindingStore } = await import("../src/binding.js"); +const { Collector } = await import("../src/collector.js"); +const { contributor } = await import("../src/env.js"); +const { initProject } = await import("../src/init.js"); +const { TreeReader } = await import("../src/trees.js"); +const { removeProject } = await import("../src/remove.js"); + +const daemon = process.argv[2]; +if (!daemon || !existsSync(daemon)) { console.log("usage: node native_real.mjs "); process.exit(2); } +const WORKSPACE = process.env.NATIVE_WORKSPACE !== "0"; +const t0 = Date.now(), at = () => `${((Date.now() - t0) / 1000).toFixed(1)}s`; +const say = (text) => console.log(`[${at()}] ${text}`); +const work = mkdtempSync(join(realpathSync(tmpdir()), "dsh-native-real-")); +const folder = join(work, "shop"); +mkdirSync(join(folder, "src"), { recursive: true }); +writeFileSync(join(folder, "README.md"), "# Shop\n"); +writeFileSync(join(folder, "src", "app.js"), "console.log('shop');\n"); +writeFileSync(join(folder, ".lww"), "%include .gitignore\n"); +writeFileSync(join(folder, ".gitignore"), "node_modules/\n"); + +const spec = resolveConfig({ sandbox: "native", daemon, home: join(work, "home"), traceDir: join(work, "traces"), localDir: join(work, "local"), stores: join(work, "stores"), stateDir: join(work, "state"), ...(process.env.NATIVE_NFS_MOUNT ? { nfsMount: process.env.NATIVE_NFS_MOUNT } : {}) }, undefined); +const mounts = () => (spawnSync("mount", [], { encoding: "utf8" }).stdout || "").split("\n").filter((l) => l.includes(` on ${spec.localDir}/`)); +const daemons = () => (spawnSync("pgrep", ["-f", `layerfs-daemon --config ${spec.localDir}/`], { encoding: "utf8" }).stdout || "").split("\n").filter(Boolean); +let ok = false; +try { + say(`work=${work} platform=${process.platform} node=${process.version} workspace=${WORKSPACE}`); + assert.deepEqual(missingDeployment(spec), [], "status: configured"); + + // Init: the Store folder, the import (counts read while it runs), the folder's default project + const steps = []; + const bindings = new BindingStore(spec.stateDir); + const project = await initProject(spec, bindings, { folder, name: "shop" }, (r) => steps.push(r.step ?? (r.imported ? "imported" : r.scanned ? "scanned" : "?"))); + say(`init: ${project.name} volume=${project.volume} steps=${[...new Set(steps)].join(",")}`); + assert.ok(existsSync(join(spec.stores, project.volume, "global", "store.sqlite")), "the Store is installed in //global"); + assert.equal(bindings.workspace(folder).projects.length, 1); + + // the bash environment of a session in that folder: what dsh gives an agent's bash call + const live = new Map(), ctx = { sessions: { get: (id) => live.get(id) } }; + const collector = new Collector(ctx, spec, bindings); + const root = { header: { id: "session-root", cwd: folder }, records: [] }; + live.set(root.header.id, root); + const vars = contributor(ctx, spec, bindings, collector).resolve({ agent: { session: root }, callId: "call_1" }); + assert.equal(vars.DSH_LAYERFS_PROJECT, "shop"); + const env = { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR ?? "/tmp", ...(process.env.LAYERFS_NATIVE ? { LAYERFS_NATIVE: process.env.LAYERFS_NATIVE } : {}), ...vars }; + const bash = (command, input) => { + const r = spawnSync("bash", ["-c", command], { cwd: folder, env, input, encoding: "utf8", timeout: 300_000 }); + say(`bash$ ${command.split("\n")[0].slice(0, 90)} -> rc=${r.status}\n${(r.stdout + r.stderr).trim().split("\n").map((l) => " " + l).join("\n")}`); + return r; + }; + const cli = 'node "$DSH_LAYERFS_CLI"'; + assert.equal(bash(`${cli} status`).status, 0); + assert.equal(bash(`${cli} fork feat`).status, 0); + assert.equal(bash(`printf 'hello native\\n' | ${cli} write feat /hello.txt "add hello"`).status, 0); + assert.equal(bash(`${cli} cat feat /hello.txt`).stdout.trim(), "hello native"); + if (WORKSPACE) { + // one Workspace through the CLI: mount, exec, Commit, unmount + const run = bash(`${cli} run feat "echo from-workspace > ws.txt && id -u && ls"`); + assert.equal(run.status, 0); + assert.match(run.stdout, /ws\.txt/); + // the mount path as a shell's working directory, as an agent's own bash would use it + const script = ` + const { LayerFS } = await import(process.env.DSH_LAYERFS_SDK); + const { spawnSync } = await import("node:child_process"); + const lf = new LayerFS(process.env.DSH_LAYERFS_PROJECT); + const ws = await lf.workspace.mount("feat"); + console.log("MOUNT", ws.sandbox, ws.path); + const sh = spawnSync("bash", ["-c", "echo direct > direct.txt && cat ws.txt && pwd -P"], { cwd: ws.path, encoding: "utf8" }); + console.log("SHELL", sh.status, sh.stdout.trim().replace(/\\n/g, " | "), sh.stderr.trim()); + const c = await lf.workspace.commit(ws, "direct write in the mount path"); + console.log("COMMIT", c.kind); + await lf.workspace.unmount(ws); + console.log("UNMOUNTED"); + await lf.close?.();`; + const direct = bash(`node --input-type=module -e '${script.replace(/'/g, "'\\''")}'`); + assert.equal(direct.status, 0); + assert.match(direct.stdout, /SHELL 0 from-workspace/); + assert.match(direct.stdout, /COMMIT Committed/); + assert.equal(bash(`${cli} cat feat /direct.txt`).stdout.trim(), "direct"); + } + // a second Branch, then merge both into a new one and publish it + assert.equal(bash(`${cli} fork side && printf 'side\\n' | ${cli} write side /side.txt "add side"`).status, 0); + const merge = bash(`node --input-type=module -e ' + const { LayerFS } = await import(process.env.DSH_LAYERFS_SDK); + const lf = new LayerFS(process.env.DSH_LAYERFS_PROJECT); + const m = await lf.branch.merge("feat", "side", "joined", "join feat and side"); + console.log("MERGE", m.kind); + const p = await lf.branch.publish("joined"); + console.log("PUBLISH", p.kind); + await lf.close?.();'`); + assert.equal(merge.status, 0); + assert.match(merge.stdout, /MERGE (Merged|Clean|Created|Forked|\w+)/); + assert.match(merge.stdout, /PUBLISH \w+/); + + // the Host's reads: History and main's files, through its shared client (as the History tab) + await new Promise((r) => setTimeout(r, spec.pollMs * 3)); + const trees = new TreeReader(spec); + const state = await trees.state(project); + const s = state.s; + const branchNames = Object.keys(s.branches ?? {}); + const commits = Object.keys(s.commits ?? {}).length; + say(`host state: branches=${JSON.stringify(Object.values(state.labels ?? {}).length)} keys=${Object.keys(s).join(",")} commits=${commits} branchKeys=${branchNames.join(",")}`); + const mainFiles = await trees.use(project, ({ layerfs }) => layerfs.fs.list("main", "/")); + const names = mainFiles.items.map((e) => e.name); + say(`main after publish: ${names.join(" ")}`); + for (const name of ["hello.txt", "side.txt", ...(WORKSPACE ? ["ws.txt", "direct.txt"] : [])]) assert.ok(names.includes(name), `main has ${name}`); + trees.dispose(); + + // trace lines of the agent's calls (the Host turns them into session records) + const traceFiles = readdirSync(spec.traceDir, { recursive: true }).filter((f) => String(f).endsWith(".jsonl") || String(f).includes("trace")); + const lines = traceFiles.flatMap((f) => { try { return readFileSync(join(spec.traceDir, String(f)), "utf8").split("\n").filter(Boolean); } catch { return []; } }); + say(`trace: ${traceFiles.length} file(s), ${lines.length} line(s), methods=${[...new Set(lines.map((l) => { try { return JSON.parse(l).method; } catch { return "?"; } }))].join(",")}`); + assert.ok(lines.length > 0, "the agent's SDK calls were traced"); + + // removal: refused while its Sandbox runs, then the Sandbox closes when idle (2 min) and it goes + if (WORKSPACE && daemons().length) { + await assert.rejects(removeProject(spec, bindings, { forget: async () => {} }, "shop"), { code: "busy" }); + say(`remove refused while the Sandbox runs (daemons=${daemons().length}, mounts=${mounts().length}); waiting for it to close itself`); + const until = Date.now() + Number(process.env.NATIVE_IDLE_WAIT ?? 150) * 1000; + while (daemons().length && Date.now() < until) await new Promise((r) => setTimeout(r, 2000)); + } + if (!daemons().length && !mounts().length) { + assert.deepEqual(await removeProject(spec, bindings, { forget: async () => {} }, "shop"), { removed: "shop", volume: project.volume }); + assert.ok(!existsSync(join(spec.stores, project.volume)) && !existsSync(join(spec.home, "shop"))); + say("removed: Store folder, home entry, Sandbox folders"); + } else say(`NOT removed: daemons=${daemons().length} mounts=${mounts().length} (still live)`); + ok = true; +} finally { + say(`leftovers: mounts=${mounts().length} daemons=${daemons().length}`); + for (const pid of daemons()) try { process.kill(Number(pid), "SIGTERM"); } catch {} + for (const line of mounts()) { + const path = / on (.*) \(/.exec(line)?.[1] ?? line.split(" on ")[1]?.split(" ")[0]; + if (path) spawnSync(process.platform === "darwin" ? "/sbin/umount" : "umount", ["-f", path]); + } + if (process.env.NATIVE_KEEP !== "1" && !mounts().length) rmSync(work, { recursive: true, force: true }); + console.log(ok ? `native_real: ok (${at()})` : `native_real: FAILED (${at()})`); +} diff --git a/sdk/js/README.md b/sdk/js/README.md index 9430f34d8f..d1efc04d4f 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -316,7 +316,8 @@ files' cached attributes at once. helper commands (`{port}` and `{dir}` are filled in). The default mount is `/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,timeo=20,retrans=6,nobrowse 127.0.0.1:/ {dir}`. `LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client -then revalidates within `actimeo`, 60 s). +then revalidates within `actimeo`, 60 s). A `DSH_LAYERFS_NFS_` stands in +for an unset `LAYERFS_NFS_` (a dsh bash call is given only `DSH_*` names). Limits: background processes (`operator.process.*`) still read `/proc` and are not available on macOS; many small writes cost about six times APFS's time; diff --git a/sdk/js/src/client.js b/sdk/js/src/client.js index 538cde9936..cd272aea3f 100644 --- a/sdk/js/src/client.js +++ b/sdk/js/src/client.js @@ -84,9 +84,20 @@ function localDeployment(options, env, sandbox) { if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); } if (!deploy.containerPrefix) delete deploy.containerPrefix; + if (sandbox === "native") nfsFallbacks(); return deploy; } +/** The native route's daemon reads its NFS mount helpers from this process's + * LAYERFS_NFS_* (a Sandbox start passes them on); a dsh bash call can only be + * given DSH_* names, so DSH_LAYERFS_NFS_ stands in for an unset one. */ +function nfsFallbacks() { + for (const name of ["MOUNT", "UNMOUNT", "FORCE", "FLUSH", "REFRESH"]) { + const fallback = process.env[`DSH_LAYERFS_NFS_${name}`]; + if (!process.env[`LAYERFS_NFS_${name}`] && fallback) process.env[`LAYERFS_NFS_${name}`] = fallback; + } +} + /** One client's deployment selection, including cross-process discovery * home and the Store this process opens. */ export function invokeClient(client, op, args) { From 009931c56534d3c173618f00c5ef57e8ce96dc7a Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Sat, 24 Oct 2026 10:48:12 +0000 Subject: [PATCH 46/64] layerfs-nfs: macOS mount tolerates slow answers; LOOKUP carries the directory attributes The macOS mount adds dumbtimer,timeo=300 and mutejukebox. Against a loopback server the round-trip estimator settles near its floor, so a soft call failed with ETIMEDOUT after a few seconds of a slow answer: a daemon paused for 20 s made a stat fail after 8.3 s, and with these options it succeeded after the pause. A dead export still fails calls within seconds (connection refused). LOOKUP answers, found or not, carry the directory post-op attributes as RFC 1813 allows, so the client need not ask for them. --- api/how-to-use/agent.md | 2 +- .../layerfs-daemon/src/control/export.rs | 8 +++++++- core/crates/layerfs-nfs/src/nfs.rs | 14 ++++++++++--- core/crates/layerfs-nfs/tests/export.rs | 11 ++++++++++ .../layerfs-nfs/tests/support/client.rs | 20 +++++++++++++++++++ 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/api/how-to-use/agent.md b/api/how-to-use/agent.md index 218a78b6d1..430f371270 100644 --- a/api/how-to-use/agent.md +++ b/api/how-to-use/agent.md @@ -307,7 +307,7 @@ at once. `LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's helper commands (`{port}` and `{dir}` are filled in). The default mount is -`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,timeo=20,retrans=6,nobrowse 127.0.0.1:/ {dir}`. +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. `LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are read; an integration that can set only its own prefixed names maps them diff --git a/core/crates/layerfs-daemon/src/control/export.rs b/core/crates/layerfs-daemon/src/control/export.rs index adbd75714f..2a8fb195a0 100644 --- a/core/crates/layerfs-daemon/src/control/export.rs +++ b/core/crates/layerfs-daemon/src/control/export.rs @@ -105,6 +105,12 @@ impl Mounter { /// Linux NFS client elsewhere (root, as the daemon is in its container). /// The macOS mount is soft (an export that ended fails calls after the /// retries instead of hanging them) and hidden from Finder (`nobrowse`). + /// `dumbtimer,timeo=300` keeps a slow answer from failing as a timeout: + /// against a loopback server the round-trip estimator settles near its + /// floor, and a soft call then gave up within seconds. `mutejukebox` + /// keeps an `NFS3ERR_JUKEBOX` answer from raising the "server not + /// responding" dialog. A dead export still fails calls (connection + /// refused), within seconds. pub fn host() -> Self { let helper = |words: &[&str]| Helper(words.iter().map(|w| (*w).to_owned()).collect()); if cfg!(target_os = "macos") { @@ -112,7 +118,7 @@ impl Mounter { mount: helper(&[ "/sbin/mount_nfs", "-o", - "vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,timeo=20,retrans=6,nobrowse", + "vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse", "127.0.0.1:/", "{dir}", ]), diff --git a/core/crates/layerfs-nfs/src/nfs.rs b/core/crates/layerfs-nfs/src/nfs.rs index 7b208f8d9d..b257125300 100644 --- a/core/crates/layerfs-nfs/src/nfs.rs +++ b/core/crates/layerfs-nfs/src/nfs.rs @@ -4,7 +4,7 @@ //! [`crate::changing`]. A read-only export answers every changing procedure //! `NFS3ERR_ROFS` with its empty weak cache consistency data. use crate::{ - attributes::{access, Status}, + attributes::{access, post_op_maybe, Status}, change::Tickets, changing, directory::{list, Listing}, @@ -91,15 +91,23 @@ pub fn call( 3 => { let parent = handle(kernel, arguments).map_err(garbage)?; let name = arguments.opaque(NAME_LIMIT).map_err(garbage)?; + // Both answers carry the directory's attributes (RFC 1813 + // LOOKUP3resok and LOOKUP3resfail), so the client keeps its + // cached view of the directory without a GETATTR of its own. + let directory = parent + .as_ref() + .ok() + .and_then(|parent| kernel.getattr(*parent).ok()); match parent.and_then(|parent| kernel.lookup(parent, name)) { Ok(stat) => { out.u32(Status::Ok.code()); out.opaque(&kernel.encode(stat.serial)); kernel.identity().post_op(out, &stat); - out.bool(false); + post_op_maybe(kernel.identity(), out, directory.as_ref()); } Err(status) => { - out.u32(status.code()).bool(false); + out.u32(status.code()); + post_op_maybe(kernel.identity(), out, directory.as_ref()); } } Ok(()) diff --git a/core/crates/layerfs-nfs/tests/export.rs b/core/crates/layerfs-nfs/tests/export.rs index 6c0a7c7617..96c25acc10 100644 --- a/core/crates/layerfs-nfs/tests/export.rs +++ b/core/crates/layerfs-nfs/tests/export.rs @@ -131,6 +131,17 @@ fn cookies_refusals_and_stale_handles() { ); // Missing names, a foreign handle and every changing procedure. assert_eq!(client.lookup(&root, b"absent").map(|_| ()), Err(NOENT)); + // Both LOOKUP answers carry the directory's attributes. + let root_attr = client.getattr(&root).unwrap(); + let (status, found, directory) = client.lookup_with_directory(&root, b"many"); + assert_eq!( + (status, found.map(|(handle, _)| handle)), + (0, Some(many.clone())) + ); + assert_eq!(directory, Some(root_attr.clone())); + let (status, found, directory) = client.lookup_with_directory(&root, b"absent"); + assert_eq!((status, found), (NOENT, None)); + assert_eq!(directory, Some(root_attr)); // Not a handle this export issues: wrong version or length. A version-1 // handle of another engine mount is stale (its key is gone); a forged // tag under this mount is refused (see protocol.rs). diff --git a/core/crates/layerfs-nfs/tests/support/client.rs b/core/crates/layerfs-nfs/tests/support/client.rs index 0c579d89b2..d98b8f6ff0 100644 --- a/core/crates/layerfs-nfs/tests/support/client.rs +++ b/core/crates/layerfs-nfs/tests/support/client.rs @@ -66,6 +66,9 @@ fn attr(reader: &mut Reader<'_>) -> Attr { fn post_op(reader: &mut Reader<'_>) -> Option { reader.bool().unwrap().then(|| attr(reader)) } +/// A LOOKUP answer: its status, the object's handle and attributes when +/// found, and the directory's attributes. +pub type Lookup = (u32, Option<(Vec, Attr)>, Option); /// One READDIR/READDIRPLUS entry. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Entry { @@ -175,6 +178,23 @@ impl Client { let attributes = post_op(&mut reader).expect("LOOKUP returns object attributes"); Ok((handle, attributes)) } + /// LOOKUP with both halves of its answer: the status, the object's + /// handle and attributes when found, and the directory's attributes. + pub fn lookup_with_directory(&mut self, directory: &[u8], name: &[u8]) -> Lookup { + let mut arguments = Writer::new(); + arguments.opaque(directory).opaque(name); + let (status, rest) = self.nfs(3, &arguments); + let mut reader = Reader::new(&rest); + let object = (status == OK).then(|| { + let handle = reader.opaque(64).unwrap().to_vec(); + ( + handle, + post_op(&mut reader).expect("LOOKUP returns object attributes"), + ) + }); + let parent = post_op(&mut reader); + (status, object, parent) + } pub fn access(&mut self, handle: &[u8], requested: u32) -> Result { let mut arguments = Writer::new(); arguments.opaque(handle).u32(requested); From a700af8f492ef360d150877e5d469031767e473f Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Sun, 25 Oct 2026 03:23:17 +0000 Subject: [PATCH 47/64] layerfs-nfs: keep AppleDouble files in the export's memory instead of refusing them On NFSv3 the macOS client writes extended attributes into a `._` companion file. The export refused creating one with NFS3ERR_ACCES, which fails every copy whose source carries an attribute. On macOS 27 that is almost every file: anything a downloader wrote carries com.apple.provenance, including the whole cargo registry, so a `cargo build` with its target in a Workspace failed in the build scripts of libsqlite3-sys and zstd-sys (std::fs::copy: Operation not permitted). A CREATE of such a regular file now makes a companion that lives only in the export's memory, under its own serial range (apple_double.rs). LOOKUP, GETATTR, ACCESS, READ, WRITE, SETATTR, COMMIT, REMOVE and a RENAME to another AppleDouble name work on it; READDIR does not list it; no mutation of it reaches the engine, so the Workspace stays clean (unmount is not refused as Dirty) and no Commit captures it. Companions are dropped when the export is released. Memory is bounded: 4096 companions of at most 1 MiB, 64 MiB in all; beyond the count a creation is refused as before. A directory, symlink, link or rename target under such a name is still refused. --- core/crates/layerfs-nfs/src/apple_double.rs | 326 ++++++++++++++++++++ core/crates/layerfs-nfs/src/change.rs | 18 ++ core/crates/layerfs-nfs/src/kernel.rs | 38 ++- core/crates/layerfs-nfs/src/lib.rs | 3 +- core/crates/layerfs-nfs/src/names.rs | 11 +- core/crates/layerfs-nfs/src/server.rs | 4 +- core/crates/layerfs-nfs/tests/write.rs | 101 +++++- 7 files changed, 492 insertions(+), 9 deletions(-) create mode 100644 core/crates/layerfs-nfs/src/apple_double.rs diff --git a/core/crates/layerfs-nfs/src/apple_double.rs b/core/crates/layerfs-nfs/src/apple_double.rs new file mode 100644 index 0000000000..b64bba7ab6 --- /dev/null +++ b/core/crates/layerfs-nfs/src/apple_double.rs @@ -0,0 +1,326 @@ +//! AppleDouble companions held by the export alone. +//! +//! On NFSv3 the macOS client keeps a file's extended attributes and resource +//! fork in a `._` companion file (see [`crate::names`]). When the export +//! keeps such names out of the Workspace, a CREATE of one makes a file that +//! lives only here, in memory, under its own serial range: LOOKUP, GETATTR, +//! ACCESS, READ, WRITE, SETATTR, COMMIT, REMOVE and a RENAME to another +//! AppleDouble name work on it, READDIR does not list it, and no mutation of +//! it ever reaches the engine, so the Workspace stays clean and no Commit can +//! capture it. Everything is dropped when the export is released. +//! +//! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] +//! each and [`TOTAL_BYTES`] in all. Beyond the count a creation is refused +//! (`NFS3ERR_ACCES`, as before this module), beyond the bytes a write +//! (`NFS3ERR_FBIG` per file, `NFS3ERR_NOSPC` in all). +use crate::{ + attributes::Status, + change::{How, Sattr, Tickets}, + kernel::Kernel, +}; +use layerfs_content::{filesystem::attributes::PortableMetadata, object::inode_leaf::InodeKind}; +use layerfs_fuse::operations::write::Stamp; +use layerfs_workspace::ViewStat; +use std::{ + collections::HashMap, + sync::atomic::Ordering, + time::{SystemTime, UNIX_EPOCH}, +}; + +/// The first serial of the export's own companions, far above any serial the +/// engine allocates. +pub const SERIALS: u64 = 1 << 62; +/// Companions held at once. +pub const HELD: usize = 4096; +/// Largest companion kept. +pub const FILE_BYTES: u64 = 1 << 20; +/// Bytes kept over all companions. +pub const TOTAL_BYTES: usize = 64 << 20; +/// Mode of a companion created without one. +const MODE: u32 = 0o644; + +struct Companion { + named: (u64, Vec), + data: Vec, + metadata: PortableMetadata, +} +/// The companions of one export. +#[derive(Default)] +pub(crate) struct Companions { + next: u64, + files: HashMap, + names: HashMap<(u64, Vec), u64>, + bytes: usize, +} +impl Companions { + pub(crate) fn len(&self) -> usize { + self.files.len() + } + fn stat(serial: u64, file: &Companion) -> ViewStat { + ViewStat { + serial, + kind: InodeKind::RegularFile, + metadata: file.metadata, + namespace_refs: 1, + logical_len: file.data.len() as u64, + subdirs: 0, + } + } + fn drop_serial(&mut self, serial: u64) { + if let Some(file) = self.files.remove(&serial) { + self.names.remove(&file.named); + self.bytes -= file.data.len(); + } + } + /// Resizes `serial` to `size`, within the bounds. + fn resize(&mut self, serial: u64, size: u64) -> Result<(), Status> { + if size > FILE_BYTES { + return Err(Status::FBig); + } + let file = self.files.get_mut(&serial).ok_or(Status::Stale)?; + let old = file.data.len(); + let new = size as usize; + if new > old && self.bytes + (new - old) > TOTAL_BYTES { + return Err(Status::NoSpc); + } + file.data.resize(new, 0); + self.bytes = self.bytes + new - old; + Ok(()) + } +} +fn now() -> PortableMetadata { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + PortableMetadata { + mode: MODE, + mtime_seconds: i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX), + mtime_nanoseconds: elapsed.subsec_nanos(), + } +} +fn touch(metadata: &mut PortableMetadata) { + let fresh = now(); + metadata.mtime_seconds = fresh.mtime_seconds; + metadata.mtime_nanoseconds = fresh.mtime_nanoseconds; +} +/// A serial of the export's own companions. +pub fn companion(serial: u64) -> bool { + serial >= SERIALS +} + +impl Kernel { + fn companions(&self) -> std::sync::MutexGuard<'_, Companions> { + self.companions + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + /// GETATTR of a companion serial. + pub(crate) fn companion_stat(&self, serial: u64) -> Result { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + Ok(Companions::stat(serial, file)) + } + /// LOOKUP of a companion name, if one is held. + pub(crate) fn companion_named(&self, parent: u64, name: &[u8]) -> Option { + let companions = self.companions(); + let serial = *companions.names.get(&(parent, name.to_vec()))?; + companions + .files + .get(&serial) + .map(|file| Companions::stat(serial, file)) + } + /// CREATE of an AppleDouble name. + pub(crate) fn companion_create( + &self, + parent: u64, + name: &[u8], + how: How, + tickets: &mut Tickets, + ) -> Result { + if let Some(stat) = self.companion_named(parent, name) { + // Opened again: UNCHECKED applies its size, as an open with + // truncation would; GUARDED and EXCLUSIVE find it taken. + let How::Unchecked(sattr) = how else { + return Err(Status::Exist); + }; + return match sattr.size { + Some(size) => { + let sized = Sattr { + size: Some(size), + ..Sattr::default() + }; + self.companion_setattr(stat.serial, sized, None, tickets) + } + None => Ok(stat), + }; + } + // A directory must exist and a companion the tree itself carries + // stays the tree's: neither is ours to shadow. + let directory = self.getattr(parent)?; + if directory.kind != InodeKind::Directory { + return Err(Status::NotDir); + } + let taken = match self.lookup(parent, name) { + Ok(_) => true, + Err(Status::NoEnt) => false, + Err(status) => return Err(status), + }; + let mut companions = self.companions(); + if taken || companions.len() >= HELD { + drop(companions); + self.names_refused.fetch_add(1, Ordering::Relaxed); + return Err(Status::Access); + } + let mut metadata = now(); + if let How::Unchecked(sattr) | How::Guarded(sattr) = how { + if let Some(mode) = sattr.mode { + metadata.mode = mode & 0o777; + } + } + let serial = SERIALS + companions.next; + companions.next += 1; + let named = (parent, name.to_vec()); + companions.names.insert(named.clone(), serial); + companions.files.insert( + serial, + Companion { + named, + data: Vec::new(), + metadata, + }, + ); + let file = &companions.files[&serial]; + let stat = Companions::stat(serial, file); + drop(companions); + self.names_detached.fetch_add(1, Ordering::Relaxed); + Ok(stat) + } + /// REMOVE of a companion name; `false` when none is held. + pub(crate) fn companion_remove(&self, parent: u64, name: &[u8]) -> bool { + let mut companions = self.companions(); + match companions.names.get(&(parent, name.to_vec())).copied() { + Some(serial) => { + companions.drop_serial(serial); + true + } + None => false, + } + } + /// RENAME of a companion name: only to another AppleDouble name, which + /// it replaces when held. `None` when no companion has that name. + pub(crate) fn companion_rename( + &self, + from: u64, + name: &[u8], + to: u64, + new_name: &[u8], + ) -> Option> { + let mut companions = self.companions(); + let serial = *companions.names.get(&(from, name.to_vec()))?; + if !crate::names::apple_double(new_name) || new_name.len() > 255 { + return Some(Err(Status::Access)); + } + let target = (to, new_name.to_vec()); + if let Some(replaced) = companions.names.get(&target).copied() { + if replaced != serial { + companions.drop_serial(replaced); + } + } + companions.names.remove(&(from, name.to_vec())); + companions.names.insert(target.clone(), serial); + if let Some(file) = companions.files.get_mut(&serial) { + file.named = target; + } + Some(Ok(())) + } + /// READ of a companion. + pub(crate) fn companion_read( + &self, + serial: u64, + offset: u64, + count: u32, + ) -> Result<(Vec, bool), Status> { + let companions = self.companions(); + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let length = file.data.len() as u64; + let start = offset.min(length) as usize; + let end = offset.saturating_add(u64::from(count)).min(length) as usize; + Ok((file.data[start..end].to_vec(), end as u64 >= length)) + } + /// WRITE to a companion: all of it or nothing. + pub(crate) fn companion_write( + &self, + serial: u64, + offset: u64, + data: &[u8], + ) -> Result<(u32, ViewStat), Status> { + let mut companions = self.companions(); + let end = offset.checked_add(data.len() as u64).ok_or(Status::FBig)?; + let length = companions + .files + .get(&serial) + .ok_or(Status::Stale)? + .data + .len() as u64; + if end > length { + companions.resize(serial, end)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + file.data[offset as usize..end as usize].copy_from_slice(data); + touch(&mut file.metadata); + let stat = Companions::stat(serial, file); + Ok((data.len() as u32, stat)) + } + /// SETATTR of a companion: mode, size and modification time; the + /// owner is the export's identity, as for every inode. + pub(crate) fn companion_setattr( + &self, + serial: u64, + sattr: Sattr, + guard: Option<(u32, u32)>, + _tickets: &mut Tickets, + ) -> Result { + let identity = self.identity(); + let foreign = sattr.uid.is_some_and(|uid| uid != identity.uid) + || sattr.gid.is_some_and(|gid| gid != identity.gid); + if foreign { + return Err(Status::Perm); + } + let mut companions = self.companions(); + if let Some(expected) = guard { + let file = companions.files.get(&serial).ok_or(Status::Stale)?; + let current = Companions::stat(serial, file); + if crate::attributes::ctime(¤t) != expected { + return Err(Status::NotSync); + } + } + if let Some(size) = sattr.size { + companions.resize(serial, size)?; + } + let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; + if let Some(mode) = sattr.mode { + file.metadata.mode = mode & 0o777; + } + match sattr.mtime { + Some(Stamp::At(time)) => { + file.metadata.mtime_seconds = time.seconds; + file.metadata.mtime_nanoseconds = time.nanoseconds; + } + Some(Stamp::Processing) => touch(&mut file.metadata), + None if sattr.size.is_some() => touch(&mut file.metadata), + None => {} + } + Ok(Companions::stat(serial, file)) + } + /// Drops every companion; their count. + pub(crate) fn companions_release(&self) -> usize { + let mut companions = self.companions(); + let held = companions.len(); + *companions = Companions::default(); + held + } + /// Companions held now. + pub(crate) fn companions_held(&self) -> usize { + self.companions().len() + } +} diff --git a/core/crates/layerfs-nfs/src/change.rs b/core/crates/layerfs-nfs/src/change.rs index 7122181f94..a1c5008d16 100644 --- a/core/crates/layerfs-nfs/src/change.rs +++ b/core/crates/layerfs-nfs/src/change.rs @@ -212,6 +212,9 @@ impl Kernel { how: How, tickets: &mut Tickets, ) -> Result { + if self.refuses_apple_double() && names::apple_double(name) && name.len() <= 255 { + return self.companion_create(parent, name, how, tickets); + } let checked = self.checked(name, true)?; let key = (parent, name.to_vec()); if let How::Exclusive(verifier) = how { @@ -353,6 +356,9 @@ impl Kernel { } /// REMOVE of a non-directory name. pub fn remove(&self, parent: u64, name: &[u8], tickets: &mut Tickets) -> Result<(), Status> { + if self.companion_remove(parent, name) { + return Ok(()); + } if self.silly_entry(parent, name).is_some() { self.drop_silly(parent, name); return Ok(()); @@ -387,6 +393,9 @@ impl Kernel { new_name: &[u8], tickets: &mut Tickets, ) -> Result<(), Status> { + if let Some(renamed) = self.companion_rename(from, name, to, new_name) { + return renamed; + } if let Some(serial) = self.silly_entry(from, name) { // Moving a silly name to another silly name keeps it ours; // giving it a real name again would need a link of an unlinked @@ -443,6 +452,9 @@ impl Kernel { name: &[u8], tickets: &mut Tickets, ) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::Access); + } let checked = self.checked(name, true)?; let stat = self .mutate(parent, None, Expect::Stat(Some(serial)), tickets, |_| { @@ -463,6 +475,9 @@ impl Kernel { guard: Option<(u32, u32)>, tickets: &mut Tickets, ) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_setattr(serial, sattr, guard, tickets); + } if let Some(expected) = guard { let current = self.getattr(serial)?; if crate::attributes::ctime(¤t) != expected { @@ -500,6 +515,9 @@ impl Kernel { data: &[u8], tickets: &mut Tickets, ) -> Result<(u32, ViewStat), Status> { + if crate::apple_double::companion(serial) { + return self.companion_write(serial, offset, data); + } let open = self.open_file(serial, true)?; if data.is_empty() { return Ok((0, self.getattr(serial)?)); diff --git a/core/crates/layerfs-nfs/src/kernel.rs b/core/crates/layerfs-nfs/src/kernel.rs index 5df1044bb6..77f0ae2aba 100644 --- a/core/crates/layerfs-nfs/src/kernel.rs +++ b/core/crates/layerfs-nfs/src/kernel.rs @@ -140,8 +140,12 @@ pub struct Kernel { /// ticket): kept, never released or undone on a guess, as the FUSE /// connection retains them. retained: Mutex>, - /// Refuse to create macOS AppleDouble (`._*`) names. + /// Keep macOS AppleDouble (`._*`) names out of the Workspace: a regular + /// file lives in the export alone (see [`crate::apple_double`]), any + /// other kind is refused. refuse_apple_double: bool, + /// The AppleDouble companions held in memory. + pub(crate) companions: Mutex, /// Lookup references kept before the least recently used are collected. lookups_held: usize, /// References collected, handles resolved again, and resolutions that @@ -152,6 +156,8 @@ pub struct Kernel { /// Silly renames served as unlinks, and creations refused by name. pub(crate) silly_renamed: AtomicU64, pub(crate) names_refused: AtomicU64, + /// AppleDouble companions created in memory. + pub(crate) names_detached: AtomicU64, } impl Kernel { pub fn new(services: Arc, mount: KernelMount, uid: u32, gid: u32) -> Self { @@ -171,12 +177,14 @@ impl Kernel { requests: AtomicU64::new(1), retained: Mutex::default(), refuse_apple_double: false, + companions: Mutex::default(), lookups_held: LOOKUPS_HELD, collected: AtomicU64::new(0), resolved: AtomicU64::new(0), unresolved: AtomicU64::new(0), silly_renamed: AtomicU64::new(0), names_refused: AtomicU64::new(0), + names_detached: AtomicU64::new(0), } } /// Whether creating `._*` names is refused (see [`crate::names`]). @@ -232,6 +240,10 @@ impl Kernel { if serial == self.root() { return Ok(()); } + if crate::apple_double::companion(serial) { + // The export's own companion holds no engine reference. + return self.companion_stat(serial).map(|_| ()); + } { let mut table = self.table(); if table.lookups.contains(&serial) { @@ -316,6 +328,9 @@ impl Kernel { /// is observed through its descriptor; any other inode no name reaches /// any more is stale. pub fn getattr(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return self.companion_stat(serial); + } let handle = { let table = self.table(); if table.pinned(serial) { @@ -356,6 +371,9 @@ impl Kernel { _ if name.len() > 255 => return Err(Status::NameTooLong), _ => {} } + if let Some(stat) = self.companion_named(parent, name) { + return Ok(stat); + } let silly = self.table().silly.get(&(parent, name.to_vec())).copied(); if let Some(serial) = silly { return self.getattr(serial); @@ -468,6 +486,9 @@ impl Kernel { } /// READLINK: the raw target bytes. pub fn readlink(&self, serial: u64) -> Result, Status> { + if crate::apple_double::companion(serial) { + return Err(Status::Inval); + } let (services, request) = self.request()?; let data = wait(KernelData::read( services, @@ -559,6 +580,9 @@ impl Kernel { /// the size last seen for the file, or comes back short. pub fn read(&self, serial: u64, offset: u64, count: u32) -> Result<(Vec, bool), Status> { let length = count.min(READ_WINDOW as u32); + if crate::apple_double::companion(serial) { + return self.companion_read(serial, offset, length); + } let open = self.open_file(serial, false)?; if offset >= open.size { return Ok((Vec::new(), true)); @@ -581,6 +605,9 @@ impl Kernel { } /// A fresh implicit directory descriptor for an enumeration from cookie 0. pub fn open_directory(&self, serial: u64) -> Result { + if crate::apple_double::companion(serial) { + return Err(Status::NotDir); + } let (_, _, directory) = self.observe(serial, None, KernelReadOperation::Opendir { serial })?; let handle = directory.ok_or(Status::ServerFault)?; @@ -632,7 +659,9 @@ impl Kernel { /// Retained mutation failures keep their custody and are only counted. pub fn release(&self) -> Released { let table = std::mem::take(&mut *self.table()); + let companions = self.companions_release(); let released = Released { + companions, files: table.files.len(), directories: table.directories.len(), lookups: table.lookups.len(), @@ -640,6 +669,7 @@ impl Kernel { retained: self.retained_count(), silly_renamed: self.silly_renamed.load(Ordering::Relaxed), names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), collected: self.collected.load(Ordering::Relaxed), resolved: self.resolved.load(Ordering::Relaxed), unresolved: self.unresolved.load(Ordering::Relaxed), @@ -665,6 +695,7 @@ impl Kernel { pub fn held(&self) -> Released { let table = self.table(); Released { + companions: self.companions_held(), files: table.files.len(), directories: table.directories.len(), lookups: table.lookups.len(), @@ -672,6 +703,7 @@ impl Kernel { retained: self.retained_count(), silly_renamed: self.silly_renamed.load(Ordering::Relaxed), names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), collected: self.collected.load(Ordering::Relaxed), resolved: self.resolved.load(Ordering::Relaxed), unresolved: self.unresolved.load(Ordering::Relaxed), @@ -766,6 +798,10 @@ pub struct Released { pub silly_renamed: u64, /// Creations refused for their AppleDouble name, over the export's life. pub names_refused: u64, + /// AppleDouble companions held in memory (see [`crate::apple_double`]). + pub companions: usize, + /// AppleDouble companions created, over the export's life. + pub names_detached: u64, /// References collected beyond the bound, over the export's life. pub collected: u64, /// Handles of collected serials resolved again by their name. diff --git a/core/crates/layerfs-nfs/src/lib.rs b/core/crates/layerfs-nfs/src/lib.rs index f9377c13ed..32b1b450f0 100644 --- a/core/crates/layerfs-nfs/src/lib.rs +++ b/core/crates/layerfs-nfs/src/lib.rs @@ -11,11 +11,12 @@ //! configured read-only answers every changing procedure `NFS3ERR_ROFS`. //! A changing procedure's publication tickets are released only after its //! reply was sent (see [`change`]); names a client invents for itself are -//! kept out of the Workspace (see [`names`]). +//! kept out of the Workspace (see [`names`] and [`apple_double`]). //! There is no third-party dependency: XDR, RPC record marking and both //! programs are implemented here over `std::net`. #![forbid(unsafe_code)] +pub mod apple_double; pub mod attributes; mod block; pub mod change; diff --git a/core/crates/layerfs-nfs/src/names.rs b/core/crates/layerfs-nfs/src/names.rs index 1bc440f639..9237471c15 100644 --- a/core/crates/layerfs-nfs/src/names.rs +++ b/core/crates/layerfs-nfs/src/names.rs @@ -10,10 +10,13 @@ //! export's table (LOOKUP, GETATTR, READ, WRITE, SETATTR and REMOVE of it //! work; READDIR does not list it). The final REMOVE gives the inode back. //! - **AppleDouble.** On NFSv3 the macOS client stores extended attributes -//! and resource forks in `._` companion files. Creating one is -//! refused with `NFS3ERR_ACCES` (when the export is configured to), so no -//! such file enters the Workspace; the client reports the extended -//! attribute as not settable. +//! and resource forks in `._` companion files. When the export is +//! configured to keep them out, such a regular file is created in the +//! export's memory alone (see [`crate::apple_double`]): the client's +//! attribute write (`copyfile`, `cp`, `std::fs::copy`, `ditto`, `tar`) +//! succeeds and reads back for the export's life, and nothing reaches the +//! Workspace. Any other kind under such a name (directory, symlink, link, +//! rename target) is refused with `NFS3ERR_ACCES`. //! //! The patterns are exact so that a user's own `.nfsrc` or `._config` is an //! ordinary name (the AppleDouble refusal is the one deliberate exception diff --git a/core/crates/layerfs-nfs/src/server.rs b/core/crates/layerfs-nfs/src/server.rs index 91ccbdf807..19b208866d 100644 --- a/core/crates/layerfs-nfs/src/server.rs +++ b/core/crates/layerfs-nfs/src/server.rs @@ -36,7 +36,9 @@ pub struct ExportConfig { pub gid: u32, /// Changing procedures answer `NFS3ERR_ROFS`. pub read_only: bool, - /// Refuse to create macOS AppleDouble `._*` names (see [`crate::names`]). + /// Keep macOS AppleDouble `._*` names out of the Workspace: a regular + /// file lives in the export's memory, any other kind is refused (see + /// [`crate::apple_double`]). pub refuse_apple_double: bool, /// Lookup references kept before the least recently used are collected /// (see [`crate::kernel`]); [`crate::kernel::LOOKUPS_HELD`] by default. diff --git a/core/crates/layerfs-nfs/tests/write.rs b/core/crates/layerfs-nfs/tests/write.rs index 4c48aae822..ba11401ff3 100644 --- a/core/crates/layerfs-nfs/tests/write.rs +++ b/core/crates/layerfs-nfs/tests/write.rs @@ -225,7 +225,10 @@ fn every_changing_procedure_and_its_statuses() { assert_eq!(c.getattr(&dir.handle), Err(STALE)); // Names the export refuses or the protocol forbids. - assert_eq!(c.touch(&root, b"._new.txt", 0o644).map(|_| ()), Err(ACCES)); + // An AppleDouble file is created detached (see the test below); other + // kinds under such a name are refused. + c.touch(&root, b"._new.txt", 0o644).unwrap(); + c.remove(&root, b"._new.txt").unwrap(); assert_eq!(c.mkdir(&root, b"._d", 0o755).map(|_| ()), Err(ACCES)); let refused = c.rename(&root, b"s", &root, b"._s").map(|_| ()); assert_eq!(refused, Err(ACCES)); @@ -255,7 +258,16 @@ fn every_changing_procedure_and_its_statuses() { // Every serial handed out holds one reference: removed ones gave theirs back. let held = server.held(); - assert_eq!((held.silly, held.retained, held.names_refused), (0, 0, 3)); + assert_eq!( + ( + held.silly, + held.retained, + held.names_refused, + held.companions, + held.names_detached + ), + (0, 0, 2, 0, 1) + ); drop(c); let released = server.stop(); let (_, outstanding) = rig.stop(released); @@ -338,6 +350,91 @@ fn silly_renames_stay_out_of_the_workspace() { fs::remove_dir_all(base).unwrap(); } +#[test] +fn apple_double_files_stay_out_of_the_workspace() { + let base = temp("appledouble"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + // What macOS copyfile does for a source carrying extended attributes: + // the data file, then its `._` companion with the attributes. + let data = c.touch(&root, b"copied.rs", 0o644).unwrap(); + c.write(&data.handle, 0, b"fn main() {}").unwrap(); + let made = c.touch(&root, b"._copied.rs", 0o644).unwrap(); + c.write(&made.handle, 0, b"AppleDouble header").unwrap(); + // It reads back under its name for the export's life ... + let (handle, attr) = c.lookup(&root, b"._copied.rs").unwrap(); + assert_eq!(handle, made.handle); + assert_eq!(attr.size, 18); + assert_eq!(c.read_all(&handle, 4096), b"AppleDouble header"); + // ... opening it again truncates it, GUARDED finds it taken ... + let truncate = Set { + size: Some(0), + ..Set::default() + }; + let again = c + .create(&root, b"._copied.rs", How::Unchecked(truncate)) + .unwrap(); + assert_eq!( + (again.handle.clone(), again.attr.size), + (made.handle.clone(), 0) + ); + let guarded = c.create(&root, b"._copied.rs", How::Guarded(Set::default())); + assert_eq!(guarded.map(|_| ()), Err(EXIST)); + // ... READDIR does not list it, it moves only to another AppleDouble name, + // and a real file cannot take such a name. + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|e| !e.name.starts_with(b"._"))); + c.rename(&root, b"._copied.rs", &root, b"._moved.rs") + .unwrap(); + assert_eq!(c.lookup(&root, b"._copied.rs").map(|_| ()), Err(NOENT)); + let back = c + .rename(&root, b"._moved.rs", &root, b"plain.rs") + .map(|_| ()); + assert_eq!(back, Err(ACCES)); + let onto = c.rename(&root, b"copied.rs", &root, b"._x").map(|_| ()); + assert_eq!(onto, Err(ACCES)); + assert_eq!(server.held().companions, 1); + // REMOVE gives it back. + c.remove(&root, b"._moved.rs").unwrap(); + assert_eq!(c.getattr(&handle), Err(STALE)); + assert_eq!(server.held().companions, 0); + // One still held at Commit is not captured; the data file is. + let kept = c.touch(&root, b"._kept", 0o644).unwrap(); + c.write(&kept.handle, 0, b"kept").unwrap(); + let done = rig.commit(); + assert!(done.history.is_some(), "{:?}", done.history); + let view = rig.view([77; 32]); + let viewer = rig.serve_view(TcpListener::bind("127.0.0.1:0").unwrap(), &view, UID, GID); + let mut v = Client::connect(viewer.local_addr(), UID, GID); + let view_root = v.mnt("/"); + let (entries, _) = v.list(&view_root, 64 * 1024, false); + let names: Vec> = entries.into_iter().map(|e| e.name).collect(); + assert!(names.contains(&b"copied.rs".to_vec())); + assert!(!names.iter().any(|n| n.starts_with(b"._"))); + drop(v); + let view_released = viewer.stop(); + assert_eq!(view_released.retained, 0); + rig.close_view(view); + drop(c); + let released = server.stop(); + assert_eq!( + ( + released.silly, + released.companions, + released.names_detached, + released.names_refused + ), + (0, 1, 2, 1) + ); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + #[test] fn tickets_are_released_only_after_the_reply() { let base = temp("tickets"); From 4c1648ed26aa9507046e15983ce5d1c1db1a02a3 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:34:37 +0800 Subject: [PATCH 48/64] fix(native): bound NFS helper reaping and detach before daemon stop Replace the unconditional wait after helper KILL with a five-second try_wait observation; retain unreaped children and report their PID. Read helper pipes nonblockingly with capped diagnostics, and keep the export and engine custody when a mount helper times out after spawn. Detach local Sandbox mounts while the daemon still serves NFS. Continue stopping after detach failure; keep Delete's checked detach as its fence. Bound macOS force-unmount helpers too, including stderr capture. Keep mount-table process imports macOS-only so Linux has no unused imports. Give discovery ownership acquisition one 70-second deadline, covering a single 60+5-second helper and return headroom. Preserve lock scope, ordering and Busy refusal; do not release ownership around an in-flight mutation. A real SDK lock test refuses at 60s before and succeeds after release at 62s with the new budget. Multiple queued holders can still yield Busy. Add process-helper regression tests and an opt-in real macOS mount proof. Mark the actual 60-second deadline test slow. No mount-option changes. Original kernel U-state hang remains unreproduced on this Mac. Validation: daemon/NFS/sandbox all-target tests 102 passed, 0 failed, 4 ignored; native smoke 48/48; macOS all-target Clippy including binding and Rust formatting pass. Prior real Stop fixture failed before and passed after. Existing product guard reports five binding inline-test violations. Linux cross-Clippy could not build SQLite/zstd without x86_64-linux-gnu-gcc; the cfg correction still needs native Linux confirmation. Fresh-context read-only review of the teardown fix found no P0/P1. Production LOC: 108058 -> 108240 (delta +182) Scope: first-party core Rust/runtime SQL plus api Rust/JS; core 101993 -> 102175, api 6065 unchanged. Daemon 7058 -> 7157 (+99), sandbox 4476 -> 4559 (+83). Other packages unchanged; helper relocation included. Method: tools/production_loc.py at a700af8f4, same counter over git archive parent a700af8f4 and exact staged tree 0edfb78380be2532fef94255742dbc8ea563fd99. Excludes tests/docs/tooling and test-only source. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- api/binding/src/discovery.rs | 5 +- .../layerfs-daemon/src/control/export.rs | 162 +++++++----------- .../src/control/export_helper.rs | 155 +++++++++++++++++ core/crates/layerfs-daemon/src/control/mod.rs | 1 + .../layerfs-nfs/tests/helper_deadline.rs | 142 +++++++++++++++ .../src/backend/local/darwin.rs | 101 ++++++++++- .../src/backend/local/sandbox.rs | 9 +- .../layerfs-sandbox/tests/local_stop_mount.rs | 48 ++++++ .../docs/architecture/nfs-native-lifecycle.md | 84 +++++++++ core/tools/ci_tests.py | 1 + 10 files changed, 594 insertions(+), 114 deletions(-) create mode 100644 core/crates/layerfs-daemon/src/control/export_helper.rs create mode 100644 core/crates/layerfs-nfs/tests/helper_deadline.rs create mode 100644 core/crates/layerfs-sandbox/tests/local_stop_mount.rs create mode 100644 core/docs/architecture/nfs-native-lifecycle.md diff --git a/api/binding/src/discovery.rs b/api/binding/src/discovery.rs index 2a18cb53f3..0538ba749f 100644 --- a/api/binding/src/discovery.rs +++ b/api/binding/src/discovery.rs @@ -142,7 +142,10 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. -const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); +// One NFS helper may use 60s plus 5s of post-KILL observation. Leave +// return/cleanup headroom without releasing ownership around an in-flight +// mutation. This is one shared deadline, not a new wait for each lock. +const IN_PROCESS_WAIT: Duration = Duration::from_secs(70); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); static HELD: Mutex> = diff --git a/core/crates/layerfs-daemon/src/control/export.rs b/core/crates/layerfs-daemon/src/control/export.rs index 2a8fb195a0..592b203e50 100644 --- a/core/crates/layerfs-daemon/src/control/export.rs +++ b/core/crates/layerfs-daemon/src/control/export.rs @@ -21,6 +21,7 @@ //! the export and its engine mount in the entry, as FUSE custody is kept. //! - ForceUnmount runs the forced helper once, then the same teardown. use super::{ + export_helper::{Helpers, Ran}, kernel::{bounded, Kernel}, registry::{bound_mut, idle}, Failure, Service, Success, @@ -33,22 +34,18 @@ use layerfs_bridge::control::{ }; use layerfs_nfs::{ExportConfig, Released, Server}; use layerfs_overlay::{CleanupState, KernelMount}; +#[cfg(target_os = "macos")] +use std::process::{Command, Stdio}; use std::{ fmt, - io::Read, net::{IpAddr, Ipv4Addr, TcpListener}, path::PathBuf, - process::{Command, Stdio}, sync::Arc, - time::{Duration, Instant}, }; /// Largest write and read the export negotiates (FSINFO wtmax/rtmax). const WINDOW: u32 = 128 * 1024; -/// How long one helper run may take before it is killed and reported. -pub const HELPER_WAIT: Duration = Duration::from_secs(60); -/// Bytes of a helper's output kept for a refusal. -const OUTPUT: usize = 2048; +pub use super::export_helper::{Helper, HELPER_WAIT}; /// Stale inodes one install refreshes through the mount point at most. pub const REFRESHED: usize = 4096; @@ -71,25 +68,6 @@ fn refresh(root: &std::path::Path, paths: &[PathBuf]) { } } } -/// One helper command; `{port}` and `{dir}` in its words are replaced. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Helper(pub Vec); -impl Helper { - /// Splits a configured command line on whitespace. - pub fn words(line: &str) -> Option { - let words: Vec = line.split_whitespace().map(str::to_owned).collect(); - (!words.is_empty()).then_some(Self(words)) - } - fn command(&self, port: u16, directory: &str) -> Option { - let mut words = self.0.iter().map(|word| { - word.replace("{port}", &port.to_string()) - .replace("{dir}", directory) - }); - let mut command = Command::new(words.next()?); - command.args(words); - Some(command) - } -} /// How the host's NFS client mounts, flushes and unmounts an export. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Mounter { @@ -178,10 +156,14 @@ impl ExportServingConfig { #[derive(Debug)] pub struct ExportServing { config: ExportServingConfig, + helpers: Helpers, } impl ExportServing { pub fn start(config: ExportServingConfig) -> Self { - Self { config } + Self { + config, + helpers: Helpers::default(), + } } pub fn config(&self) -> &ExportServingConfig { &self.config @@ -202,9 +184,9 @@ impl ExportServing { (points.into_iter()) .map(|point| { let text = point.display().to_string(); - match run(&self.config.mounter.force, 0, &text) { + match self.helpers.run(&self.config.mounter.force, 0, &text) { Ran::Done => format!("forced off {text}"), - Ran::Busy(detail) | Ran::Failed(detail) => { + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { format!("left {text}: {detail}") } } @@ -340,64 +322,6 @@ pub(super) fn block(kernel: &Kernel) -> Option { _ => return None, }) } -/// What one helper run answered. -enum Ran { - Done, - /// The client reported the mount busy; nothing changed. - Busy(String), - Failed(String), -} -fn run(helper: &Helper, port: u16, directory: &str) -> Ran { - let Some(mut command) = helper.command(port, directory) else { - return Ran::Failed("empty helper command".into()); - }; - let child = command - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn(); - let mut child = match child { - Ok(child) => child, - Err(error) => return Ran::Failed(format!("{:?}: {error}", helper.0.first())), - }; - let deadline = Instant::now() + HELPER_WAIT; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Ok(status), - Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(5)), - Ok(None) => { - drop(child.kill()); - drop(child.wait()); - break Err(format!("helper did not finish in {HELPER_WAIT:?}; killed")); - } - Err(error) => break Err(format!("helper wait: {error}")), - } - }; - let mut output = String::new(); - for pipe in [ - child.stderr.take().map(|p| Box::new(p) as Box), - child.stdout.take().map(|p| Box::new(p) as Box), - ] - .into_iter() - .flatten() - { - let mut bytes = Vec::new(); - drop(pipe.take(OUTPUT as u64).read_to_end(&mut bytes)); - output.push_str(&String::from_utf8_lossy(&bytes)); - } - match status { - Ok(status) if status.success() => Ran::Done, - Ok(status) => { - let detail = format!("{status}: {}", output.trim()); - if output.to_ascii_lowercase().contains("busy") { - Ran::Busy(detail) - } else { - Ran::Failed(detail) - } - } - Err(detail) => Ran::Failed(format!("{detail}: {}", output.trim())), - } -} fn receipt(token: WorkspaceToken, directory: String, mount: KernelMount, port: u16) -> ReadyMount { ReadyMount { token, @@ -577,8 +501,25 @@ impl Service { } }; let port = server.local_addr().port(); - match run(&config.mounter.mount, port, &text) { + match serving.helpers.run(&config.mounter.mount, port, &text) { Ran::Done => {} + Ran::Interrupted(detail) => { + // The mount helper may have reached the kernel before its + // deadline. Keep the export alive and do not revoke/delete + // through a possibly mounted directory. + return Err(Stop::Retained(Box::new(ExportKept { + ready: None, + stage: TeardownStage::Detach, + detail, + work: Some(work(&server)), + evidence: Box::new(Exported { + server: Arc::new(server), + mount, + ready: receipt(token, text, mount, port), + }), + forced: None, + }))); + } Ran::Busy(detail) | Ran::Failed(detail) => { let released = server.stop(); drop(std::fs::remove_dir(&directory)); @@ -637,12 +578,17 @@ impl Service { }; match &serving.config.mounter.flush { None => nix::unistd::sync(), - Some(helper) => match run(helper, server.local_addr().port(), &directory) { - Ran::Done => {} - Ran::Busy(detail) | Ran::Failed(detail) => { - return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) + Some(helper) => { + match serving + .helpers + .run(helper, server.local_addr().port(), &directory) + { + Ran::Done => {} + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) + } } - }, + } } server.close_files(); Ok(()) @@ -710,10 +656,15 @@ impl Service { .as_ref() .map(|serving| serving.config.mounter.unmount.clone()); let port = exported.server.local_addr().port(); - let ran = match &helper { - Some(helper) => run(helper, port, &exported.ready.directory), - None => Ran::Failed("export serving not assembled".into()), - }; + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; match ran { Ran::Done => {} Ran::Busy(detail) => { @@ -724,7 +675,7 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } - Ran::Failed(detail) => { + Ran::Failed(detail) | Ran::Interrupted(detail) => { return Err(self.keep_export( token, TeardownStage::Detach, @@ -885,13 +836,18 @@ impl Service { .as_ref() .map(|serving| serving.config.mounter.force.clone()); let port = exported.server.local_addr().port(); - let ran = match &helper { - Some(helper) => run(helper, port, &exported.ready.directory), - None => Ran::Failed("export serving not assembled".into()), - }; + let ran = + match &helper { + Some(helper) => self.export.as_ref().unwrap().helpers.run( + helper, + port, + &exported.ready.directory, + ), + None => Ran::Failed("export serving not assembled".into()), + }; match ran { Ran::Done => facts.detach = DetachDisposition::Detached, - Ran::Busy(detail) | Ran::Failed(detail) => { + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { facts.detach = DetachDisposition::Failed; return Err(self.keep_export( token, diff --git a/core/crates/layerfs-daemon/src/control/export_helper.rs b/core/crates/layerfs-daemon/src/control/export_helper.rs new file mode 100644 index 0000000000..a4796ab725 --- /dev/null +++ b/core/crates/layerfs-daemon/src/control/export_helper.rs @@ -0,0 +1,155 @@ +//! One bounded NFS client helper attempt, including post-KILL observation. +use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use std::{ + io::{self, Read}, + process::{Child, Command, Stdio}, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// How long one helper run may take before it is killed and reported. +pub const HELPER_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const PAUSE: Duration = Duration::from_millis(5); +const OUTPUT: usize = 2048; + +/// One helper command; `{port}` and `{dir}` in its words are replaced. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Helper(pub Vec); +impl Helper { + /// Splits a configured command line on whitespace. + pub fn words(line: &str) -> Option { + let words: Vec = line.split_whitespace().map(str::to_owned).collect(); + (!words.is_empty()).then_some(Self(words)) + } + fn command(&self, port: u16, directory: &str) -> Option { + let mut words = self.0.iter().map(|word| { + word.replace("{port}", &port.to_string()) + .replace("{dir}", directory) + }); + let mut command = Command::new(words.next()?); + command.args(words); + Some(command) + } +} + +pub(super) enum Ran { + Done, + Busy(String), + Failed(String), + /// Timeout or observation failure after spawn: effects are not known. + Interrupted(String), +} + +/// Retained children are never waited on synchronously or signalled twice. +/// A later helper attempt may only reap their observed exits, not replay them. +#[derive(Debug, Default)] +pub(super) struct Helpers(Mutex>); +impl Helpers { + pub fn run(&self, helper: &Helper, port: u16, directory: &str) -> Ran { + { + let mut held = self.0.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let Some(mut command) = helper.command(port, directory) else { + return Ran::Failed("empty helper command".into()); + }; + let mut child = match command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(error) => return Ran::Failed(format!("{:?}: {error}", helper.0.first())), + }; + let mut pipes: Vec> = Vec::new(); + let mut pipe_error = None; + // Even an exited helper can leave a descendant holding its pipe open. + // Never wait for EOF, and drain while it runs so verbose helpers progress. + for pipe in [ + child.stderr.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + child.stdout.take().map(|pipe| { + let result = fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)); + (Box::new(pipe) as Box, result) + }), + ] + .into_iter() + .flatten() + { + match pipe.1 { + Ok(_) => pipes.push(pipe.0), + Err(error) => pipe_error = Some(format!("helper output: {error}")), + } + } + let mut output = Vec::new(); + let deadline = Instant::now() + HELPER_WAIT; + let status = loop { + drain(&mut pipes, &mut output); + if let Some(error) = pipe_error.take() { + break Err(error); + } + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) => break Err(format!("helper did not finish in {HELPER_WAIT:?}")), + Err(error) => break Err(format!("helper wait: {error}")), + } + }; + let status = match status { + Ok(status) => Ok(status), + Err(detail) => { + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break false, + } + }; + let pid = child.id(); + let disposition = if reaped { + "exit reaped".to_owned() + } else { + self.0 + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; helper may remain in kernel wait; child custody retained".into() + }; + Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) + } + }; + drain(&mut pipes, &mut output); + let output = String::from_utf8_lossy(&output); + match status { + Ok(status) if status.success() => Ran::Done, + Ok(status) => { + let detail = format!("{status}: {}", output.trim()); + if output.to_ascii_lowercase().contains("busy") { + Ran::Busy(detail) + } else { + Ran::Failed(detail) + } + } + Err(detail) => Ran::Interrupted(format!("{detail}: {}", output.trim())), + } + } +} + +fn drain(pipes: &mut [Box], output: &mut Vec) { + for pipe in pipes { + let mut bytes = [0; OUTPUT]; + // One bounded read per pipe and observation: even continuous output + // cannot defer the helper's deadline. Excess bytes are discarded. + match pipe.read(&mut bytes) { + Ok(count) => output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]), + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(_) => {} + } + } +} diff --git a/core/crates/layerfs-daemon/src/control/mod.rs b/core/crates/layerfs-daemon/src/control/mod.rs index 87c3bd6073..75abdf4e74 100644 --- a/core/crates/layerfs-daemon/src/control/mod.rs +++ b/core/crates/layerfs-daemon/src/control/mod.rs @@ -6,6 +6,7 @@ mod cleanup; mod detach; mod dirty; mod export; +mod export_helper; mod failure; mod force; mod inspect; diff --git a/core/crates/layerfs-nfs/tests/helper_deadline.rs b/core/crates/layerfs-nfs/tests/helper_deadline.rs new file mode 100644 index 0000000000..781e82710a --- /dev/null +++ b/core/crates/layerfs-nfs/tests/helper_deadline.rs @@ -0,0 +1,142 @@ +//! Real control Service with process helpers, no kernel NFS mount required. +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use layerfs_bridge::control::{MountPhase, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + path::PathBuf, + process::Command, + sync::{mpsc, Arc}, + time::Duration, +}; + +fn base(label: &str) -> PathBuf { + let root = std::env::var_os("H07_TEST_ROOT") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + root.join(format!("helper-{}-{label}", std::process::id())) +} +fn run_case(label: &str, helper: Helper, before: impl FnOnce()) -> (String, MountPhase) { + let base = base(label); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let service = rig.service(Arc::new(ExportServing::start(ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper::words("true").unwrap(), + unmount: helper, + force: Helper::words("true").unwrap(), + flush: None, + }, + )))); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([33; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + service.execute_control(&Request::Attach(token)).unwrap(); + before(); + let reply = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + let phase = match service.execute_control(&Request::Status(token)) { + Ok(done) => match done.reply { + Reply::Status(status) => status.kernel.unwrap().phase, + other => panic!("{other:?}"), + }, + Err(_) => MountPhase::Unattached, + }; + if phase == MountPhase::Retained { + let repeated = format!( + "{:?}", + service.execute_control(&Request::Unmount(UnmountControl { + token, + discard: true + })) + ); + assert_eq!( + reply, repeated, + "retained request must not launch another helper" + ); + } + // No actual kernel mount exists in this fixture. Service owns retained + // export custody until drop. + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + (reply, phase) +} +#[test] +fn exited_helper_does_not_wait_for_a_descendants_pipe_eof() { + let root = base("pipe-holder"); + fs::create_dir_all(&root).unwrap(); + let pidfile = root.join("descendant.pid"); + let script = format!("sleep 20 & echo $! > {}; exit 0", pidfile.display()); + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = run_case( + "pipes", + Helper(vec!["sh".into(), "-c".into(), script]), + || {}, + ); + let _ = send.send(result); + }); + let result = receive.recv_timeout(Duration::from_secs(3)); + // Always release the descendant, including on failure, before asserting. + if let Ok(pid) = fs::read_to_string(&pidfile) { + let _ = Command::new("kill").args(["-KILL", pid.trim()]).status(); + } + worker.join().unwrap(); + fs::remove_dir_all(root).unwrap(); + let (reply, _) = result.expect("exited helper blocked on descendant's open pipe"); + assert!(reply.contains("Unmounted"), "{reply}"); +} +#[test] +fn timed_out_helper_returns_retained_and_does_not_replay() { + // STOP makes the helper ignore TERM; KILL must reap it without wait(). + // A normal process cannot ignore SIGKILL: this tests the actual deadline, + // not the kernel's uninterruptible U state. + let (reply, phase) = run_case( + "timeout", + Helper(vec!["sh".into(), "-c".into(), "kill -STOP $$".into()]), + || {}, + ); + assert!( + reply.contains("Retained") && reply.contains("60s"), + "{reply}" + ); + assert_eq!(phase, MountPhase::Retained); + assert_eq!(HELPER_WAIT, Duration::from_secs(60)); +} + +#[test] +fn verbose_helper_preserves_busy_refusal_without_pipe_deadlock() { + let (reply, phase) = run_case( + "verbose", + Helper(vec!["sh".into(), "-c".into(), "echo busy >&2; i=0; while [ $i -lt 5000 ]; do echo verbose-helper-output; i=$((i+1)); done; exit 32".into()]), + || {}, + ); + assert!(reply.to_ascii_lowercase().contains("busy"), "{reply}"); + assert_eq!(phase, MountPhase::Ready); +} diff --git a/core/crates/layerfs-sandbox/src/backend/local/darwin.rs b/core/crates/layerfs-sandbox/src/backend/local/darwin.rs index 151357084b..de33b0bf0d 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/darwin.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/darwin.rs @@ -5,17 +5,24 @@ //! (`umount -f`). Each tool is one run with a fixed path and the C locale, //! read once; nothing is retried. use super::process::Stat; -use nix::libc; +use nix::{ + fcntl::{fcntl, FcntlArg, OFlag}, + libc, +}; use std::{ - io, + io::{self, Read}, path::{Path, PathBuf}, - process::{Command, Stdio}, - sync::OnceLock, + process::{Child, Command, Stdio}, + sync::{Mutex, OnceLock}, + time::{Duration, Instant}, }; const SYSCTL: &str = "/usr/sbin/sysctl"; const MOUNT: &str = "/sbin/mount"; const UMOUNT: &str = "/sbin/umount"; +const UNMOUNT_WAIT: Duration = Duration::from_secs(60); +const REAP_WAIT: Duration = Duration::from_secs(5); +const OUTPUT: usize = 2048; /// One tool's standard output, or its failure with what it said. fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { @@ -176,19 +183,95 @@ fn mount_point(line: &str) -> Option { /// Forces the mount at `point` off once; a point no longer mounted is done. pub(super) fn force_unmount(point: &Path) -> io::Result<()> { - let out = Command::new(UMOUNT) + static RETAINED: Mutex> = Mutex::new(Vec::new()); + { + let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + } + let mut child = Command::new(UMOUNT) .arg("-f") .arg(point) .env_clear() .env("LC_ALL", "C") .stdin(Stdio::null()) - .output()?; - if out.status.success() || !mount_points()?.iter().any(|p| p == point) { + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stderr = child.stderr.take().and_then(|pipe| { + // If nonblocking setup fails, drop the pipe rather than risk an + // unbounded read while stopping a Sandbox. + fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)) + .ok() + .map(|_| pipe) + }); + let mut output = Vec::new(); + let deadline = Instant::now() + UNMOUNT_WAIT; + let status = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) => { + break Err(io::Error::new( + io::ErrorKind::TimedOut, + "umount exceeded 60s", + )) + } + Err(error) => break Err(error), + } + }; + let status = match status { + Ok(status) => status, + Err(error) => { + let pid = child.id(); + let kill = child.kill(); + let deadline = Instant::now() + REAP_WAIT; + let reaped = loop { + unmount_output(&mut stderr, &mut output); + match child.try_wait() { + Ok(Some(_)) => break true, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(5)); + } + Ok(None) | Err(_) => break false, + } + }; + let disposition = if reaped { + "exit reaped" + } else { + RETAINED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + "exit not observed after 5s; child custody retained (possible kernel wait)" + }; + return Err(io::Error::new( + error.kind(), + format!( + "umount -f {}: {error}; pid={pid}; KILL={kill:?}; {disposition}: {}", + point.display(), + String::from_utf8_lossy(&output).trim() + ), + )); + } + }; + unmount_output(&mut stderr, &mut output); + if status.success() || !mount_points()?.iter().any(|p| p == point) { return Ok(()); } Err(io::Error::other(format!( - "umount -f {}: {}", + "umount -f {}: {status}: {}", point.display(), - String::from_utf8_lossy(&out.stderr).trim() + String::from_utf8_lossy(&output).trim() ))) } + +fn unmount_output(stderr: &mut Option, output: &mut Vec) { + let Some(pipe) = stderr else { return }; + let mut bytes = [0; OUTPUT]; + if let Ok(count) = pipe.read(&mut bytes) { + output.extend_from_slice(&bytes[..count.min(OUTPUT - output.len())]); + } +} diff --git a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs index 2f85dc9a13..b11bb16059 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs @@ -140,7 +140,8 @@ impl Sandbox { } Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) } - /// Stops the daemon once: TERM, up to `seconds` for it to end, then + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then /// KILL (a daemon this process started has ended once reaped); then /// KILL to every process group of its commands and its daemon, as a /// container's end kills what ran in it. A daemon already gone is the @@ -150,6 +151,12 @@ impl Sandbox { return Err(self.invalid("original local Stop admission")); } self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } if let Some(process) = self.process { self.end( process, diff --git a/core/crates/layerfs-sandbox/tests/local_stop_mount.rs b/core/crates/layerfs-sandbox/tests/local_stop_mount.rs new file mode 100644 index 0000000000..765fb0eac1 --- /dev/null +++ b/core/crates/layerfs-sandbox/tests/local_stop_mount.rs @@ -0,0 +1,48 @@ +//! Opt-in host proof against one caller-owned local Sandbox fixture. +//! Supply LAYERFS_TEST_STOP_DIRECTORY, _PID and _PORT. The fixture must be +//! disposable: this test stops its daemon and force-detaches its mounts. +#![cfg(target_os = "macos")] +use layerfs_sandbox::{ + backend::local::{LocalProcess, Sandbox}, + ContainerId, +}; +use std::{path::PathBuf, process::Command}; + +#[test] +#[ignore = "requires an explicitly supplied disposable mounted Sandbox"] +fn stop_detaches_mounts_before_returning_without_delete() { + let directory = PathBuf::from(std::env::var("LAYERFS_TEST_STOP_DIRECTORY").unwrap()); + let pid = std::env::var("LAYERFS_TEST_STOP_PID") + .unwrap() + .parse() + .unwrap(); + let port = std::env::var("LAYERFS_TEST_STOP_PORT") + .unwrap() + .parse() + .unwrap(); + assert!(directory.is_absolute()); + let mounts = || { + let out = Command::new("/sbin/mount").output().unwrap(); + assert!(out.status.success()); + String::from_utf8(out.stdout) + .unwrap() + .lines() + .filter(|line| line.contains(&format!(" on {}/", directory.display()))) + .map(str::to_owned) + .collect::>() + }; + assert!(!mounts().is_empty(), "fixture must start with a real mount"); + let process = LocalProcess::observe(pid).unwrap(); + let mut sandbox = Sandbox::reopen( + directory.clone(), + ContainerId::parse(&"1".repeat(64)).unwrap(), + process, + port, + Vec::new(), + ) + .unwrap(); + sandbox.stop(1).unwrap(); + assert!(!process.alive(), "fixture daemon survived Stop"); + assert!(directory.exists(), "Stop must not delete its directory"); + assert!(mounts().is_empty(), "Stop left mounts until Delete"); +} diff --git a/core/docs/architecture/nfs-native-lifecycle.md b/core/docs/architecture/nfs-native-lifecycle.md new file mode 100644 index 0000000000..78b7b20361 --- /dev/null +++ b/core/docs/architecture/nfs-native-lifecycle.md @@ -0,0 +1,84 @@ +# Native NFS helper and local Sandbox teardown + +Status: implemented on the macOS native adaptation branch. This describes the +NFS route in `layerfs-daemon`, not the upstream FUSE force-drain protocol in +[forced teardown](80-forced-teardown.md). No persistence profile, wire format, +mount option or Store interface changes here. + +## Helper ownership and deadlines + +[`control/export_helper.rs`](../../crates/layerfs-daemon/src/control/export_helper.rs) +owns each NFS mount, unmount, force and optional flush helper attempt. The +public `Helper` and `HELPER_WAIT` exports remain available from `control`. +One attempt gets 60 seconds of exit observation. At expiry or an observation +error it sends one KILL and polls `try_wait` for at most five more seconds. +It never calls an unconditional blocking `wait` after KILL: macOS can leave an +unmount helper in kernel wait where a signal does not establish process exit. + +An unobserved exit is an error with PID, KILL result and retained-child detail. +The process-wide export serving owner retains the `Child`; a later helper call +can reap an exit already observed with `try_wait`, but never signals or reruns +that attempt. No claim is made that the kernel released a mount or that KILL +rolled back a helper's effects. At daemon exit, process ownership passes to the +operating system as for any remaining child. + +Output pipes are nonblocking, drained in bounded chunks during observation and +capped at 2048 retained bytes. An exited helper's descendant can keep a pipe +open, so EOF is not a completion requirement. Output beyond the cap is discarded; +continuous output cannot postpone the exit deadline. A normal nonzero helper +answer containing `busy` keeps the existing reversible Busy disposition. An +interrupted attempt is never classified Busy from partial diagnostic output. + +The control layer keeps a timed-out unmount's export and engine mount at +`Retained/Detach`; subsequent requests report the same custody without replay. +A timed-out **mount** may already have reached the kernel, so Attach also retains +the serving export, engine mount and directory rather than revoking or deleting +through an unknown client mount. No Ready acknowledgement is issued for it. + +## Discovery ownership wait + +The Node binding keeps Sandbox discovery ownership while a control mutation is +in flight. Its existing single deadline covers both the in-process turn and +the cross-process `ownership.lock` acquisition. That wait is 70 seconds: the +60-second helper plus five seconds of post-KILL observation and five seconds +of return/cleanup headroom. Lock scope, ordering, one-attempt semantics and Busy +refusal are unchanged. Releasing ownership early would allow another call to +observe or mutate discovery records before the original result is recorded. + +This budget covers one helper, not an arbitrary queue or a Stop over several +mounts; a longer holder can still cause Busy. A successful lock acquisition is +readiness to begin the new operation, never permission to replay a failed one. + +## Stop before daemon termination + +[`backend/local/sandbox.rs`](../../crates/layerfs-sandbox/src/backend/local/sandbox.rs) +attempts detach below that Sandbox's own directory before TERM/KILL, while its +NFS server can still answer client writeback. A detach failure is logged and +does not prevent the existing daemon and process-group stop. Stop remains crash +teardown, with no Commit or graceful daemon-wide drain. It does not delete the +Sandbox directory. Delete retains its checked detach before directory removal; +failed detach never authorizes removal through a mount. + +Linux continues using `MNT_DETACH`, deepest mount first; macOS uses one +`umount -f` per mount, deepest first. The macOS force helper likewise gets a +60-second observation and a five-second bounded post-KILL reap. Unreaped children +are retained, and failure includes the exact PID and observation rather than +claiming a successful detach. The mount table is checked after ordinary helper +completion. No global NFS operation or cleanup of another Sandbox is performed. + +## Evidence and limits + +- `layerfs-nfs/tests/helper_deadline.rs` drives the real control Service with + process helpers: descendant-held pipe, verbose Busy answer, stopped helper + crossing the real 60-second deadline and repeated Retained requests. +- `layerfs-sandbox/tests/local_stop_mount.rs` is an opt-in macOS test using an + explicitly supplied disposable Sandbox. It proves Stop removes the mount + before Delete and leaves the directory. It never selects an existing user + Sandbox automatically. The normal suite lists this test as ignored. +- These tests cannot simulate an unkillable kernel U-state process: userspace + cannot ignore SIGKILL. Bounded observation of an unreaped U-state helper is + source-checked; the original long kernel hang has not been reproduced. +- A retained helper is reaped opportunistically on a later helper invocation, + not by a background reaper. Helper cleanup and kernel resource recovery remain + distinct facts. Stop may take the sum of deadlines over several mounts; + there is no promise that a multi-mount stop completes in one helper budget. diff --git a/core/tools/ci_tests.py b/core/tools/ci_tests.py index e981a9254e..272347451a 100644 --- a/core/tools/ci_tests.py +++ b/core/tools/ci_tests.py @@ -39,6 +39,7 @@ # (package, test target): run seconds on the macOS host, 2026-10-23. SLOW = { + ("layerfs-nfs", "helper_deadline"): 60.1, ("layerfs-api", "scale_lineage_cap"): 24.3, ("layerfs-api", "sdk_merge_scale"): 23.8, ("layerfs-daemon", "complete_installed_roots"): 9.0, From b3f8f80a5541ceaff7b43e3b4f19ab786c862410 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:31:17 +0800 Subject: [PATCH 49/64] fix(binding): release Sandbox ownership while unmount waits Register a Workspace-scoped unmount intent and exclusive flock under the Sandbox guard, then release Sandbox ownership during the daemon exchange. Publish the known result under the Workspace lease before shared-record cleanup. Repeated requests read progress or the saved outcome, never resend. Use existing Busy/Unknown refusals with Unmounting/Retained state and elapsed milliseconds. Gate token/directory and backend reads for that Workspace while other Workspaces and discovery reads proceed. Keep the 60+5-second helper budget and restore the discovery readiness wait to 60 seconds. Explicit Sandbox close may take over orphaned or retained unmounts but still refuses unrelated active Workspaces. Preserve Stop's detach-before-termination and Delete's checked detach. Keep known daemon success on cleanup failure and report cleanupError separately. Nonblocking shared lease probes cannot block one another; known-token lookup avoids scanning historical result records. Add a real native SDK regression with paused helpers, cross-process progress, Busy/Dirty reversibility, publication during a held discovery lock, caller crash, saved failure, and explicit close takeover. The old binding fails its 900ms discovery-read assertion; the new binding completes unrelated calls in milliseconds. Update lifecycle documentation and describe interface details. Validation: daemon/NFS/sandbox all-target tests 102 passed/0 failed/4 ignored; binding 9/9; native smoke 48/48; real concurrency fixture PASS; macOS Clippy (all targets, including binding), fmt, JS syntax and diff checks pass. Linux cross-Clippy blocked by missing x86_64-linux-gnu-gcc in SQLite/zstd; existing five binding product-boundary violations remain unchanged in kind. Fresh-context review identified two recovery issues, both fixed and verified. No public method or daemon wire change; new detail shapes and close takeover semantics require maintainer alignment before upstream publication. Production LOC: 108240 -> 108557 (delta +317) Scope: first-party core Rust/runtime SQL plus api Rust/JS; core 102175 unchanged, api 6065 -> 6382, binding 3412 -> 3729; other packages unchanged. Method: tools/production_loc.py over git archive first parent 4c1648ed2 and exact staged tree 5394d161ed4559d9d98f29405564cdcbc7bd7f60, same counter, excluding tests/docs/tooling and test-only source. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- api/binding/src/discovery.rs | 18 +- api/binding/src/dispatch.rs | 3 + api/binding/src/lib.rs | 1 + api/binding/src/registry.rs | 11 +- api/binding/src/sandbox.rs | 12 +- api/binding/src/unmount.rs | 286 ++++++++++++++++++ api/binding/src/workspace.rs | 69 ++++- api/binding/tests/unmount-concurrency.mjs | 138 +++++++++ .../docs/architecture/nfs-native-lifecycle.md | 78 ++++- 9 files changed, 578 insertions(+), 38 deletions(-) create mode 100644 api/binding/src/unmount.rs create mode 100644 api/binding/tests/unmount-concurrency.mjs diff --git a/api/binding/src/discovery.rs b/api/binding/src/discovery.rs index 0538ba749f..ff704c96cd 100644 --- a/api/binding/src/discovery.rs +++ b/api/binding/src/discovery.rs @@ -49,7 +49,7 @@ pub(crate) fn home(args: &SdkValue) -> Result { directory(&home)?; Ok(home) } -fn private(path: &Path, dir: bool) -> Result<()> { +pub(crate) fn private(path: &Path, dir: bool) -> Result<()> { let meta = fs::symlink_metadata(path).map_err(failed)?; if meta.file_type().is_symlink() || meta.is_dir() != dir @@ -142,10 +142,9 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. -// One NFS helper may use 60s plus 5s of post-KILL observation. Leave -// return/cleanup headroom without releasing ownership around an in-flight -// mutation. This is one shared deadline, not a new wait for each lock. -const IN_PROCESS_WAIT: Duration = Duration::from_secs(70); +// Unmount exchanges run outside this guard; one shared readiness deadline +// covers the in-process turn and the cross-process discovery lock. +const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); static HELD: Mutex> = @@ -694,6 +693,12 @@ fn recorded_identity(value: &SdkValue) -> CommandIdentity { } /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { + refresh_except(state, &[]) +} +/// Explicit Sandbox close may dispose retained unmounts without asking their +/// daemon for a Ready receipt that no longer exists. Other records still +/// undergo the ordinary refresh and MountsActive admission. +pub(crate) fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); let mut processes = std::collections::BTreeMap::new(); for item in fs::read_dir(&state.directory).map_err(failed)? { @@ -704,6 +709,9 @@ pub(crate) fn refresh(state: &SandboxState) -> Result<()> { continue; }; let path = registry::text(&value, "path")?; + if retained.iter().any(|kept| kept == path) { + continue; + } let token = layerfs_bridge::control::WorkspaceToken { workspace: layerfs_history::WorkspaceId::from_slice(&bytes::<32>( &value, diff --git a/api/binding/src/dispatch.rs b/api/binding/src/dispatch.rs index 99c80b12b6..365004dc7f 100644 --- a/api/binding/src/dispatch.rs +++ b/api/binding/src/dispatch.rs @@ -75,6 +75,9 @@ fn listed(answer: SdkValue) -> Outcome { /// the daemon, so it is not sent on. fn daemon_backend(op: &str, args: SdkValue) -> Outcome { let state = registry::sandbox_live(&args)?; + if let Some(path) = args.get_text("path") { + crate::unmount::check(&state, path)?; + } let args = match args { SdkValue::Map(fields) => SdkValue::Map( fields diff --git a/api/binding/src/lib.rs b/api/binding/src/lib.rs index 6139526dd0..f7f89ae788 100644 --- a/api/binding/src/lib.rs +++ b/api/binding/src/lib.rs @@ -14,6 +14,7 @@ mod registry; mod runtime; mod sandbox; mod trace; +mod unmount; mod workspace; use job::Job; diff --git a/api/binding/src/registry.rs b/api/binding/src/registry.rs index 5f1e902287..c2483a2b92 100644 --- a/api/binding/src/registry.rs +++ b/api/binding/src/registry.rs @@ -78,7 +78,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// tables as they stand: no daemon call (#352 F-029). A call that names a /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes -/// [`sandbox_refreshed`]. +/// `discovery::refresh`. pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -131,14 +131,6 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { SdkValue::Map(fields) } -/// The Sandbox an operation names, its tables read again from the shared -/// records and the daemon (one status call per mount). -pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { - let state = sandbox(args)?; - crate::discovery::refresh(&state)?; - Ok(state) -} - impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { @@ -156,6 +148,7 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { + crate::unmount::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/api/binding/src/sandbox.rs b/api/binding/src/sandbox.rs index 08ca063342..b9ce2464d0 100644 --- a/api/binding/src/sandbox.rs +++ b/api/binding/src/sandbox.rs @@ -399,8 +399,16 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { } let state = crate::registry::sandbox_live(args)?; let _guard = crate::discovery::guard(&state.directory)?; - crate::discovery::refresh(&state)?; - let mounted = lock(&state.mounts)?.keys().cloned().collect::>(); + let recoverable = crate::unmount::check_all(&state)?; + crate::discovery::refresh_except(&state, &recoverable)?; + // Explicit close can take over an orphaned/retained unmount, but must + // still refuse every unrelated live Workspace. Stop detaches before + // killing the daemon; Delete independently checks physical detach. + let mounted = lock(&state.mounts)? + .keys() + .filter(|path| !recoverable.contains(path)) + .cloned() + .collect::>(); if !mounted.is_empty() { return Err(SdkRefusal { code: "MountsActive".to_owned(), diff --git a/api/binding/src/unmount.rs b/api/binding/src/unmount.rs new file mode 100644 index 0000000000..99cc4c4fb0 --- /dev/null +++ b/api/binding/src/unmount.rs @@ -0,0 +1,286 @@ +//! Workspace-scoped unmount ownership; Sandbox discovery is held only at +//! admission and shared-record cleanup, never during the daemon's helper wait. +//! Mount paths are unique within one daemon lifetime; Sandbox removal clears +//! the terminal records before that directory can be reused. +use crate::{ + discovery, + job::{failed, refused, Outcome}, + registry::SandboxState, +}; +use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; +use std::{ + fs::{File, OpenOptions}, + os::unix::{fs::OpenOptionsExt, io::AsRawFd}, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +extern "C" { + fn flock(fd: i32, operation: i32) -> i32; +} +const LOCK_SH: i32 = 1; +const LOCK_EX: i32 = 2; +const LOCK_NB: i32 = 4; +const LOCK_UN: i32 = 8; +fn millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(i64::MAX as u128) as i64 +} +fn record(state: &SandboxState, token: WorkspaceToken) -> PathBuf { + discovery::workspace_file(state, token).with_extension("unmount") +} +struct Lease(File); +impl Drop for Lease { + fn drop(&mut self) { + unsafe { flock(self.0.as_raw_fd(), LOCK_UN) }; + } +} +fn lease(path: &std::path::Path, operation: i32) -> Result, SdkRefusal> { + #[cfg(target_os = "macos")] + let no_follow = 0x100; + #[cfg(target_os = "linux")] + let no_follow = 0x20000; + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(no_follow) + .open(path) + .map_err(failed)?; + discovery::private(path, false)?; + if unsafe { flock(file.as_raw_fd(), operation | LOCK_NB) } == 0 { + return Ok(Some(Lease(file))); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::WouldBlock { + Ok(None) + } else { + Err(failed(error)) + } +} +/// Explicit unlock matters when a concurrent fork inherited the descriptor. +pub(crate) struct Attempt { + _lease: Lease, + path: PathBuf, + started: i64, +} +fn result(value: &SdkValue) -> Outcome { + if value.get_text("state") == Some("Unmounted") { + return Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])); + } + let detail = SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "elapsedMs", + SdkValue::Int( + millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0), + ), + ), + ("original", value.get("detail").cloned().unwrap_or_default()), + ]); + Err(SdkRefusal { + code: value.get_text("code").unwrap_or("Unknown").into(), + message: value + .get_text("message") + .unwrap_or("unmount outcome retained") + .into(), + detail, + }) +} +fn progress(value: &SdkValue, active: bool) -> SdkRefusal { + let elapsed = millis() + .saturating_sub(value.get_int("startedMs").unwrap_or(millis())) + .max(0); + SdkRefusal { + code: if active { "Busy" } else { "Unknown" }.into(), + message: if active { + format!("Workspace is Unmounting; waited {elapsed}ms") + } else { + format!("unmount caller ended without publishing its result after {elapsed}ms; inspect daemon custody, do not resend") + }, + detail: SdkValue::map([ + ( + "state", + SdkValue::text(if active { "Unmounting" } else { "Retained" }), + ), + ("elapsedMs", SdkValue::Int(elapsed)), + ]), + } +} +fn observed(path: &std::path::Path) -> Result, SdkRefusal> { + match discovery::load(path) { + Ok(value) => Ok(Some(value)), + Err(_) if !path.exists() => Ok(None), + Err(error) => Err(error), + } +} +/// Consulted before token lookup, including when another process removed +/// the Workspace record. Does not acquire the Sandbox guard or dial daemon. +fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { + let Some(value) = observed(path)? else { + return Ok(None); + }; + if value.get_text("state") != Some("Unmounting") { + return Ok(Some(value)); + } + let available = lease(&path.with_extension("unmount-lock"), LOCK_SH)?; + // Publication may have completed between our read and lease acquisition. + let latest = observed(path)?; + match latest { + Some(latest) if latest.get_text("state") != Some("Unmounting") => Ok(Some(latest)), + None => Ok(None), + Some(_) if available.is_none() => Err(progress(&value, true)), + Some(_) => { + let error = progress(&value, false); + Ok(Some(SdkValue::map([ + ("state", SdkValue::text("Retained")), + ( + "startedMs", + value.get("startedMs").cloned().unwrap_or_default(), + ), + ("path", value.get("path").cloned().unwrap_or_default()), + ("code", SdkValue::text(error.code)), + ("message", SdkValue::text(error.message)), + ("detail", error.detail), + ]))) + } + } +} +pub(crate) fn check(state: &SandboxState, path: &str) -> Result<(), SdkRefusal> { + if let Some(value) = find(state, path)? { + if value.get_text("state") == Some("Unmounted") { + return Err(refused("NotMounted", "Workspace already unmounted")); + } + return result(&value).map(drop); + } + Ok(()) +} +pub(crate) fn find(state: &SandboxState, path: &str) -> Result, SdkRefusal> { + // Steady-state token/directory lookups already have a token: avoid a + // directory scan proportional to the lifetime's completed unmounts. + let known = crate::registry::lock(&state.mounts)? + .get(path) + .map(|mount| mount.workspace.bound.token); + if let Some(token) = known { + return inspect(&record(state, token)); + } + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + if let Some(value) = observed(&entry.path())? { + if value.get_text("path") == Some(path) { + return inspect(&entry.path()); + } + } + } + } + Ok(None) +} +/// Closing a running Sandbox may not race a registered terminal operation. +pub(crate) fn check_all(state: &SandboxState) -> Result, SdkRefusal> { + let mut recoverable = Vec::new(); + for entry in std::fs::read_dir(&state.directory).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry.path().extension().is_some_and(|ext| ext == "unmount") { + // Even a published result may still be cleaning its Workspace + // record. Close must not delete/reuse the Sandbox directory in + // that short interval while the original lease is held. + let probe = lease(&entry.path().with_extension("unmount-lock"), LOCK_SH)?; + if probe.is_none() { + if let Some(value) = observed(&entry.path())? { + return Err(progress(&value, true)); + } + } + if let Some(value) = inspect(&entry.path())? { + if value.get_text("state") == Some("Retained") { + recoverable.push(crate::registry::text(&value, "path")?.to_owned()); + } + } + } + } + Ok(recoverable) +} +/// Called under the Sandbox guard, after all before-effect checks. +pub(crate) fn begin( + state: &SandboxState, + token: WorkspaceToken, + path: &str, +) -> Result { + let path_record = record(state, token); + let file = lease(&path_record.with_extension("unmount-lock"), LOCK_EX)? + .ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?; + let attempt = Attempt { + _lease: file, + path: path_record, + started: millis(), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text("Unmounting")), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ]), + )?; + Ok(attempt) +} +/// Publish while the Workspace lease prevents another terminal request. +/// No Sandbox guard is needed to replace this attempt's own result file. +pub(crate) fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> { + match outcome { + Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => { + std::fs::remove_file(&attempt.path).map_err(failed)?; + } + _ => { + let (state, code, message, detail) = match outcome { + Ok(_) => ("Unmounted", "", "", SdkValue::Null), + Err(error) => ( + "Retained", + error.code.as_str(), + error.message.as_str(), + error.detail.clone(), + ), + }; + discovery::save( + &attempt.path, + SdkValue::map([ + ("state", SdkValue::text(state)), + ("startedMs", SdkValue::Int(attempt.started)), + ("path", SdkValue::text(path)), + ("code", SdkValue::text(code)), + ("message", SdkValue::text(message)), + ("detail", detail), + ]), + )?; + } + } + Ok(()) +} +pub(crate) fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> { + std::fs::remove_file(&attempt.path).map_err(failed) +} +pub(crate) fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome { + // Do not replace a known daemon success with a cleanup Busy/Failed. + match &mut outcome { + Ok(SdkValue::Map(fields)) => { + fields.push(("cleanupError".into(), SdkValue::text(error.message))); + } + Err(original) => { + original.detail = SdkValue::map([ + ("original", original.detail.clone()), + ("cleanupError", SdkValue::text(error.message)), + ]); + } + _ => {} + } + outcome +} +pub(crate) fn repeat(value: &SdkValue) -> Outcome { + result(value) +} diff --git a/api/binding/src/workspace.rs b/api/binding/src/workspace.rs index bb08d914c5..28ea0c6116 100644 --- a/api/binding/src/workspace.rs +++ b/api/binding/src/workspace.rs @@ -8,9 +8,7 @@ //! by a mount and reclaimed by the next `sandbox.create` or //! `runtime.sandbox.reclaim`. use crate::job::{failed, Outcome}; -use crate::registry::{ - lock, naming, random, sandbox, sandbox_live, sandbox_refreshed, text, Mount, -}; +use crate::registry::{lock, naming, random, sandbox, sandbox_live, text, Mount}; use crate::sandbox::{operation, WORKSPACES}; use layerfs_api::internal::workspace; use layerfs_api::sandbox::values; @@ -257,10 +255,19 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { /// `workspace.unmount(ws, { discard? })`: `Busy` while a background /// process this binding started still runs there, then the daemon's answer. pub(crate) fn unmount(args: &SdkValue) -> Outcome { - let state = sandbox_refreshed(args)?; - let _guard = crate::discovery::guard(&state.directory)?; + let state = sandbox(args)?; let path = text(args, "path")?; + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } + let guard = crate::discovery::guard(&state.directory)?; + // Another process may have registered while this call waited for the + // short admission section. Check again before sending anything. + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } let token = state.token(path)?; + crate::discovery::refresh(&state)?; // The runtime owner knows its own processes; the daemon's kernel probe // still answers Busy for any other holder. let running = crate::exec::running_in(&state, path)?; @@ -278,11 +285,49 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { )]), }); } - let answer = workspace(&mut *crate::trace::control(&state.controls)?) - .unmount_checked(token, args.get_bool("discard")) - .map_err(operation)?; - let value = values::unmounted(&answer)?; - std::fs::remove_file(crate::discovery::workspace_file(&state, token)).map_err(failed)?; - lock(&state.mounts)?.remove(path); - Ok(value) + let attempt = crate::unmount::begin(&state, token, path)?; + drop(guard); + // Only this Workspace's lease remains held. Other Workspaces and + // processes can take Sandbox ownership while the daemon helper waits. + let mut sent = false; + let outcome = (|| { + let mut control = crate::trace::control(&state.controls)?; + sent = true; + match workspace(&mut control).unmount_checked(token, args.get_bool("discard")) { + Ok(answer) => values::unmounted(&answer), + Err(error) => Err(operation(error)), + } + })(); + let outcome = match outcome { + Err(error) if error.code == "Missing" => { + Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])) + } + other => other, + }; + // Preserve the daemon's known outcome before any discovery-lock wait. + // A file/cleanup failure is separate from an already successful detach. + let publication = if !sent { + crate::unmount::cancel(&attempt) + } else { + crate::unmount::finish(&attempt, path, &outcome) + }; + if let Err(error) = publication { + return crate::unmount::cleanup_failure(outcome, error); + } + if outcome.is_ok() { + let cleanup = (|| { + let _guard = crate::discovery::guard(&state.directory)?; + match std::fs::remove_file(crate::discovery::workspace_file(&state, token)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + lock(&state.mounts)?.remove(path); + Ok(()) + })(); + if let Err(error) = cleanup { + return crate::unmount::cleanup_failure(outcome, error); + } + } + outcome } diff --git a/api/binding/tests/unmount-concurrency.mjs b/api/binding/tests/unmount-concurrency.mjs new file mode 100644 index 0000000000..065aafd14f --- /dev/null +++ b/api/binding/tests/unmount-concurrency.mjs @@ -0,0 +1,138 @@ +// Real native SDK regression. Explicit binaries + isolated root are required. +// node api/binding/tests/unmount-concurrency.mjs /abs/root /abs/addon.node /abs/daemon +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, writeFile, readFile, access, readdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +const [base, addon, daemon, childMode, serialized] = process.argv.slice(2); +assert.ok(base?.startsWith('/') && addon?.startsWith('/') && daemon?.startsWith('/'), 'explicit absolute fixture paths'); +const repo = fileURLToPath(new URL('../../..', import.meta.url)); +Object.assign(process.env, { + LAYERFS_SANDBOX: 'native', LAYERFS_NATIVE: addon, LAYERFS_DAEMON: daemon, + LAYERFS_HOME: base+'/home', LAYERFS_LOCAL_DIR: base+'/local', LAYERFS_STORE_PATH: base+'/store/store.sqlite', + LAYERFS_NFS_UNMOUNT: `sh ${base}/helper.sh {dir}`, +}); +const { LayerFS } = await import(repo+'/api/agent/sdk/src/index.js'); +const { LayerFSBackend } = await import(repo+'/api/backend/src/index.js'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function deadline(promise, ms) { + let timer; + try { return await Promise.race([promise, new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error('deadline '+ms)),ms);})]); } + finally { clearTimeout(timer); } +} +async function waitFile(path) { + const end=Date.now()+5000; + while(Date.now()=0);return error.detail;} +} +function child(mode, ws) { + const p=spawn(process.execPath,[fileURLToPath(import.meta.url),base,addon,daemon,mode,JSON.stringify(ws)],{env:process.env,stdio:['ignore','pipe','pipe']}); + let output='';p.stdout.on('data',d=>output+=d);p.stderr.on('data',d=>output+=d); + const ended=new Promise(resolve=>p.once('exit',(code,signal)=>resolve({code,signal,output}))); + return {p,ended}; +} +if(childMode){ + const lf=await LayerFS.open('unmount');const ws=JSON.parse(serialized); + if(childMode==='probe')await fast('cross-process-same-workspace',()=>refused(()=>lf.workspace.unmount(ws,true))); + if(childMode==='owner')await lf.workspace.unmount(ws,true); + await lf.close(); +}else{ + assert.equal(process.platform,'darwin','this fixture uses host mount_nfs'); + await mkdir(base+'/local',{recursive:true});await mkdir(base+'/seed',{recursive:true});await mkdir(base+'/store',{recursive:true,mode:0o700}); + await writeFile(base+'/seed/file','base\n'); + await writeFile(base+'/helper.sh',`#!/bin/sh\nprintf '%s' "$1" > '${base}/started'\ni=0\nwhile [ ! -f '${base}/release' ] && [ "$i" -lt 200 ]; do sleep 0.1; i=$((i+1)); done\nif [ -f '${base}/fail' ]; then echo injected-helper-failure >&2; exit 1; fi\nexec /sbin/umount "$1"\n`); + const lf=await LayerFS.init('unmount',base+'/seed');let owner;let active; + try{ + await lf.branch.fork('a');await lf.branch.fork('b'); + const sb=await lf.sandbox.create();const a=await lf.workspace.mount(sb.sandbox,'a');const b=await lf.workspace.mount(sb.sandbox,'b'); + active=lf.workspace.unmount(a,true);active.catch(()=>{}); + await waitFile(base+'/started'); + await fast('sandbox-log-needs-discovery-lock',()=>LayerFSBackend.of(lf).daemon.sandbox.get_log(sb.sandbox,{lines:2})); + await fast('other-workspace-status',()=>LayerFSBackend.of(lf).daemon.workspace.status(b)); + await fast('other-workspace-exec',()=>lf.workspace.exec(b,'printf ready'),r=>assert.equal(r.stdout,'ready')); + await fast('same-workspace-status',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.status(a))); + await fast('same-workspace-exec',()=>refused(()=>lf.workspace.exec(a,'true'))); + await fast('same-workspace-commit',()=>refused(()=>lf.workspace.commit(a,'must not run'))); + await fast('same-workspace-list-changes',()=>refused(()=>LayerFSBackend.of(lf).daemon.workspace.list_changes(a))); + await fast('repeat-unmount',()=>refused(()=>lf.workspace.unmount(a,true))); + await fast('sandbox-close',()=>refused(()=>lf.sandbox.close(sb.sandbox))); + const probe=child('probe',a);assert.equal((await deadline(probe.ended,5000)).code,0); + const another=await fast('new-workspace-mount',()=>lf.workspace.mount(sb.sandbox,'b')); + const locker=spawn('python3',['-c','import fcntl,sys; f=open(sys.argv[1],"r+"); fcntl.flock(f,fcntl.LOCK_EX); print("LOCKED",flush=True); sys.stdin.read(1)',base+'/home/unmount/sandboxes/'+sb.sandbox+'/ownership.lock'],{stdio:['pipe','pipe','pipe']}); + try { + await deadline(new Promise((resolve,reject)=>{locker.stdout.once('data',resolve);locker.once('error',reject);}),2000); + await writeFile(base+'/release','go');await sleep(200); + // The terminal result is visible before cleanup can reacquire this lock. + await fast('published-success-before-cleanup-lock',()=>lf.workspace.unmount(a,true)); + } finally { locker.stdin.end('x');await deadline(new Promise(r=>locker.once('exit',r)),2000); } + await deadline(active,5000);active=null; + await fast('repeat-completed-unmount',()=>lf.workspace.unmount(a,true)); + // Busy and Dirty are reversible: release only this attempt's intent. + await lf.workspace.exec(b,'printf dirty > dirty'); + await assert.rejects(lf.workspace.unmount(b),e=>e.code==='Dirty'); + await lf.workspace.exec(b,'printf still-usable'); + const holder=spawn('/bin/sleep',['5'],{cwd:b.path,stdio:'ignore'}); + try { await assert.rejects(lf.workspace.unmount(b,true),e=>e.code==='Busy'); } + finally { holder.kill('SIGTERM');await deadline(new Promise(r=>holder.once('exit',r)),2000); } + await lf.workspace.unmount(b,true);await lf.workspace.unmount(another,true);await lf.sandbox.close(sb.sandbox); + // A killed binding loses its lease, not its intent. No subsequent caller + // may send a second unmount when the daemon may still own the first. + const {unlink}=await import('node:fs/promises');await unlink(base+'/release');await unlink(base+'/started'); + const sb2=await lf.sandbox.create();const lost=await lf.workspace.mount(sb2.sandbox,'a'); + owner=child('owner',lost);await waitFile(base+'/started');owner.p.kill('SIGKILL');await deadline(owner.ended,5000); + await fast('lost-owner-no-replay',()=>refused(()=>lf.workspace.unmount(lost,true),'Retained')); + await writeFile(base+'/release','go'); + await sleep(300); + // Explicit close takes over the orphan while its daemon is healthy; + // it never resends the unknown Workspace unmount. + await fast('lost-owner-close-takeover',()=>lf.sandbox.close(sb2.sandbox)); + // A failed helper leaves daemon custody and the original SDK error. + // Repeated calls read that terminal result rather than launching again. + const sb3=await lf.sandbox.create();const kept=await lf.workspace.mount(sb3.sandbox,'a'); + await writeFile(base+'/fail','fail'); + const error=await lf.workspace.unmount(kept,true).then(()=>null,e=>e);assert.ok(error); + await fast('repeat-retained-result',async()=>{ + const repeated=await lf.workspace.unmount(kept,true).then(()=>null,e=>e); + assert.equal(repeated.code,error.code);assert.equal(repeated.message,error.message);assert.equal(repeated.detail.state,'Retained'); + }); + await fast('retained-close-takeover',()=>lf.sandbox.close(sb3.sandbox)); + console.log('UNMOUNT_CONCURRENCY PASS'); + }finally{ + await writeFile(base+'/release','cleanup'); + if(owner?.p.exitCode===null&&owner?.p.signalCode===null)owner.p.kill('SIGKILL'); + if(active)await deadline(active.catch(()=>{}),5000); + // Always give the fixture helpers permission to finish. A failed test + // must not leave a stopped helper or a held client request behind. + await lf.close(); + // Failure cleanup is scoped by the explicit fixture root. Detach while + // its daemon is still alive; never remove directories through mounts. + const {spawnSync}=await import('node:child_process'); + const table=spawnSync('/sbin/mount',[],{encoding:'utf8',timeout:5000}); + for(const line of (table.stdout||'').split('\n')){ + const point=line.match(/ on (.*) \(/)?.[1]; + if(point?.startsWith(base+'/local/')){ + const off=spawnSync('/sbin/umount',['-f',point],{encoding:'utf8',timeout:15000,killSignal:'SIGKILL'}); + assert.equal(off.status,0,'fixture detach: '+off.stderr); + } + } + const processes=spawnSync('/bin/ps',['-axo','pid=,command='],{encoding:'utf8',timeout:5000}); + for(const line of (processes.stdout||'').split('\n')){ + if(line.includes(daemon+' --config '+base+'/local/')){ + const pid=Number(line.trim().split(/\s+/)[0]); + try{process.kill(pid,'SIGTERM');}catch(e){if(e.code!=='ESRCH')throw e;} + } + } + } +} diff --git a/core/docs/architecture/nfs-native-lifecycle.md b/core/docs/architecture/nfs-native-lifecycle.md index 78b7b20361..3549b816c9 100644 --- a/core/docs/architecture/nfs-native-lifecycle.md +++ b/core/docs/architecture/nfs-native-lifecycle.md @@ -7,6 +7,12 @@ mount option or Store interface changes here. ## Helper ownership and deadlines +The helper budget remains 60 seconds plus five seconds for post-KILL exit +observation. A 15-second cutoff is not qualified: a normal slow writeback may +exceed it, and interrupting a helper cannot prove its kernel effect was undone. +Removing Sandbox ownership from the wait resolves the serialization problem +without trading it for earlier unknown outcomes. + [`control/export_helper.rs`](../../crates/layerfs-daemon/src/control/export_helper.rs) owns each NFS mount, unmount, force and optional flush helper attempt. The public `Helper` and `HELPER_WAIT` exports remain available from `control`. @@ -37,17 +43,63 @@ through an unknown client mount. No Ready acknowledgement is issued for it. ## Discovery ownership wait -The Node binding keeps Sandbox discovery ownership while a control mutation is -in flight. Its existing single deadline covers both the in-process turn and -the cross-process `ownership.lock` acquisition. That wait is 70 seconds: the -60-second helper plus five seconds of post-KILL observation and five seconds -of return/cleanup headroom. Lock scope, ordering, one-attempt semantics and Busy -refusal are unchanged. Releasing ownership early would allow another call to -observe or mutate discovery records before the original result is recorded. +The Node binding registers a Workspace-scoped unmount intent under Sandbox +`ownership.lock`, then drops Sandbox ownership before waiting for the daemon. +The existing NAPI worker waits synchronously on one control connection: no +new background helper or client process is introduced. The daemon still owns +its helper, reap deadline and export custody. Another Workspace can use another +pooled control connection; Sandbox discovery reads also proceed during the wait. + +`api/binding/src/unmount.rs` owns a private `workspace-.unmount` result +record and an exclusive nonblocking `workspace-.unmount-lock` flock. The +lease spans registration, the lock-free exchange and publication. It is explicitly +unlocked on drop, including when another fork briefly inherited its descriptor. +Lock order is Sandbox guard → nonblocking Workspace lease at admission; on +completion it publishes the known result using only its Workspace lease, then +reacquires the Sandbox guard for shared-record cleanup. A cleanup failure adds +`cleanupError` to a known successful response rather than replacing success with +Busy. Close checks the lease even for a published result, so it cannot delete or +reuse the Sandbox directory while that cleanup is in progress. Observers use +nonblocking shared flock probes, so they neither wait on the Workspace lease nor +mistake another observer for the exclusive operation owner. +The shared discovery readiness deadline is back to 60 seconds; it no longer +needs to cover the 60+5-second helper wait. Other long discovery operations or +control-pool exhaustion can still cause waiting; this is not a global latency +promise for all calls. + +State transitions: + +- No intent → Unmounting (path and start time saved before sending a request). +- Unmounting → Unmounted after known daemon success (including an explicit + Missing answer); shared-record cleanup follows. Repeated unmount reads the + saved result without sending another request. +- Busy/Dirty → no intent, preserving their reversible semantics. +- A control connection failure before the unmount exchange removes the intent; + no operation was sent. Other failure → Retained with the original refusal + saved, never replayed. +- Unmounting with no lease holder → Unknown/Retained, including client crash + before send or after daemon completion. Observation cannot decide whether an + effect occurred. It neither removes the Workspace nor repeats the request. + +The public method signatures and daemon protocol are unchanged. During active +unmount, existing `Busy` refusals carry `detail.state = "Unmounting"` and +`elapsedMs`; owner loss uses existing `Unknown` with `state = "Retained"`. +Workspace token/directory consumers and backend reads check this state before +sending a call; Sandbox close refuses an active intent. A new mount creates a +different Workspace and remains admissible. This detail shape and completed +unmount replay behavior need maintainer alignment before upstream publication. -This budget covers one helper, not an arbitrary queue or a Stop over several -mounts; a longer holder can still cause Busy. A successful lock acquisition is -readiness to begin the new operation, never permission to replay a failed one. +Terminal result records remain until Sandbox deletion; they are bounded by +that Sandbox's lifetime rather than silently evicted. Known token lookups use a +direct filename, not a scan of completed records; unknown-path recovery and +Sandbox close scan the saved records. An explicit Sandbox close can take over an +orphaned or terminal Retained unmount while the daemon is still healthy, provided +no unrelated live Workspace remains. It performs the existing Stop/Delete, +never a replay of the uncertain Workspace operation. If the daemon dies, the +existing stopped-Sandbox reclaim path owns physical detach and record removal. +Stop still detaches before daemon termination, and Delete retains its checked +detach fence. A failed result-file write is reported as cleanupError on known +success; durable recovery of that lost result is not claimed. ## Stop before daemon termination @@ -75,6 +127,12 @@ completion. No global NFS operation or cleanup of another Sandbox is performed. explicitly supplied disposable Sandbox. It proves Stop removes the mount before Delete and leaves the directory. It never selects an existing user Sandbox automatically. The normal suite lists this test as ignored. +- `api/binding/tests/unmount-concurrency.mjs` requires explicitly supplied + disposable paths and binaries. It runs real NFS with a paused helper, checks + another Workspace and discovery read below one second, checks same-Workspace + refusals and cross-process progress, reversible Busy/Dirty, saved failures, + publication while discovery cleanup is blocked, and explicit close takeover + after client-owner crash or retained failure. - These tests cannot simulate an unkillable kernel U-state process: userspace cannot ignore SIGKILL. Bounded observation of an unreaped U-state helper is source-checked; the original long kernel hang has not been reproduced. From d3a2a8c36a00fa8ca5adc760d7f840c0099a0983 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:12:06 +0800 Subject: [PATCH 50/64] fix(binding): use libc::O_NOFOLLOW for the unmount lease The Workspace unmount lease opened its record with a hand-written O_NOFOLLOW: 0x100 on macOS and 0x20000 on Linux. 0x20000 is O_NOFOLLOW only on x86_64 Linux; on aarch64 Linux O_NOFOLLOW is 0x8000 and 0x20000 is O_LARGEFILE, so the lease would silently follow a symlink there. Take the constant from libc (already in the lock at =0.2.189 through other crates) so every target gets its own value. No behavior change on macOS or x86_64 Linux. The binding's existing discovery::no_follow() carries the same hand-written values and is left as it is (it predates this branch). --- api/binding/Cargo.toml | 1 + api/binding/src/unmount.rs | 6 +----- core/Cargo.lock | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/binding/Cargo.toml b/api/binding/Cargo.toml index 5d05210071..dcbaa6c484 100644 --- a/api/binding/Cargo.toml +++ b/api/binding/Cargo.toml @@ -21,6 +21,7 @@ layerfs-sandbox = { path = "../../core/crates/layerfs-sandbox" } layerfs-storage = { path = "../../core/crates/layerfs-storage" } layerfs-store-service = { path = "../../core/crates/layerfs-store-service" } layerfs-telemetry = { path = "../../core/crates/layerfs-telemetry" } +libc = "=0.2.189" napi = "=2.16.17" napi-derive = "=2.16.13" diff --git a/api/binding/src/unmount.rs b/api/binding/src/unmount.rs index 99cc4c4fb0..34b0c34b95 100644 --- a/api/binding/src/unmount.rs +++ b/api/binding/src/unmount.rs @@ -39,16 +39,12 @@ impl Drop for Lease { } } fn lease(path: &std::path::Path, operation: i32) -> Result, SdkRefusal> { - #[cfg(target_os = "macos")] - let no_follow = 0x100; - #[cfg(target_os = "linux")] - let no_follow = 0x20000; let file = OpenOptions::new() .read(true) .write(true) .create(true) .mode(0o600) - .custom_flags(no_follow) + .custom_flags(libc::O_NOFOLLOW) .open(path) .map_err(failed)?; discovery::private(path, false)?; diff --git a/core/Cargo.lock b/core/Cargo.lock index 77b3cc0d27..9a21edd726 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -648,6 +648,7 @@ dependencies = [ "layerfs-storage", "layerfs-store-service", "layerfs-telemetry", + "libc", "napi", "napi-build", "napi-derive", From 766e67e648e32c74cbc173f367de998c3162008c Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:16:17 +0000 Subject: [PATCH 51/64] fix(sandbox): detach local mounts before stopping the daemon Sandbox::stop() ended the daemon first and left the Sandbox's mounts in place until Delete detached them. Detach them first, while the daemon can still answer, then stop the daemon as before. A failed detach is reported and does not block the crash teardown; Delete keeps its own checked detach before removing anything through a mount. On Linux FUSE this is ordering only: Stop now returns with the mounts already gone (measured 21.6 ms with a live holder and a dirty writer; before the change the mounts stayed until Delete), and reclaim is unchanged (0.09-0.10 s). MNT_DETACH does not flush dirty pages, so it adds no data guarantee; Stop stays the crash teardown it was. Same hunk as the local-backend part of the macOS native branch's NFS unmount-hang fix, so merging this branch there stays clean. --- core/crates/layerfs-sandbox/src/backend/local/sandbox.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs index 2f85dc9a13..b11bb16059 100644 --- a/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs +++ b/core/crates/layerfs-sandbox/src/backend/local/sandbox.rs @@ -140,7 +140,8 @@ impl Sandbox { } Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) } - /// Stops the daemon once: TERM, up to `seconds` for it to end, then + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then /// KILL (a daemon this process started has ended once reaped); then /// KILL to every process group of its commands and its daemon, as a /// container's end kills what ran in it. A daemon already gone is the @@ -150,6 +151,12 @@ impl Sandbox { return Err(self.invalid("original local Stop admission")); } self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } if let Some(process) = self.process { self.end( process, From 8b11778396cc860cf48dfccfbca19e4e74da4dd0 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 01:08:49 +0000 Subject: [PATCH 52/64] style: rustfmt --- engine/crates/layerfs-node/src/runtime.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs index 471d9f49ea..4f0ea7b82e 100644 --- a/engine/crates/layerfs-node/src/runtime.rs +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -164,7 +164,3 @@ impl Runtime { Ok((code, stdout, stderr)) } } - -#[cfg(test)] -#[path = "../tests/unit/runtime.rs"] -mod runtime_tests; From eb21fbbe7ddd6f53bec3aed04b82803bbac8477c Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 02:35:41 +0000 Subject: [PATCH 53/64] test(node): cover local Sandbox records through the built addon (#402) Upstream now keeps the Node binding's tests outside the crate source: they load the ordinary built addon and call its exported methods, never including private binding source (engine/AGENTS.md, engine/crates/layerfs-node/tests). The #402 unit tests that were included by runtime.rs and sandbox.rs (tests/unit/{runtime,reclaim}.rs before D8) are rewritten as engine/crates/layerfs-node/tests/native_local.rs, which writes a Sandbox record as an earlier call would and calls the public `runtime.sandbox.reclaim` with a local deployment: - a local record naming any other directory (a data directory, another LAYERFS_LOCAL_DIR, another Sandbox or name, a relative or `..` path, `/`) is refused InvalidInput, "neither used nor deleted", and nothing is removed; - a local record is used only with an absolute, plain deploy.localDir; - a stopped local Sandbox where this deployment puts it is deleted and its number freed; a record from before boot ids is read without one; a damaged boot or port is refused and removes nothing; - a Docker record is not read as a local Sandbox. The Docker reclaim cases of the old reclaim.rs are covered by upstream's native_discovery.rs. Selection of a misspelled deploy.sandbox is refused only by sandbox.create after the project's Store is installed, so it is not reachable without a Store and is left to the SDK test (local_deployment.mjs). Production LOC: 103762 -> 103762 (delta 0) (tools/production_loc.py --root ., core+engine+api; test-only commit) --- .../crates/layerfs-node/tests/native_local.rs | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 engine/crates/layerfs-node/tests/native_local.rs diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs new file mode 100644 index 0000000000..031c0a46a0 --- /dev/null +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -0,0 +1,244 @@ +//! External tests of the built Node binding's local Sandbox records +//! (LAYERFS_SANDBOX=local). No private Rust source is included: every case +//! writes a Sandbox record as a previous call would have, then calls the +//! addon's public `runtime.sandbox.reclaim` and observes the reply and the +//! files it left. Neither Docker, FUSE, nor a Store is needed. +#![cfg(target_os = "linux")] +mod support; +use layerfs_bridge::control::{Answer, Reply, SdkValue}; +use std::{ + io::Write, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + time::Duration, +}; +use support::*; + +/// This boot's id as a record keeps it: 32 lowercase hex digits. +fn boot() -> String { + std::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .unwrap() + .trim() + .replace('-', "") +} + +/// Sandbox 1 of `proj` recorded as a `runtime` (local or native) Sandbox +/// in `directory` named `name`, its daemon long gone (no process has id +/// u32::MAX). `edit` changes the fields before the record is written. +fn record( + home: &Home, + runtime: &str, + directory: &str, + name: &str, + edit: impl FnOnce(&mut Vec<(String, SdkValue)>), +) { + let bytes = |n: u8| SdkValue::Bytes(vec![n; 32]); + let mut fields: Vec<(String, SdkValue)> = vec![ + ("id".into(), SdkValue::text("1")), + ("project".into(), SdkValue::text("proj")), + ("name".into(), SdkValue::text(name)), + ("container".into(), SdkValue::text(CID)), + ("runtime".into(), SdkValue::text(runtime)), + ("directory".into(), SdkValue::text(directory)), + ("pid".into(), SdkValue::Int(i64::from(u32::MAX))), + ("start".into(), SdkValue::text("1")), + ("boot".into(), SdkValue::text(boot())), + ("port".into(), SdkValue::Int(40404)), + ("image".into(), SdkValue::text("local")), + ("secret".into(), bytes(1)), + ("peer".into(), bytes(2)), + ("instance".into(), bytes(3)), + ]; + edit(&mut fields); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(home.0.join("proj/sandboxes/1/sandbox.record")) + .unwrap(); + file.write_all( + &Answer { + id: 1, + reply: Reply::Sdk(SdkValue::Map(fields)), + } + .encode() + .unwrap(), + ) + .unwrap(); +} + +/// Replaces (or with `None` removes) one field of a record. +fn set(key: &'static str, value: Option) -> impl FnOnce(&mut Vec<(String, SdkValue)>) { + move |fields| { + fields.retain(|(k, _)| k != key); + if let Some(value) = value { + fields.push((key.into(), value)); + } + } +} + +/// `LAYERFS_LOCAL_DIR` of these tests, created like a deployment's. +fn root(home: &Home) -> PathBuf { + let root = home.0.join("local"); + std::fs::create_dir_all(&root).unwrap(); + root +} + +/// Where a `root` deployment puts Sandbox 1 of `proj`, created with a file. +fn own(root: &Path) -> PathBuf { + let own = root.join("layerfs-proj-1"); + std::fs::create_dir_all(own.join("tmp")).unwrap(); + std::fs::write(own.join("daemon.log"), b"log").unwrap(); + own +} + +/// The public reclaim of a `sandbox` (local or native) deployment, with +/// `localDir` when given, followed by `then` (JavaScript, `r` is the reply). +fn reclaim(home: &Home, sandbox: &str, local_dir: Option<&str>, then: &str) { + let deploy = match local_dir { + Some(dir) => format!("{{sandbox:{sandbox:?}, localDir:{dir:?}}}"), + None => format!("{{sandbox:{sandbox:?}}}"), + }; + home.node( + &format!( + "const r = await native.backendCall('runtime.sandbox.reclaim', {{...args, deploy:{deploy}}});\n{then}" + ), + None, + Duration::from_secs(10), + ); +} + +const NEITHER: &str = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /neither used nor deleted/);"; +const RECLAIMED: &str = "assert.deepEqual(r, {value:{kind:'ok',reclaimed:['1']}});"; + +fn text(path: &Path) -> &str { + path.to_str().unwrap() +} + +#[test] +fn a_local_record_naming_another_directory_is_neither_used_nor_deleted() { + let home = Home::new(); + let root = root(&home); + // a data directory a damaged or edited record names by its own name + let data = home.0.join("data"); + std::fs::create_dir(&data).unwrap(); + std::fs::write(data.join("keep"), b"x").unwrap(); + let old = home.0.join("old/layerfs-proj-1"); + std::fs::create_dir_all(&old).unwrap(); + let r = text(&root); + for (directory, name) in [ + (text(&data).to_owned(), "data"), + (format!("{r}/data"), "data"), + // another LAYERFS_LOCAL_DIR, another Sandbox, another name + (text(&old).to_owned(), "layerfs-proj-1"), + (format!("{r}/layerfs-proj-2"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1"), "layerfs-proj-2"), + (format!("{r}/layerfs-proj-1/sub"), "sub"), + ("relative/layerfs-proj-1".to_owned(), "layerfs-proj-1"), + (format!("{r}/../etc/layerfs-proj-1"), "layerfs-proj-1"), + ("/".to_owned(), ""), + ] { + record(&home, "local", &directory, name, |_| {}); + reclaim(&home, "local", Some(r), NEITHER); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "the record of {directory} was removed" + ); + } + assert!(data.join("keep").exists(), "a delete reached {}", data.display()); + assert!(old.exists(), "a delete reached {}", old.display()); +} + +#[test] +fn a_local_record_is_used_only_with_its_deployment_s_absolute_local_dir() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + let refused = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r));"; + reclaim(&home, "local", None, refused); + reclaim(&home, "local", Some("relative"), refused); + reclaim( + &home, + "local", + Some(&format!("{}/../local", text(&root))), + refused, + ); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} + +#[test] +fn a_stopped_local_sandbox_where_its_deployment_puts_it_is_deleted_and_reclaimed() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "local", Some(text(&root)), RECLAIMED); + assert!(!own.exists(), "the stopped Sandbox's directory is deleted"); + assert!( + !home.0.join("proj/sandboxes/1").exists(), + "its number is free again" + ); + assert!(root.exists(), "LAYERFS_LOCAL_DIR itself stays"); +} + +#[test] +fn a_record_from_before_boot_ids_is_read_without_one() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + record( + &home, + "local", + text(&own), + "layerfs-proj-1", + set("boot", None), + ); + reclaim(&home, "local", Some(text(&root)), RECLAIMED); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} + +#[test] +fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + for (key, value, code) in [ + ("boot", SdkValue::text("not-hex"), "InvalidInput"), + ("port", SdkValue::Int(70000), "Failed"), + ] { + record( + &home, + "local", + text(&own), + "layerfs-proj-1", + set(key, Some(value)), + ); + reclaim( + &home, + "local", + Some(text(&root)), + &format!("assert.equal(r.refused?.code, {code:?}, JSON.stringify(r));"), + ); + assert!(own.join("daemon.log").exists(), "{key}"); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists(), "{key}"); + } +} + +#[test] +fn a_docker_record_stays_docker_without_a_local_deployment() { + // the default record of Home is a Docker one: with no Engine at its + // socket it is neither reclaimed nor read as a local Sandbox + let home = Home::new(); + let root = root(&home); + reclaim( + &home, + "local", + Some(text(&root)), + "assert.equal(r.refused?.code, 'Failed', JSON.stringify(r));", + ); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); +} From 8b17038f215aae68b391da7301fd5a892861e218 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 02:37:02 +0000 Subject: [PATCH 54/64] style: rustfmt Production LOC: 103762 -> 103762 (delta 0) (tools/production_loc.py --root ., core+engine+api; formatting of a test file only) --- engine/crates/layerfs-node/tests/native_local.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs index 031c0a46a0..b4ff932b1b 100644 --- a/engine/crates/layerfs-node/tests/native_local.rs +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -147,7 +147,11 @@ fn a_local_record_naming_another_directory_is_neither_used_nor_deleted() { "the record of {directory} was removed" ); } - assert!(data.join("keep").exists(), "a delete reached {}", data.display()); + assert!( + data.join("keep").exists(), + "a delete reached {}", + data.display() + ); assert!(old.exists(), "a delete reached {}", old.display()); } @@ -224,7 +228,10 @@ fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { &format!("assert.equal(r.refused?.code, {code:?}, JSON.stringify(r));"), ); assert!(own.join("daemon.log").exists(), "{key}"); - assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists(), "{key}"); + assert!( + home.0.join("proj/sandboxes/1/sandbox.record").exists(), + "{key}" + ); } } From 1dcd1fb83c1d882ac31a3944200734f6983882c2 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 02:38:22 +0000 Subject: [PATCH 55/64] test(node): native Sandbox records and the unmount lease regression after D8 (#427) - engine/crates/layerfs-node/tests/native_local.rs gains local_and_native_sandboxes_are_never_mixed: through the built addon's public runtime.sandbox.reclaim, a native record is refused (InvalidInput, "never mixed") by a local deployment and a local record by a native one, nothing is removed, and a native deployment reclaims its own stopped record. It replaces the native cases of the old included api/binding/tests/unit/runtime.rs (removed in the previous merge). - The real native SDK regression of the unmount lease moves from api/binding/tests/unmount-concurrency.mjs (api/binding/ is gone after D8) to api/agent/test/verify/native_unmount_real.mjs, next to upstream's other *_real.mjs verifications, with its imports updated to D8's layout (api/agent/src, api/backend/src). It stays macOS-only and explicit. - discovery.rs: the "never mixed" refusal lost its line continuation in the v1.1 port and carried a run of spaces inside the message; restored. Production LOC: 108595 -> 108596 (delta +1) (tools/production_loc.py --root ., core+engine+api; the restored message continuation splits one source line in two) --- .../test/verify/native_unmount_real.mjs} | 6 ++--- engine/crates/layerfs-node/src/discovery.rs | 3 ++- .../crates/layerfs-node/tests/native_local.rs | 24 ++++++++++++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) rename api/{binding/tests/unmount-concurrency.mjs => agent/test/verify/native_unmount_real.mjs} (97%) diff --git a/api/binding/tests/unmount-concurrency.mjs b/api/agent/test/verify/native_unmount_real.mjs similarity index 97% rename from api/binding/tests/unmount-concurrency.mjs rename to api/agent/test/verify/native_unmount_real.mjs index 065aafd14f..d3c6b71b6a 100644 --- a/api/binding/tests/unmount-concurrency.mjs +++ b/api/agent/test/verify/native_unmount_real.mjs @@ -1,18 +1,18 @@ // Real native SDK regression. Explicit binaries + isolated root are required. -// node api/binding/tests/unmount-concurrency.mjs /abs/root /abs/addon.node /abs/daemon +// node api/agent/test/verify/native_unmount_real.mjs /abs/root /abs/addon.node /abs/daemon import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { mkdir, writeFile, readFile, access, readdir } from 'node:fs/promises'; import { fileURLToPath } from 'node:url'; const [base, addon, daemon, childMode, serialized] = process.argv.slice(2); assert.ok(base?.startsWith('/') && addon?.startsWith('/') && daemon?.startsWith('/'), 'explicit absolute fixture paths'); -const repo = fileURLToPath(new URL('../../..', import.meta.url)); +const repo = fileURLToPath(new URL('../../../..', import.meta.url)); Object.assign(process.env, { LAYERFS_SANDBOX: 'native', LAYERFS_NATIVE: addon, LAYERFS_DAEMON: daemon, LAYERFS_HOME: base+'/home', LAYERFS_LOCAL_DIR: base+'/local', LAYERFS_STORE_PATH: base+'/store/store.sqlite', LAYERFS_NFS_UNMOUNT: `sh ${base}/helper.sh {dir}`, }); -const { LayerFS } = await import(repo+'/api/agent/sdk/src/index.js'); +const { LayerFS } = await import(repo+'/api/agent/src/index.js'); const { LayerFSBackend } = await import(repo+'/api/backend/src/index.js'); const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); async function deadline(promise, ms) { diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index fdd5ef8469..981acb3f27 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -552,7 +552,8 @@ fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result return Err(refused( "InvalidInput", format!( - "Sandbox {id} is a {} Sandbox, but this deployment is {wanted} (LAYERFS_SANDBOX): local and native Sandboxes are never mixed", + "Sandbox {id} is a {} Sandbox, but this deployment is {wanted} \ + (LAYERFS_SANDBOX): local and native Sandboxes are never mixed", value.get_text("runtime").unwrap_or_default() ), )); diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs index b4ff932b1b..25ab5f76b4 100644 --- a/engine/crates/layerfs-node/tests/native_local.rs +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -1,5 +1,5 @@ //! External tests of the built Node binding's local Sandbox records -//! (LAYERFS_SANDBOX=local). No private Rust source is included: every case +//! (LAYERFS_SANDBOX=local or native). No private Rust source is included: every case //! writes a Sandbox record as a previous call would have, then calls the //! addon's public `runtime.sandbox.reclaim` and observes the reply and the //! files it left. Neither Docker, FUSE, nor a Store is needed. @@ -249,3 +249,25 @@ fn a_docker_record_stays_docker_without_a_local_deployment() { ); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } + +#[test] +fn local_and_native_sandboxes_are_never_mixed() { + let home = Home::new(); + let root = root(&home); + let own = own(&root); + let mixed = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); + assert.match(r.refused.message, /never mixed/);"; + // a native record is not used by a local deployment + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "local", Some(text(&root)), mixed); + // nor a local record by a native deployment + record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), mixed); + assert!(own.join("daemon.log").exists()); + assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); + // a native deployment uses its own record: stopped, it is reclaimed + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); + assert!(!own.exists()); + assert!(!home.0.join("proj/sandboxes/1").exists()); +} From a6b53a936c392b97522e0707b2fc24dc00981031 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 02:47:58 +0000 Subject: [PATCH 56/64] fix(tools): place layerfs-nfs in the engine layer of the boundary map (#427) D8 (#465) made tools/check_product_boundary.py fail closed on any LayerFS crate missing from its core/engine/api layer map. #427's layerfs-nfs (the loopback NFSv3 export, beside layerfs-fuse under engine/crates) was not in it, so the full and the CI (--skip-sdk) runs reported "crate missing from layer map: layerfs-nfs" for layerfs-nfs and for layerfs-daemon, which depends on it. It is an engine crate like layerfs-fuse; its dependencies (content, overlay, workspace, fuse) are at or below that layer. Production LOC: 108596 -> 108596 (delta 0) (tools/production_loc.py --root ., core+engine+api; development tool only) --- tools/check_product_boundary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index df0473e7d9..5d0fd653df 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -141,6 +141,7 @@ "layerfs-bridge": ENGINE, "layerfs-daemon": ENGINE, "layerfs-fuse": ENGINE, + "layerfs-nfs": ENGINE, "layerfs-overlay": ENGINE, "layerfs-persistence": ENGINE, "layerfs-project": ENGINE, From bf50a5f7030a8915a2545583575c24108763ceda Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 03:05:01 +0000 Subject: [PATCH 57/64] docs(node): list native_local in the binding test guide (#402) Production LOC: 103762 -> 103762 (delta 0) (tools/production_loc.py --root ., core+engine+api; documentation only) --- engine/crates/layerfs-node/tests/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/engine/crates/layerfs-node/tests/README.md b/engine/crates/layerfs-node/tests/README.md index 75af6d4e58..db6fd370eb 100644 --- a/engine/crates/layerfs-node/tests/README.md +++ b/engine/crates/layerfs-node/tests/README.md @@ -12,6 +12,7 @@ cargo +1.85.1 build --manifest-path Cargo.toml --locked -p layerfs-node cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --tests --no-run timeout 30 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_discovery timeout 90 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_admission +timeout 60 cargo +1.85.1 test --manifest-path Cargo.toml --locked -p layerfs-node --test native_local ``` Use the worktree's registered Cargo resources for these commands. Node 20 or @@ -24,6 +25,12 @@ number replacement during inspection, and discovery-lock release while a forked child retains the descriptor. Its C fixture interposes `flock` only in the child Node process; the fixture's fork is bounded and released on test unwinding. +`native_local` covers local Sandbox records (`LAYERFS_SANDBOX=local`) through +`runtime.sandbox.reclaim`: a record naming any directory other than where its +deployment puts that Sandbox is refused and nothing is deleted, the configured +`localDir` must be absolute and plain, a stopped local Sandbox is deleted and its +number freed, and a damaged boot or port is refused. + `native_admission` covers initial connection admission and its `Busy` deadline. The deadline test pays the product's full 60-second readiness wait. The mock daemon has bounded socket waits and stops independently of a test assertion. From 94781905e66d45ff198d5ec55a8cce9cc117e841 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Tue, 27 Oct 2026 03:05:01 +0000 Subject: [PATCH 58/64] docs(node): native records in the binding test guide (#427) Production LOC: 108596 -> 108596 (delta 0) (tools/production_loc.py --root ., core+engine+api; documentation only) --- engine/crates/layerfs-node/tests/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/crates/layerfs-node/tests/README.md b/engine/crates/layerfs-node/tests/README.md index db6fd370eb..d575dce6d1 100644 --- a/engine/crates/layerfs-node/tests/README.md +++ b/engine/crates/layerfs-node/tests/README.md @@ -29,7 +29,8 @@ Node process; the fixture's fork is bounded and released on test unwinding. `runtime.sandbox.reclaim`: a record naming any directory other than where its deployment puts that Sandbox is refused and nothing is deleted, the configured `localDir` must be absolute and plain, a stopped local Sandbox is deleted and its -number freed, and a damaged boot or port is refused. +number freed, a damaged boot or port is refused, and local and native +(`LAYERFS_SANDBOX=native`) records are never used by the other's deployment. `native_admission` covers initial connection admission and its `Busy` deadline. The deadline test pays the product's full 60-second readiness wait. The mock From d55acc15f8dfeee73522f54ffbd66292ad5fe574 Mon Sep 17 00:00:00 2001 From: jiangkoumo Date: Wed, 28 Oct 2026 02:35:23 +0000 Subject: [PATCH 59/64] refactor(node): native is the only local deployment; local is no longer selectable (#427) #427 is the macOS native adaptation. The local-daemon runtime it builds on (first proposed in the closed #402) is now part of this PR, and its only selectable form is `native`: on macOS a daemon of this user serves its Workspaces over loopback NFS to the host's own client. Linux keeps the Docker route exactly as on main. - deploy.sandbox (LAYERFS_SANDBOX) accepts unset, `docker` or `native`; `local` is refused InvalidInput like any other value, in the SDK (api/agent/src/client.js) and in the binding (runtime::local). - local::start refuses `native` off macOS as Unsupported before any effect, so no deployment starts a local daemon on Linux. The root check and the Local::new (Linux FUSE) branch go: native always takes Local::native, LAYERFS_KERNEL=nfs and this user's identity. discovery::root() had no other caller and goes too. - Records: a new record's runtime is always `native`. A native record is used only by a native deployment; a record of the former `local` runtime (an earlier, unmerged build) is used by no deployment, refused InvalidInput ("never mixed", "neither used nor deleted"), and nothing it names is deleted. - Tests: engine/crates/layerfs-node/tests/native_local.rs writes native records (record handling is the same code on Linux) and gains a_record_of_the_former_local_runtime_is_never_used_nor_deleted; api/agent/test/local_deployment.mjs becomes native_deployment.mjs with `local` among the refused values. - api/docs/agent.md drops the "Local Sandboxes" section and keeps its layout, record and reclaim facts under "Native Sandboxes (macOS)". The local backend in layerfs-sandbox (engine/crates/layerfs-sandbox/src/ backend/local) is kept as it is: native uses its process, exec, group, mount and record code; Local::new and its Linux tests stay as tests of that shared code. Production LOC: 109158 -> 109142 (delta -16) (tools/production_loc.py --root .) --- api/agent/src/client.js | 19 ++-- api/agent/src/index.js | 9 +- ...l_deployment.mjs => native_deployment.mjs} | 50 +++++----- api/docs/agent.md | 95 ++++++++----------- engine/crates/layerfs-node/src/discovery.rs | 43 ++++----- engine/crates/layerfs-node/src/local.rs | 45 ++++----- engine/crates/layerfs-node/src/runtime.rs | 27 +++--- engine/crates/layerfs-node/src/sandbox.rs | 8 +- engine/crates/layerfs-node/tests/README.md | 10 +- .../crates/layerfs-node/tests/native_local.rs | 59 ++++++------ 10 files changed, 162 insertions(+), 203 deletions(-) rename api/agent/test/{local_deployment.mjs => native_deployment.mjs} (71%) diff --git a/api/agent/src/client.js b/api/agent/src/client.js index 64e2bb5c3c..5d64cf5035 100644 --- a/api/agent/src/client.js +++ b/api/agent/src/client.js @@ -61,18 +61,17 @@ export function sandboxArg(sandbox) { /** Deployment settings from the client options, else the environment * (LAYERFS_). A Workspace's Sandbox needs * every one but the container prefix and daemon host. With `sandbox` - * (LAYERFS_SANDBOX) `local`, a Sandbox is a daemon process on this machine - * instead of a container: it needs the daemon and `localDir` - * (LAYERFS_LOCAL_DIR) and no Docker setting. `native` is a local daemon - * whose Workspaces this host's own NFS client mounts, with commands run as - * this user (no root, no Docker image). Unset or `docker` is Docker; any - * other value is refused. */ + * (LAYERFS_SANDBOX) `native` (macOS), a Sandbox is a daemon process of this + * user whose Workspaces this host's own NFS client mounts, with commands run + * as this user (no root, no Docker image): it needs the daemon and + * `localDir` (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is + * Docker; any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || undefined; const sandbox = options.sandbox ?? env("SANDBOX"); - if (sandbox === "local" || sandbox === "native") return localDeployment(options, env, sandbox); + if (sandbox === "native") return nativeDeployment(options, env, sandbox); if (sandbox !== undefined && sandbox !== "docker") { - throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or local (or native), not ${sandbox}`, null); + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or native (macOS), not ${sandbox}`, null); } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", @@ -91,9 +90,9 @@ export function deployment(options = {}) { return deploy; } -/** A local Sandbox's settings: the daemon, the directory its Sandboxes +/** A native Sandbox's settings: the daemon, the directory its Sandboxes * live in, and the optional name prefix. */ -function localDeployment(options, env, sandbox) { +function nativeDeployment(options, env, sandbox) { const deploy = { sandbox, daemon: options.daemon ?? env("DAEMON"), diff --git a/api/agent/src/index.js b/api/agent/src/index.js index b632e30c90..20395fbc87 100644 --- a/api/agent/src/index.js +++ b/api/agent/src/index.js @@ -100,10 +100,11 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON; a * Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. LAYERFS_SANDBOX=local - * starts each Sandbox as a daemon process on this machine instead of a - * container, in a directory under LAYERFS_LOCAL_DIR (no Docker setting is - * read; it isolates nothing, so use it only inside a VM or CI box). + * container it starts `-`. On macOS, + * LAYERFS_SANDBOX=native starts each Sandbox as a daemon process of this + * user instead of a container, in a directory under LAYERFS_LOCAL_DIR, + * its Workspaces mounted over loopback NFS (no Docker setting is read; it + * isolates nothing beyond the caller's own sandbox). * LAYERFS_DAEMON_HOST is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on diff --git a/api/agent/test/local_deployment.mjs b/api/agent/test/native_deployment.mjs similarity index 71% rename from api/agent/test/local_deployment.mjs rename to api/agent/test/native_deployment.mjs index 72b2bcd114..fb68ed0d52 100644 --- a/api/agent/test/local_deployment.mjs +++ b/api/agent/test/native_deployment.mjs @@ -1,9 +1,10 @@ // Which runtime a Sandbox uses. No real addon, Store or Docker (the // stand-in binding answers the client's calls): -// node api/agent/sdk/test/local_deployment.mjs +// node api/agent/test/native_deployment.mjs // LAYERFS_SANDBOX unset or `docker` gives exactly the Docker deployment it -// always gave; `local` needs only the daemon and LAYERFS_LOCAL_DIR (no Docker -// setting); any other value is refused rather than taken as either. +// always gave; `native` (macOS) needs only the daemon and LAYERFS_LOCAL_DIR +// (no Docker setting); any other value, the former `local` included, is +// refused rather than taken as either. import assert from "node:assert/strict"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; @@ -41,30 +42,25 @@ assert.deepEqual(deployment(), docker); delete process.env.LAYERFS_IMAGE; assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_IMAGE/.test(e.message)); -// local: daemon + localDir, no Docker setting read or required +// native: daemon + localDir, no Docker setting read or required clear(); -process.env.LAYERFS_SANDBOX = "local"; +process.env.LAYERFS_SANDBOX = "native"; process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; -process.env.LAYERFS_LOCAL_DIR = "/store/local"; -assert.deepEqual(deployment(), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); -// an image (sandbox.create's option) does not change a local deployment -assert.deepEqual(deployment({ image: docker.image }), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); -// the container prefix names local Sandbox directories too +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; +assert.deepEqual(deployment(), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// an image (sandbox.create's option) does not change a native deployment +assert.deepEqual(deployment({ image: docker.image }), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); +// the container prefix names native Sandbox directories too process.env.LAYERFS_CONTAINER_PREFIX = "ci-"; assert.equal(deployment().containerPrefix, "ci-"); delete process.env.LAYERFS_CONTAINER_PREFIX; // each required setting is named when missing delete process.env.LAYERFS_LOCAL_DIR; assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_LOCAL_DIR/.test(e.message)); -process.env.LAYERFS_LOCAL_DIR = "/store/local"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; delete process.env.LAYERFS_DAEMON; assert.throws(() => deployment(), (e) => e.code === "InvalidInput" && /LAYERFS_DAEMON/.test(e.message)); -// native: the same two settings, its own runtime name -clear(); -process.env.LAYERFS_SANDBOX = "native"; process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; -process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; -assert.deepEqual(deployment(), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); // a DSH_LAYERFS_NFS_ name is not mapped (an integration maps its own names) process.env.DSH_LAYERFS_NFS_MOUNT = "/bin/true"; deployment(); @@ -72,12 +68,12 @@ assert.equal(process.env.LAYERFS_NFS_MOUNT, undefined); // since #428 the SDK reads LAYERFS_ names only: a DSH_LAYERFS_ twin selects nothing clear(); setDocker(); -process.env.DSH_LAYERFS_SANDBOX = "local"; +process.env.DSH_LAYERFS_SANDBOX = "native"; process.env.DSH_LAYERFS_LOCAL_DIR = "/l"; assert.deepEqual(deployment(), docker); -// anything else is refused, never silently Docker -for (const typo of ["Local", "lokal", "LOCAL", "podman"]) { +// anything else, the former `local` included, is refused, never silently Docker +for (const typo of ["local", "Native", "nativ", "NATIVE", "podman"]) { clear(); setDocker(); process.env.LAYERFS_SANDBOX = typo; @@ -85,22 +81,22 @@ for (const typo of ["Local", "lokal", "LOCAL", "podman"]) { } // a client reads LAYERFS_SANDBOX and LAYERFS_LOCAL_DIR once, and -// sandbox.create sends the local deployment (no Docker setting) +// sandbox.create sends the native deployment (no Docker setting) clear(); store(); -process.env.LAYERFS_SANDBOX = "local"; +process.env.LAYERFS_SANDBOX = "native"; process.env.LAYERFS_DAEMON = "/opt/layerfs-daemon"; -process.env.LAYERFS_LOCAL_DIR = "/store/local"; +process.env.LAYERFS_LOCAL_DIR = "/Users/me/lfs"; const { LayerFS } = await import("../src/index.js"); const client = await LayerFS.open("website"); -assert.equal(client.options.deploy.sandbox, "local"); -assert.equal(client.options.deploy.localDir, "/store/local"); +assert.equal(client.options.deploy.sandbox, "native"); +assert.equal(client.options.deploy.localDir, "/Users/me/lfs"); const before = native.deploys.length; await client.sandbox.create(); -assert.deepEqual(native.deploys.slice(before), [{ sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }]); +assert.deepEqual(native.deploys.slice(before), [{ sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }]); // an image given to sandbox.create does not turn it into a Docker deployment await client.sandbox.create({ image: docker.image }); -assert.deepEqual(native.deploys.at(-1), { sandbox: "local", daemon: "/opt/layerfs-daemon", localDir: "/store/local" }); +assert.deepEqual(native.deploys.at(-1), { sandbox: "native", daemon: "/opt/layerfs-daemon", localDir: "/Users/me/lfs" }); await client.close(); // unset: the client carries no new key clear(); @@ -110,4 +106,4 @@ const plain = await LayerFS.open("website"); assert.equal("sandbox" in plain.options.deploy, false); assert.equal("localDir" in plain.options.deploy, false); await plain.close(); -console.log("local_deployment: ok"); +console.log("native_deployment: ok"); diff --git a/api/docs/agent.md b/api/docs/agent.md index 353e9f40d1..a7c3719523 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -145,8 +145,8 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | -| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `local`: a Sandbox is a daemon process on this machine, with no container and no Docker setting read (see [Local Sandboxes](#local-sandboxes-no-docker)). `native`: the same on macOS without root: Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Any other value is refused as `InvalidInput` before anything is made. | -| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=local` (or `native`), required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each local Sandbox gets its own directory under it. | +| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `native` (macOS only): a Sandbox is a daemon process of this user, with no container, no root and no Docker setting read; Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Elsewhere `sandbox.create` refuses `native` as `Unsupported`. Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=native`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each native Sandbox gets its own directory under it. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | @@ -246,72 +246,52 @@ Daemon/container restart recovery and replay of uncertain operations are exclude Background process ownership is in the native executor, including persisted process identity; the filesystem daemon launches and supervises no commands. -### Local Sandboxes (no Docker) - -`LAYERFS_SANDBOX=local` runs each Sandbox's daemon as a process of this -machine instead of in a container, for a Linux VM or CI job that has no Docker -and is itself the isolation boundary (a microVM, for instance). **It isolates -nothing**: the daemon runs as root in this machine's namespaces, and commands -share its processes, network and filesystem; they only run as their own uid -501 and gid 20, with no supplementary groups. Use it only where everything on -the machine belongs to the same trust domain. Docker stays the default. - -It needs root (the daemon mounts FUSE and commands run as another identity; -otherwise `workspace.mount` is refused `Unsupported`), `/dev/fuse` (the -daemon narrows it to 0600, as it does in a container), the daemon executable -(`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` and the client's Store path -(`LAYERFS_STORE_PATH`), which the daemon opens directly: the Store calls and -the daemons use the same file, on ext4, xfs or btrfs. `LAYERFS_DOCKER`, -`_IMAGE`, `_VOLUME` and `_DAEMON_HOST` are not read, and an `image` given to -`sandbox.create` is ignored. Everything else above holds: numbered Sandboxes -of up to 16 Workspaces, `sandbox.create` as the only start, `sandbox.close`, -idle close after 2 minutes, reclaim, records under -`LAYERFS_HOME//sandboxes//`, background processes, start intents -and their refusals. - -Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: -`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), -`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's -`path` is under this directory), `tmp/` (1777, the commands' `HOME` and the -background processes' logs) and `daemon.log` (the daemon's output, what -`docker logs` shows for a container and what -`backend.daemon.sandbox.get_log` reads; a failed start's report carries its -last 20 lines). The daemon listens on 127.0.0.1 only. Its records name the -daemon by process id and start time, so a reused id is never taken for it. -`sandbox.create`, the mount result and `project.list_sandboxes()` give a local -Sandbox's `image` as `local`. A local record is used only by a client whose -`LAYERFS_LOCAL_DIR` and prefix put that Sandbox exactly where the record -says; any other is refused `InvalidInput` and nothing is deleted. A daemon -that ends (idle close, crash, kill) is the stopped observation: Workspace -calls to it are refused, and the next `sandbox.create` or -`runtime.sandbox.reclaim` reclaims it, detaching any mount the dead daemon -left before deleting its directory. Nothing outside that directory is -removed; the Store is never touched. `@layerfs/runtime`'s Docker-side -`runtime.sandbox.list` and `sweep` do not see local Sandboxes. - ### Native Sandboxes (macOS) -`LAYERFS_SANDBOX=native` is the local runtime for macOS, with no root, no +`LAYERFS_SANDBOX=native` is the native runtime for macOS, with no root, no Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon runs as a process of this user and serves its Workspaces over NFSv3 on 127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them (`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands -run as this user, in this machine's namespaces: like `local`, **it isolates -nothing** beyond what the caller's own sandbox (for instance Seatbelt) -imposes. A command starts with this process's `PATH` (so `node`, Homebrew's -tools and the like are found as in the caller's own shell) and its own -environment, nothing else of this process's; a `local` Sandbox on Linux keeps -a container's fixed search path. +run as this user, in this machine's namespaces: **it isolates nothing** +beyond what the caller's own sandbox (for instance Seatbelt) imposes. A +command starts with this process's `PATH` (so `node`, Homebrew's tools and +the like are found as in the caller's own shell) and its own environment, +nothing else of this process's. Docker stays the default, and Linux keeps +the Docker route only. It is the in-process route: the client loads the macOS addon and opens the Store itself, so no worker endpoint (`LAYERFS_WORKER_ENDPOINT`) is set. It needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned by this user, 0755; the directories above it need not be traversable by others) and `LAYERFS_STORE_PATH` on a local apfs or hfs volume; a Store on any other -filesystem is refused `Unsupported`. Directory layout, records, idle close -and reclaim are those of local Sandboxes; a record's `runtime` is `native`, -and a native record is used only by a client whose deployment is `native` -with the same `LAYERFS_LOCAL_DIR` and prefix. Before a Commit the daemon +filesystem is refused `Unsupported`. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` +and `_DAEMON_HOST` are not read, and an `image` given to `sandbox.create` is +ignored. Everything else above holds: numbered Sandboxes of up to 16 +Workspaces, `sandbox.create` as the only start, `sandbox.close`, idle close +after 2 minutes, reclaim, records under `LAYERFS_HOME//sandboxes//` +and start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME`) and +`daemon.log` (the daemon's output, what `backend.daemon.sandbox.get_log` +reads; a failed start's report carries its last 20 lines). The daemon listens +on 127.0.0.1 only. Its records name the daemon by process id and start time, +so a reused id is never taken for it. `sandbox.create`, the mount result and +`project.list_sandboxes()` give a native Sandbox's `image` as `local`. A +record's `runtime` is `native`; it is used only by a client whose deployment +is `native` with the same `LAYERFS_LOCAL_DIR` and prefix, so it puts that +Sandbox exactly where the record says; any other is refused `InvalidInput` +and nothing is deleted. A record of the former `local` runtime (an earlier, +unmerged build) is never used: it is refused the same way and nothing it +names is deleted. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next `sandbox.create` +or `runtime.sandbox.reclaim` reclaims it, forcing off any mount the dead +daemon left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. `@layerfs/runtime`'s Docker-side +`runtime.sandbox.list` and `sweep` do not see native Sandboxes. Before a Commit the daemon writes back what the client buffered; after a Commit or merge installs another root, it makes the client drop the replaced files' cached attributes at once. @@ -326,8 +306,7 @@ itself. Limits: background processes still read `/proc` and are not available on macOS; many small writes cost about six times APFS's time; checked on Apple -silicon with macOS 27.2. On Linux the Store install needs root, which a -native Sandbox's commands may not be: use `local` there. +silicon with macOS 27.2. ### Exec output diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index 981acb3f27..333b2844ff 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -72,12 +72,6 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } -/// Whether this process runs as root, which the local runtime needs: its -/// daemon mounts FUSE and its commands run as another uid/gid. -#[cfg(any(target_os = "linux", target_os = "macos"))] -pub(crate) fn root() -> bool { - unsafe { geteuid() == 0 } -} /// This process's effective uid and gid: a native Sandbox's command /// identity (its commands run as this user). pub(crate) fn user() -> (u32, u32) { @@ -393,14 +387,7 @@ pub(crate) fn save_sandbox( .boot .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; fields.extend([ - ( - "runtime", - SdkValue::text(if crate::runtime::native(deploy) { - crate::runtime::NATIVE - } else { - crate::runtime::LOCAL - }), - ), + ("runtime", SdkValue::text(crate::runtime::NATIVE)), ("uid", SdkValue::Int(i64::from(state.identity.uid))), ("gid", SdkValue::Int(i64::from(state.identity.gid))), ( @@ -490,7 +477,7 @@ fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; Ok((docker, container, request)) } -/// What a Sandbox record names: a Docker container, or (`runtime` `local`) +/// What a Sandbox record names: a Docker container, or (`runtime` `native`) /// a local daemon process, its directory and port. pub(crate) enum Recorded { Docker(Docker, ContainerId, SandboxRequest), @@ -541,20 +528,22 @@ fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result ), )); } - // with no placement (#428), a mount names its Sandbox: local and native - // Sandboxes are still never mixed, so a record is used only by a - // deployment of its own runtime - let wanted = match args.get("deploy").map(crate::runtime::native) { - Some(true) => crate::runtime::NATIVE, - _ => crate::runtime::LOCAL, - }; - if value.get_text("runtime") != Some(wanted) { + // with no placement (#428), a mount names its Sandbox: a record is used + // only by a native deployment, and only a native record is. A record of + // the `local` runtime (an earlier, unmerged build) is never used, so + // nothing it names is deleted + let native = args.get("deploy").is_some_and(crate::runtime::native); + if !native || value.get_text("runtime") != Some(crate::runtime::NATIVE) { return Err(refused( "InvalidInput", format!( - "Sandbox {id} is a {} Sandbox, but this deployment is {wanted} \ - (LAYERFS_SANDBOX): local and native Sandboxes are never mixed", - value.get_text("runtime").unwrap_or_default() + "Sandbox {id} is a {} Sandbox, but only a native deployment \ + (LAYERFS_SANDBOX=native) uses a native record and none uses another; \ + Sandboxes of other runtimes are never mixed, so the record is neither used \ + nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + value.get_text("runtime").unwrap_or_default(), + registry::text(args, "project")? ), )); } @@ -591,7 +580,7 @@ fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { Err(refused( "Unsupported", - format!("Sandbox {id} is a local Sandbox (LAYERFS_SANDBOX=local), which needs Linux"), + format!("Sandbox {id} is a native Sandbox (LAYERFS_SANDBOX=native), which needs macOS"), )) } impl Recorded { diff --git a/engine/crates/layerfs-node/src/local.rs b/engine/crates/layerfs-node/src/local.rs index 8fcf370764..8f456ea20c 100644 --- a/engine/crates/layerfs-node/src/local.rs +++ b/engine/crates/layerfs-node/src/local.rs @@ -1,7 +1,6 @@ -//! Local Sandboxes (`deploy.sandbox` `local`, LAYERFS_SANDBOX=local): the -//! project's daemon runs as a process of this kernel instead of in a -//! container, for a VM or CI box that is itself the isolation boundary. It -//! isolates nothing from the rest of that machine. +//! Native Sandboxes (`deploy.sandbox` `native`, LAYERFS_SANDBOX=native, macOS): +//! the project's daemon runs as a process of this user instead of in a +//! container. It isolates nothing beyond the caller's own sandbox. //! //! Each Sandbox gets a directory `/-` //! with what a container held at fixed paths: its setup @@ -9,10 +8,9 @@ //! Workspace mounts (`/workspaces`). The daemon opens the installed Store //! where this process opens it (`storePath`), so the daemon setup and the //! Store's locator name that path instead of `/layerfs-store`. The daemon -//! listens on 127.0.0.1 only. Commands run as the same nonroot identity as -//! in a container. A native Sandbox (`deploy.sandbox` `native`) needs no -//! root: its daemon serves Workspaces over loopback NFS (`LAYERFS_KERNEL=nfs`) -//! to this host's own client, and its commands run as this user. Start, +//! listens on 127.0.0.1 only. A native Sandbox needs no root: its daemon +//! serves Workspaces over loopback NFS (`LAYERFS_KERNEL=nfs`) to this host's +//! own client, and its commands run as this user. Start, //! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's //! daemon stopped and its directory deleted, each once. use crate::job::{failed, refused}; @@ -28,7 +26,8 @@ use std::net::{SocketAddr, TcpStream}; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; -/// Starts the project's next Sandbox as a local daemon. Called by +/// Starts the project's next Sandbox as a native daemon (macOS only; +/// elsewhere refused before any effect). Called by /// [`crate::sandbox::start`] under the project's guard, with the project /// and its installed manifest already read. pub(crate) fn start( @@ -39,27 +38,23 @@ pub(crate) fn start( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; - let native = crate::runtime::native(deploy); // Inputs are read first, so a refused request leaves no creation intent. - if !native && !crate::discovery::root() { + if !cfg!(target_os = "macos") { return Err(refused( "Unsupported", - "LAYERFS_SANDBOX=local needs root: its daemon mounts FUSE and its commands run as \ - uid 501, gid 20. Run it as root inside the VM or CI box that isolates it", + "LAYERFS_SANDBOX=native mounts its Workspaces with this host's own NFS client and \ + runs commands as this user, which is supported on macOS only; use Docker here", )); } let root = deploy.get_text("localDir").ok_or_else(|| { refused( "InvalidInput", - "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a local Sandbox needs it", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a native Sandbox needs it", ) })?; - let local = if native { - Local::native(root).map(|local| local.environment(native_environment())) - } else { - Local::new(root) - } - .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let local = Local::native(root) + .map(|local| local.environment(native_environment())) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; let executable = PathBuf::from(text(deploy, "daemon")?); let store = match args.get_text("storePath") { Some(path) if Path::new(path).is_absolute() => path.to_owned(), @@ -81,7 +76,7 @@ pub(crate) fn start( &saved_directory, &id, project, - (local, executable, store, native), + (local, executable, store), ); if made.is_err() { let _ = std::fs::remove_dir(&saved_directory); @@ -97,7 +92,7 @@ fn start_claimed( saved_directory: &Path, id: &str, (project, installed): (&crate::registry::Project, StoreManifest), - (local, executable, store, native): (Local, PathBuf, String, bool), + (local, executable, store): (Local, PathBuf, String), ) -> Result, SdkRefusal> { let name = text(args, "project")?; let sandbox_name = sandbox_name(args, id)?; @@ -113,10 +108,8 @@ fn start_claimed( let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; let mut setup = crate::sandbox::setup(daemon.private, controller.public); - if native { - // the native route's commands run as this user, unconfined - (setup.command_uid, setup.command_gid) = crate::discovery::user(); - } + // the native route's commands run as this user, unconfined + (setup.command_uid, setup.command_gid) = crate::discovery::user(); setup.listen = "127.0.0.1:0".into(); setup.store = store.clone(); setup.overlay = path_text(&paths.overlay)?; diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs index 7ad7ddcb6e..ea7d830eea 100644 --- a/engine/crates/layerfs-node/src/runtime.rs +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -1,12 +1,10 @@ -//! A Sandbox's runtime: a Docker container (the default), or with -//! `deploy.sandbox` `local` (LAYERFS_SANDBOX=local) a daemon process on this -//! kernel, for a VM or CI box that is itself the isolation boundary. Both run -//! the same daemon and answer the same calls; only how it is started, how -//! commands are launched and how it is observed differ. With `native` -//! (LAYERFS_SANDBOX=native) the local daemon serves its Workspaces over -//! loopback NFS to this host's own client and commands run as this user, -//! with no root (macOS). The local runtimes run on Linux and macOS; -//! elsewhere selecting one is refused as Unsupported. +//! A Sandbox's runtime: a Docker container (the default), or on macOS with +//! `deploy.sandbox` `native` (LAYERFS_SANDBOX=native) a local daemon process +//! of this user that serves its Workspaces over loopback NFS to this host's +//! own client, its commands run as this user, with no root. Both run the +//! same daemon and answer the same calls; only how it is started, how +//! commands are launched and how it is observed differ. Off macOS selecting +//! `native` is refused as Unsupported before anything is made. use crate::job::{failed, refused}; use crate::output::Spool; use layerfs_bridge::control::{SdkRefusal, SdkValue}; @@ -16,7 +14,8 @@ use layerfs_sandbox::backend::local; use layerfs_sandbox::{CommandIdentity, ContainerId, WireFailure}; use std::net::SocketAddr; -/// A local Sandbox record's `runtime`, and the image it records (a Docker +/// The image a native Sandbox records and reports, and the `runtime` of a +/// record from the former `local` runtime, which is never used (a Docker /// record has no `runtime`). pub(crate) const LOCAL: &str = "local"; /// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local @@ -31,15 +30,15 @@ pub(crate) enum Runtime { } /// Whether the deployment selects a local daemon: `deploy.sandbox` unset or -/// `docker` is Docker, `local` and `native` are a local daemon; anything -/// else is refused before any effect. +/// `docker` is Docker, `native` is a local daemon; anything else (the +/// former `local` included) is refused before any effect. pub(crate) fn local(deploy: &SdkValue) -> Result { match deploy.get_text("sandbox") { None | Some("docker") => Ok(false), - Some(LOCAL) | Some(NATIVE) => Ok(true), + Some(NATIVE) => Ok(true), Some(other) => Err(refused( "InvalidInput", - format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or local (or native), not {other}"), + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or native (macOS), not {other}"), )), } } diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index 48da38db25..750c0c5454 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -121,8 +121,8 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result) -> SdkR /// `sandbox.create({ image })`: under the project's guard, reclaims the /// project's idle-closed Sandboxes, then starts the lowest free number on /// the call's image (`deploy.image`) and waits for its daemon: -/// `{ sandbox, name, image }`. With `deploy.sandbox` `local` it starts a -/// daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures +/// `{ sandbox, name, image }`. With `deploy.sandbox` `native` (macOS) it +/// starts a daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures /// are `Failed` or `Unknown`, never retried. pub(crate) fn create(args: &SdkValue) -> Outcome { let project = crate::discovery::project_dir(args)?; diff --git a/engine/crates/layerfs-node/tests/README.md b/engine/crates/layerfs-node/tests/README.md index d575dce6d1..3a36bb51a8 100644 --- a/engine/crates/layerfs-node/tests/README.md +++ b/engine/crates/layerfs-node/tests/README.md @@ -25,12 +25,14 @@ number replacement during inspection, and discovery-lock release while a forked child retains the descriptor. Its C fixture interposes `flock` only in the child Node process; the fixture's fork is bounded and released on test unwinding. -`native_local` covers local Sandbox records (`LAYERFS_SANDBOX=local`) through +`native_local` covers native Sandbox records (`LAYERFS_SANDBOX=native`) through `runtime.sandbox.reclaim`: a record naming any directory other than where its deployment puts that Sandbox is refused and nothing is deleted, the configured -`localDir` must be absolute and plain, a stopped local Sandbox is deleted and its -number freed, a damaged boot or port is refused, and local and native -(`LAYERFS_SANDBOX=native`) records are never used by the other's deployment. +`localDir` must be absolute and plain, a stopped native Sandbox is deleted and its +number freed, a damaged boot or port is refused, a native record is used only by +a native deployment, and a record of the former `local` runtime is never used or +deleted. Record handling is the same code on Linux and macOS, so it runs here; +starting a native Sandbox is macOS only. `native_admission` covers initial connection admission and its `Busy` deadline. The deadline test pays the product's full 60-second readiness wait. The mock diff --git a/engine/crates/layerfs-node/tests/native_local.rs b/engine/crates/layerfs-node/tests/native_local.rs index 25ab5f76b4..b6c8a7ba2d 100644 --- a/engine/crates/layerfs-node/tests/native_local.rs +++ b/engine/crates/layerfs-node/tests/native_local.rs @@ -1,5 +1,5 @@ -//! External tests of the built Node binding's local Sandbox records -//! (LAYERFS_SANDBOX=local or native). No private Rust source is included: every case +//! External tests of the built Node binding's native Sandbox records +//! (LAYERFS_SANDBOX=native). No private Rust source is included: every case //! writes a Sandbox record as a previous call would have, then calls the //! addon's public `runtime.sandbox.reclaim` and observes the reply and the //! files it left. Neither Docker, FUSE, nor a Store is needed. @@ -22,7 +22,7 @@ fn boot() -> String { .replace('-', "") } -/// Sandbox 1 of `proj` recorded as a `runtime` (local or native) Sandbox +/// Sandbox 1 of `proj` recorded as a `runtime` (native, or the former local) Sandbox /// in `directory` named `name`, its daemon long gone (no process has id /// u32::MAX). `edit` changes the fields before the record is written. fn record( @@ -93,7 +93,7 @@ fn own(root: &Path) -> PathBuf { own } -/// The public reclaim of a `sandbox` (local or native) deployment, with +/// The public reclaim of a `sandbox` (native or docker) deployment, with /// `localDir` when given, followed by `then` (JavaScript, `r` is the reply). fn reclaim(home: &Home, sandbox: &str, local_dir: Option<&str>, then: &str) { let deploy = match local_dir { @@ -118,7 +118,7 @@ fn text(path: &Path) -> &str { } #[test] -fn a_local_record_naming_another_directory_is_neither_used_nor_deleted() { +fn a_native_record_naming_another_directory_is_neither_used_nor_deleted() { let home = Home::new(); let root = root(&home); // a data directory a damaged or edited record names by its own name @@ -140,8 +140,8 @@ fn a_local_record_naming_another_directory_is_neither_used_nor_deleted() { (format!("{r}/../etc/layerfs-proj-1"), "layerfs-proj-1"), ("/".to_owned(), ""), ] { - record(&home, "local", &directory, name, |_| {}); - reclaim(&home, "local", Some(r), NEITHER); + record(&home, "native", &directory, name, |_| {}); + reclaim(&home, "native", Some(r), NEITHER); assert!( home.0.join("proj/sandboxes/1/sandbox.record").exists(), "the record of {directory} was removed" @@ -156,17 +156,17 @@ fn a_local_record_naming_another_directory_is_neither_used_nor_deleted() { } #[test] -fn a_local_record_is_used_only_with_its_deployment_s_absolute_local_dir() { +fn a_native_record_is_used_only_with_its_deployment_s_absolute_local_dir() { let home = Home::new(); let root = root(&home); let own = own(&root); - record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); let refused = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r));"; - reclaim(&home, "local", None, refused); - reclaim(&home, "local", Some("relative"), refused); + reclaim(&home, "native", None, refused); + reclaim(&home, "native", Some("relative"), refused); reclaim( &home, - "local", + "native", Some(&format!("{}/../local", text(&root))), refused, ); @@ -175,12 +175,12 @@ fn a_local_record_is_used_only_with_its_deployment_s_absolute_local_dir() { } #[test] -fn a_stopped_local_sandbox_where_its_deployment_puts_it_is_deleted_and_reclaimed() { +fn a_stopped_native_sandbox_where_its_deployment_puts_it_is_deleted_and_reclaimed() { let home = Home::new(); let root = root(&home); let own = own(&root); - record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); - reclaim(&home, "local", Some(text(&root)), RECLAIMED); + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); assert!(!own.exists(), "the stopped Sandbox's directory is deleted"); assert!( !home.0.join("proj/sandboxes/1").exists(), @@ -196,12 +196,12 @@ fn a_record_from_before_boot_ids_is_read_without_one() { let own = own(&root); record( &home, - "local", + "native", text(&own), "layerfs-proj-1", set("boot", None), ); - reclaim(&home, "local", Some(text(&root)), RECLAIMED); + reclaim(&home, "native", Some(text(&root)), RECLAIMED); assert!(!home.0.join("proj/sandboxes/1").exists()); } @@ -216,14 +216,14 @@ fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { ] { record( &home, - "local", + "native", text(&own), "layerfs-proj-1", set(key, Some(value)), ); reclaim( &home, - "local", + "native", Some(text(&root)), &format!("assert.equal(r.refused?.code, {code:?}, JSON.stringify(r));"), ); @@ -236,14 +236,14 @@ fn a_damaged_boot_or_port_is_refused_and_removes_nothing() { } #[test] -fn a_docker_record_stays_docker_without_a_local_deployment() { +fn a_docker_record_stays_docker_under_a_native_deployment() { // the default record of Home is a Docker one: with no Engine at its - // socket it is neither reclaimed nor read as a local Sandbox + // socket it is neither reclaimed nor read as a native Sandbox let home = Home::new(); let root = root(&home); reclaim( &home, - "local", + "native", Some(text(&root)), "assert.equal(r.refused?.code, 'Failed', JSON.stringify(r));", ); @@ -251,22 +251,23 @@ fn a_docker_record_stays_docker_without_a_local_deployment() { } #[test] -fn local_and_native_sandboxes_are_never_mixed() { +fn a_record_of_the_former_local_runtime_is_never_used_nor_deleted() { let home = Home::new(); let root = root(&home); let own = own(&root); let mixed = "assert.equal(r.refused?.code, 'InvalidInput', JSON.stringify(r)); - assert.match(r.refused.message, /never mixed/);"; - // a native record is not used by a local deployment - record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); - reclaim(&home, "local", Some(text(&root)), mixed); - // nor a local record by a native deployment + assert.match(r.refused.message, /never mixed/); + assert.match(r.refused.message, /neither used nor deleted/);"; + // a record of the former `local` runtime is used by no deployment record(&home, "local", text(&own), "layerfs-proj-1", |_| {}); reclaim(&home, "native", Some(text(&root)), mixed); + reclaim(&home, "docker", Some(text(&root)), mixed); + // nor is a native record by any deployment but a native one + record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); + reclaim(&home, "docker", Some(text(&root)), mixed); assert!(own.join("daemon.log").exists()); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); // a native deployment uses its own record: stopped, it is reclaimed - record(&home, "native", text(&own), "layerfs-proj-1", |_| {}); reclaim(&home, "native", Some(text(&root)), RECLAIMED); assert!(!own.exists()); assert!(!home.0.join("proj/sandboxes/1").exists()); From 25187697902bb60bce9c8fbbe1d5c0a849432797 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:51:39 +0800 Subject: [PATCH 60/64] test(sandbox): wait for local daemon reaping (H29) Poll endpoint through the Sandbox that owns the Child before asserting that no zombie remains. Bound the observation by WAIT and preserve the stopped and not-gone assertions without changing product behavior. Validation: cargo +1.85.1 fmt --all -- --check; cargo +1.85.1 clippy --locked -p layerfs-sandbox --all-targets -- -D warnings. Both passed on macOS with this worktree private target. Linux-only runtime tests were not executed; Hark owns Linux verification. Production LOC: 109210 -> 109210 (delta +0). Scope: core 41400, engine 65131, API 2679; reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Tests, tools, docs, dependencies and generated artifacts excluded; inline test code excluded by the counter. Method: tools/production_loc.py at Git blob 3341e4d0362c822b6d9a08419c432942b3a13800; export parent 3147916ef87bf8ac5666c04958b41cffbf9df6b1 and final staged tree d97b40d5742988ce316f080102a56d919bdc275a using git archive, then run python3 -B tools/production_loc.py --root --json. Both invocations exited 0. --- .../crates/layerfs-sandbox/tests/local_runtime.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/engine/crates/layerfs-sandbox/tests/local_runtime.rs b/engine/crates/layerfs-sandbox/tests/local_runtime.rs index e569fb1a99..205e27e590 100644 --- a/engine/crates/layerfs-sandbox/tests/local_runtime.rs +++ b/engine/crates/layerfs-sandbox/tests/local_runtime.rs @@ -331,8 +331,20 @@ fn a_daemon_that_ends_is_the_stopped_observation_and_only_its_delete_remains() { }; assert!(failure.container_stopped(), "{failure:?}"); assert!(!failure.container_gone()); + // Stopped can be observed before the child is reapable. Keep observing + // through its owner so try_wait reaps it once all its threads have ended. + let deadline = Instant::now() + WAIT; + let stat = loop { + let failure = sandbox.endpoint().unwrap_err(); + assert!(failure.container_stopped(), "{failure:?}"); + assert!(!failure.container_gone()); + let stat = fs::read_to_string(format!("/proc/{}/stat", process.pid)); + if stat.as_ref().map_or(true, |stat| !stat.contains(") Z ")) || Instant::now() >= deadline { + break stat; + } + std::thread::sleep(Duration::from_millis(20)); + }; // this process reaped its child: no zombie remains - let stat = fs::read_to_string(format!("/proc/{}/stat", process.pid)); assert!(stat.is_err() || !stat.unwrap().contains(") Z ")); // a reopen from the record observes the same let reopened = Sandbox::reopen( From 5de875348a90894b537ba8db50042ac92dfd771f Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:59:08 +0800 Subject: [PATCH 61/64] perf(nfs): suppress ignored macOS access-time updates Mount the macOS native export with noatime. The export drops SETATTR's access time and exposes the stored mtime for all three timestamps, so client access-time updates add work without persisting a distinct atime. Keep the Linux mount options unchanged. Document the native default and cover the macOS helper option while preserving actimeo=60. Exploratory Git scans reduced per-command SETATTR from 4831 to 2; wall time was noisy and does not establish a speedup. Imported Git index stat mismatches and attribute-less READDIRPLUS remain separate limitations. No Git stat checks, stored timestamps, inode identities or formats change. Validation: fmt; locked warning-denying Clippy for layerfs-daemon and layerfs-nfs all targets; NFS tests (17 passed before adding the helper test); final helper test (1 passed); product boundary guard; native mount observed noatime. No tests deleted. Linux native execution not run. Production LOC: 109210 -> 109210 (delta +0) Scope: core 41400, engine 65131, API 2679; dsh absent. Method: tools/production_loc.py at parent 251876979, identical counter on git archive of HEAD and staged tree 86a187429203d7a881be66a42b35a8e16ef6f871; nonblank/noncomment product Rust, runtime SQL and JS, excluding tests. reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- api/docs/agent.md | 5 ++++- .../crates/layerfs-daemon/src/control/export.rs | 8 ++++++-- engine/crates/layerfs-nfs/tests/mounter.rs | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 engine/crates/layerfs-nfs/tests/mounter.rs diff --git a/api/docs/agent.md b/api/docs/agent.md index a7c3719523..824f351768 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -298,7 +298,10 @@ at once. `LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's helper commands (`{port}` and `{dir}` are filled in). The default mount is -`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,noatime,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`noatime` suppresses client access-time updates, which can otherwise add a +SETATTR for each file read. The export keeps one timestamp, shows it as atime, +mtime and ctime, and drops the access time decoded from SETATTR. `LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are read; an integration that can set only its own prefixed names maps them diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 592b203e50..00bbd91a8f 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -88,7 +88,11 @@ impl Mounter { /// floor, and a soft call then gave up within seconds. `mutejukebox` /// keeps an `NFS3ERR_JUKEBOX` answer from raising the "server not /// responding" dialog. A dead export still fails calls (connection - /// refused), within seconds. + /// refused), within seconds. `noatime` stops the client from updating the + /// access time: this export keeps one timestamp and shows it as atime, + /// mtime and ctime alike, and `layerfs-nfs` decodes and drops a SETATTR's + /// access time, so the update would cost one SETATTR per file a read or a + /// scan touches and change nothing. pub fn host() -> Self { let helper = |words: &[&str]| Helper(words.iter().map(|w| (*w).to_owned()).collect()); if cfg!(target_os = "macos") { @@ -96,7 +100,7 @@ impl Mounter { mount: helper(&[ "/sbin/mount_nfs", "-o", - "vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse", + "vers=3,tcp,port={port},mountport={port},locallocks,nfc,noatime,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse", "127.0.0.1:/", "{dir}", ]), diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs new file mode 100644 index 0000000000..968ae2e0e0 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -0,0 +1,17 @@ +//! The native macOS client must not send atime updates the export ignores. +#![cfg(target_os = "macos")] +use layerfs_daemon::control::Mounter; + +#[test] +fn host_mount_disables_access_time_updates() { + let mounter = Mounter::host(); + let options = mounter + .mount + .0 + .windows(2) + .find(|pair| pair[0] == "-o") + .expect("mount options"); + assert!(options[1].split(',').any(|option| option == "noatime")); + // Keep the existing attribute-cache contract; no global noac tradeoff. + assert!(options[1].split(',').any(|option| option == "actimeo=60")); +} From 2aef367abb78eb56e4c2e2ae17c9d548b445c93f Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:15:35 +0800 Subject: [PATCH 62/64] docs: explain SQLite WAL compatibility on macOS NFS Document the system SQLite VFS restriction, safe pre-import journal conversion and cache placement. Link the guide from the SDK and documentation index. System nfsd comparison remains blocked by unavailable administrator authentication. Production LOC: 109210 -> 109210 (delta +0). Method: python3 -B tools/production_loc.py --json, counter blob 3341e4d0362c822b6d9a08419c432942b3a13800. First-parent and staged product trees are identical; only documentation changes. Counter scopes: core 41400, engine 65131, API 2679; layerfs-nfs is classified under core by this pinned counter. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- api/docs/agent.md | 4 +- docs/README.md | 1 + docs/general/sqlite-on-network-filesystems.md | 72 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 docs/general/sqlite-on-network-filesystems.md diff --git a/api/docs/agent.md b/api/docs/agent.md index a7c3719523..d42421cfe7 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -306,7 +306,9 @@ itself. Limits: background processes still read `/proc` and are not available on macOS; many small writes cost about six times APFS's time; checked on Apple -silicon with macOS 27.2. +silicon with macOS 27.2. Imported application SQLite databases in WAL mode +can fail on this NFS route with the system SQLite VFS; see +[SQLite database preparation and cache placement](../../docs/general/sqlite-on-network-filesystems.md). ### Exec output diff --git a/docs/README.md b/docs/README.md index 793b96abdd..0f8f57cf55 100644 --- a/docs/README.md +++ b/docs/README.md @@ -61,6 +61,7 @@ - [Core concepts](general/concepts.md) - [Sandbox cache design](general/sandbox-cache-design.md): cluster-two ownership, branch sharing, source structure and remaining qualification - [Workspace filesystem view](general/workspace-filesystem-view.md): global Store base, daemon overlay, lazy FUSE read path and ASCII diagrams +- [SQLite in macOS NFS Workspaces](general/sqlite-on-network-filesystems.md): imported WAL databases, local conversion and cache placement - [Workspace queue scheduling](general/workspace-queue-scheduling.md): archived; the shared-database queue and credits removed by #321 - [Optimization guide](general/optimization-guide.md): binding performance-engineering rules - [Optimization handbook](general/optimization-handbook.md): diagnostic groups, existing counters, scaling checks, procedure and the current candidate list diff --git a/docs/general/sqlite-on-network-filesystems.md b/docs/general/sqlite-on-network-filesystems.md new file mode 100644 index 0000000000..b3e90cc1af --- /dev/null +++ b/docs/general/sqlite-on-network-filesystems.md @@ -0,0 +1,72 @@ +# SQLite databases in macOS NFS Workspaces + +> **Status:** Current general guide. + +A SQLite database imported from a local filesystem can remain in WAL mode even +when its `-wal` and `-shm` files are absent. With macOS's system SQLite and its +default VFS, querying that database in an NFS Workspace can fail with +`unable to open database file` (`SQLITE_CANTOPEN`, SQLite result code 14). +Opening the connection alone can succeed; the first query exposes the failure. +This concerns application databases inside the Workspace. The LayerFS Store +stays on its supported local filesystem and keeps its configured profile. + +## Cause and scope + +SQLite's macOS automatic locking selection uses an NFS-specific I/O method table +for an `nfs` filesystem. That table has version 1 and no `xShmMap` callback, so it +cannot provide the shared-memory WAL index through the default route. This is a +SQLite VFS policy; a null callback does not establish a failed kernel `mmap` +system call. Result code 14 is not POSIX errno 14. + +A diagnostic on macOS 27.2 with system SQLite 3.54.0 and the native LayerFS NFSv3 +backend observed: + +| Application database | Local APFS | Imported or created in NFS Workspace | +| --- | --- | --- | +| Closed WAL database, querying a saved row | Success | `SQLITE_CANTOPEN` (14) | +| WAL database converted to DELETE before import | Success | Same saved row returned | +| New database requesting `journal_mode=WAL` | Returns `wal` | Returns `delete`; simple read/write succeeds | + +The LayerFS mount already used `locallocks`. Changing an NFS server's handling +of `-shm` files does not provide the callback disabled by the client's VFS. +SQLite builds or explicitly selected VFS implementations can behave differently; +this observation does not qualify every SQLite application or concurrent workload. +A separate macOS system-nfsd export was not tested because starting it required +administrator authentication unavailable to the diagnostic session. + +## Preparing application databases + +For a database the application permits in rollback-journal mode, stop every +process using it, keep a recoverable backup, then use SQLite on the original +local filesystem before import: + +```sh +sqlite3 -bail /absolute/local/path/application.db \ + 'PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode=DELETE;' +``` + +Proceed only after the checkpoint reports no busy readers/writers and the mode +change returns `delete`. Keep the application stopped until import finishes. +The mode change persists in the database header. Do not edit that header by hand +or delete a `-wal` file: committed data may still be in the WAL. Copying an active +database and its sidecars separately does not guarantee a consistent snapshot. + +For disposable caches, use the application's supported cache reset while it is +stopped, or configure its cache on a local APFS directory outside the mount. +The application may request WAL again on startup; check its actual returned mode +and its own requirements. Simple DELETE-mode success does not establish general +network-filesystem database safety or multi-host lock correctness. + +Import preserves application bytes and does not convert databases automatically. +Checking only sidecar names misses closed WAL databases. A future advisory check +could inspect the SQLite signature and header bytes 18/19 (value 2 denotes WAL +read/write format), without opening or modifying application databases. It would +need an explicit diagnostic surface and a bounded read strategy; it must not +exclude files, rewrite content or claim that a live import is consistent. + +## References + +- [SQLite WAL overview and network-filesystem restrictions](https://www.sqlite.org/wal.html) +- [SQLite Unix VFS source (`nfsIoMethods` and `autolockIoFinderImpl`)](https://www.sqlite.org/src/doc/trunk/src/os_unix.c) +- [SQLite database header format](https://www.sqlite.org/fileformat2.html#the_database_header) +- [SQLite result codes](https://www.sqlite.org/rescode.html#cantopen) From a32da57326d72e100f070a9e5be1e51dc1da0921 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:11:24 +0800 Subject: [PATCH 63/64] feat(sandbox): define full Runtime/Sandbox interface and Docker provider (Area B1) Extract the complete Sandbox runtime interface and shared native/node plumbing from 9aa10e65c onto upstream/main (35d36575c), implementing the Docker backend and leaving same-signature Unsupported slots for backend/{linux,macos,namespaces}/ so B2, B3 and #515 only replace their own backend directories. In layerfs-sandbox: - Define concrete Runtime (Docker, MacosNative, LinuxNative, Namespaces, Windows) and Sandbox (Docker, Native, Namespaces) enums in backend/runtime.rs. - Add provider capabilities (including LoopbackNfs, MicroVm, RemoteCloud), LaunchPhase/LaunchFailure, ProviderLaunchOptions (Local, Namespaces, Docker) and SandboxRecord (Local, Namespaces, Docker). - Add types::Image and cross-backend ExecRequest; move shared json and strings helpers to backend/. - Implement DockerProvider and DockerInstance over existing Docker Engine lifecycle and streaming execution. - Include shared backend/native/* lifecycle, identity, listener and streams modules under cfg(any(linux, macos)), with same-signature Unsupported stubs in backend/{linux,macos,namespaces}/. In layerfs-api: - Route SandboxApi through Arc via with_runtime and create_provider_file, retaining local resource ownership and command identity validation. - Share TCP/Noise/Hello and Store install pipeline in sandbox/connection.rs. In layerfs-node and @layerfs/agent: - Take verbatim 9aa10e65c shared binding modules (local, discovery, runtime, sandbox, workspace, exec, registry, lib) with a no-op unmount slot for B3. - Expose deploy.sandbox, deploy.localDir, deploy.sandboxBackend and deploy.imageArchive settings in @layerfs/agent. Production LOC: 102831 -> 106022 (delta +3191) First-parent comparison: upstream/main 35d36575c1eb4d23d4b54534a7030d855db890e7 (tree ddea2b5453ec2294a3a3096ff1341d29467c9ff5) -> committed tree ae48e2983d154f444b3ec7ff04211e2fec5b4fd1. Scopes: - core: 38395 -> 38395 (delta 0) - engine: 61777 -> 64938 (delta +3161) * layerfs-sandbox: 3417 -> 5526 (delta +2109) * layerfs-api: 1940 -> 2081 (delta +141) * layerfs-node: 2744 -> 3655 (delta +911) - api: 2659 -> 2689 (delta +30) * @layerfs/agent: 849 -> 879 (delta +30) - dsh: 0 -> 0 (delta 0) - reference: absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py (blob 3341e4d0362c822b6d9a08419c432942b3a13800), --root --json for both snapshots; first-party production Rust, runtime SQL and JavaScript; tests, docs, tooling, generated and third-party files excluded. --- api/agent/src/client.js | 48 +- api/agent/src/index.js | 19 +- api/docs/agent.md | 130 ++++- engine/crates/layerfs-api/PROVIDERS.md | 13 + .../examples/sandbox_kernel_mount.rs | 190 ++++---- .../layerfs-api/examples/sandbox_lifecycle.rs | 76 +-- .../crates/layerfs-api/src/base/internal.rs | 51 +- engine/crates/layerfs-api/src/lib.rs | 2 +- .../crates/layerfs-api/src/sandbox/close.rs | 12 +- .../layerfs-api/src/sandbox/connection.rs | 108 +++++ .../crates/layerfs-api/src/sandbox/create.rs | 349 +------------- engine/crates/layerfs-api/src/sandbox/exec.rs | 52 +- engine/crates/layerfs-api/src/sandbox/mod.rs | 10 +- .../crates/layerfs-api/src/sandbox/mount.rs | 9 +- .../layerfs-api/src/sandbox/provider.rs | 373 +++++++++++++++ .../layerfs-api/tests/provider_ownership.rs | 143 ++++++ .../layerfs-api/tests/sandbox_operations.rs | 114 +++-- engine/crates/layerfs-node/src/discovery.rs | 450 +++++++++++++++--- engine/crates/layerfs-node/src/exec.rs | 58 ++- engine/crates/layerfs-node/src/lib.rs | 4 + engine/crates/layerfs-node/src/local.rs | 440 +++++++++++++++++ engine/crates/layerfs-node/src/registry.rs | 21 +- engine/crates/layerfs-node/src/runtime.rs | 156 ++++++ engine/crates/layerfs-node/src/sandbox.rs | 165 ++++--- engine/crates/layerfs-node/src/unmount.rs | 45 ++ engine/crates/layerfs-node/src/workspace.rs | 100 +++- .../layerfs-node/tests/native_discovery.rs | 26 +- .../layerfs-node/tests/support/engine.rs | 34 +- engine/crates/layerfs-sandbox/Cargo.toml | 11 +- engine/crates/layerfs-sandbox/PROVIDERS.md | 134 ++++++ .../src/backend/docker/container.rs | 4 +- .../src/backend/docker/endpoint.rs | 74 ++- .../src/backend/docker/inspect.rs | 6 +- .../layerfs-sandbox/src/backend/docker/mod.rs | 8 +- .../src/backend/docker/provider.rs | 124 +++++ .../src/backend/docker/provider_exec.rs | 100 ++++ .../src/backend/docker/provider_failure.rs | 32 ++ .../src/backend/docker/request.rs | 59 +-- .../src/backend/docker/topology.rs | 6 +- .../src/backend/docker/version.rs | 2 +- .../layerfs-sandbox/src/backend/identity.rs | 16 + .../src/backend/{docker => }/json.rs | 0 .../layerfs-sandbox/src/backend/linux/mod.rs | 7 + .../src/backend/linux/mounts.rs | 20 + .../src/backend/linux/process.rs | 17 + .../layerfs-sandbox/src/backend/listener.rs | 68 +++ .../src/backend/macos/darwin.rs | 28 ++ .../layerfs-sandbox/src/backend/macos/mod.rs | 7 + .../src/backend/macos/mounts.rs | 16 + .../crates/layerfs-sandbox/src/backend/mod.rs | 20 +- .../src/backend/namespaces/mod.rs | 6 + .../src/backend/namespaces/provider.rs | 74 +++ .../src/backend/namespaces/sandbox.rs | 75 +++ .../src/backend/native/create.rs | 284 +++++++++++ .../src/backend/native/exec.rs | 133 ++++++ .../src/backend/native/groups.rs | 149 ++++++ .../layerfs-sandbox/src/backend/native/mod.rs | 40 ++ .../src/backend/native/process.rs | 85 ++++ .../src/backend/native/provider.rs | 178 +++++++ .../src/backend/native/sandbox.rs | 335 +++++++++++++ .../src/backend/native/search.rs | 26 + .../layerfs-sandbox/src/backend/runtime.rs | 307 ++++++++++++ .../layerfs-sandbox/src/backend/streams.rs | 23 + .../src/backend/{docker => }/strings.rs | 21 + .../src/backend/windows/README.md | 49 ++ engine/crates/layerfs-sandbox/src/lib.rs | 11 +- .../src/provider/capabilities.rs | 30 ++ .../layerfs-sandbox/src/provider/launch.rs | 26 + .../layerfs-sandbox/src/provider/mod.rs | 12 + .../layerfs-sandbox/src/provider/options.rs | 193 ++++++++ .../layerfs-sandbox/src/types/failure.rs | 23 + .../crates/layerfs-sandbox/src/types/image.rs | 43 ++ .../crates/layerfs-sandbox/src/types/mod.rs | 4 + .../layerfs-sandbox/src/types/request.rs | 48 ++ .../layerfs-sandbox/tests/docker_provider.rs | 394 +++++++++++++++ .../layerfs-sandbox/tests/engine_lifecycle.rs | 44 ++ .../layerfs-sandbox/tests/native_platform.rs | 71 +++ .../layerfs-sandbox/tests/runtime_enum.rs | 15 + tools/check_product_boundary.py | 14 +- tools/test_check_product_boundary.py | 10 +- 80 files changed, 5800 insertions(+), 880 deletions(-) create mode 100644 engine/crates/layerfs-api/PROVIDERS.md create mode 100644 engine/crates/layerfs-api/src/sandbox/connection.rs create mode 100644 engine/crates/layerfs-api/src/sandbox/provider.rs create mode 100644 engine/crates/layerfs-api/tests/provider_ownership.rs create mode 100644 engine/crates/layerfs-node/src/local.rs create mode 100644 engine/crates/layerfs-node/src/runtime.rs create mode 100644 engine/crates/layerfs-node/src/unmount.rs create mode 100644 engine/crates/layerfs-sandbox/PROVIDERS.md create mode 100644 engine/crates/layerfs-sandbox/src/backend/docker/provider.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/identity.rs rename engine/crates/layerfs-sandbox/src/backend/{docker => }/json.rs (100%) create mode 100644 engine/crates/layerfs-sandbox/src/backend/linux/mod.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/linux/process.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/listener.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/macos/mod.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/create.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/exec.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/groups.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/mod.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/process.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/provider.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/native/search.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/runtime.rs create mode 100644 engine/crates/layerfs-sandbox/src/backend/streams.rs rename engine/crates/layerfs-sandbox/src/backend/{docker => }/strings.rs (86%) create mode 100644 engine/crates/layerfs-sandbox/src/backend/windows/README.md create mode 100644 engine/crates/layerfs-sandbox/src/provider/capabilities.rs create mode 100644 engine/crates/layerfs-sandbox/src/provider/launch.rs create mode 100644 engine/crates/layerfs-sandbox/src/provider/mod.rs create mode 100644 engine/crates/layerfs-sandbox/src/provider/options.rs create mode 100644 engine/crates/layerfs-sandbox/src/types/image.rs create mode 100644 engine/crates/layerfs-sandbox/src/types/request.rs create mode 100644 engine/crates/layerfs-sandbox/tests/docker_provider.rs create mode 100644 engine/crates/layerfs-sandbox/tests/native_platform.rs create mode 100644 engine/crates/layerfs-sandbox/tests/runtime_enum.rs diff --git a/api/agent/src/client.js b/api/agent/src/client.js index 7e1ae74a98..8455d1cb28 100644 --- a/api/agent/src/client.js +++ b/api/agent/src/client.js @@ -17,8 +17,9 @@ export const BACKEND = /^(backend|runtime)\./; /** Runs one operation once and returns its value or throws its refusal; * `options` are the client's settings, whose `trace` selects its trace file; - * `transport` is the client's binding (default: the one a new client would use). */ -export async function invoke(op, args, options, transport = binding()) { + * `transport` is the client's binding (default: the one a new client of + * `args.project` would use). */ +export async function invoke(op, args, options, transport = binding(args.project)) { return outcome(await traced(options, op, args, (scope) => { if (BACKEND.test(op)) throw new LayerFSError("UnknownMethod", "backend and runtime methods are @layerfs/backend's and @layerfs/runtime's", null); return transport.call(op, observed(args, scope)); @@ -26,7 +27,7 @@ export async function invoke(op, args, options, transport = binding()) { } /** One call on the binding's backend table (`backend.*`, `runtime.*`). */ -export async function invokeBackend(op, args, options, transport = binding()) { +export async function invokeBackend(op, args, options, transport = binding(args?.project)) { return outcome(await traced(options, op, args, (scope) => transport.backendCall(op, observed(args, scope)))); } @@ -60,9 +61,19 @@ export function sandboxArg(sandbox) { /** Deployment settings from the client options, else the environment * (LAYERFS_). A Workspace's Sandbox needs - * every one but the container prefix and daemon host. */ + * every one but the container prefix and daemon host. With `sandbox` + * (LAYERFS_SANDBOX) `native` (Linux or macOS), a Sandbox is a host daemon: + * Linux uses FUSE with a root daemon and nonroot commands (501:20); + * macOS uses loopback NFS and this user. It needs the daemon and + * `localDir` (LAYERFS_LOCAL_DIR) and no Docker setting. Unset or `docker` is + * Docker; any other value is refused. */ export function deployment(options = {}) { const env = (name) => process.env[`LAYERFS_${name}`] || undefined; + const sandbox = options.sandbox ?? env("SANDBOX"); + if (sandbox === "native") return nativeDeployment(options, env, sandbox); + if (sandbox !== undefined && sandbox !== "docker") { + throw new LayerFSError("InvalidInput", `deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not ${sandbox}`, null); + } const deploy = { docker: options.docker ?? env("DOCKER") ?? "/var/run/docker.sock", image: options.image ?? env("IMAGE"), @@ -80,6 +91,35 @@ export function deployment(options = {}) { return deploy; } +/** A native Sandbox's settings: the daemon, the directory its Sandboxes + * live in, the optional name prefix, and the backend inside that directory + * (`sandboxBackend`, LAYERFS_SANDBOX_BACKEND: `native` when unset, or + * `namespaces` on Linux). The namespaces backend gives each Sandbox a root + * filesystem from an image, so it also needs `image` (LAYERFS_IMAGE, the + * same digest Docker deploys) and `imageArchive` (LAYERFS_IMAGE_ARCHIVE, the + * docker-archive the bundle carries). */ +function nativeDeployment(options, env, sandbox) { + const deploy = { + sandbox, + daemon: options.daemon ?? env("DAEMON"), + localDir: options.localDir ?? env("LOCAL_DIR"), + containerPrefix: options.containerPrefix ?? env("CONTAINER_PREFIX"), + }; + const backend = options.sandboxBackend ?? env("SANDBOX_BACKEND"); + if (backend !== undefined) deploy.sandboxBackend = backend; + const required = [["daemon", "DAEMON"], ["localDir", "LOCAL_DIR"]]; + if (backend === "namespaces") { + deploy.image = options.image ?? env("IMAGE"); + deploy.imageArchive = options.imageArchive ?? env("IMAGE_ARCHIVE"); + required.push(["image", "IMAGE"], ["imageArchive", "IMAGE_ARCHIVE"]); + } + for (const [name, setting] of required) { + if (!deploy[name]) throw new LayerFSError("InvalidInput", `deploy.${name} is not configured (LAYERFS_${setting})`, null); + } + if (!deploy.containerPrefix) delete deploy.containerPrefix; + return deploy; +} + /** Refuses every call on a closed client (or one no factory opened). */ export function live(client) { if (client?.[STATE]?.closed !== false) { diff --git a/api/agent/src/index.js b/api/agent/src/index.js index 446b59a534..1dc96b3c7e 100644 --- a/api/agent/src/index.js +++ b/api/agent/src/index.js @@ -100,7 +100,15 @@ export class LayerFS { * settings read LAYERFS_HOME, _DOCKER, _IMAGE, _VOLUME and _DAEMON; a * Sandbox this client starts runs _IMAGE * on _VOLUME. LAYERFS_CONTAINER_PREFIX (default `layerfs-`) names every - * container it starts `-`. + * container it starts `-`. On Linux or macOS, + * LAYERFS_SANDBOX=native starts each Sandbox as a host daemon in a directory + * under LAYERFS_LOCAL_DIR. Linux requires root for FUSE and runs commands + * as 501:20; macOS uses loopback NFS and the caller's identity. No Docker + * setting is read; native provides no isolation from the host. + * LAYERFS_SANDBOX_BACKEND picks the backend inside that directory: `native` + * (unset) or, on Linux, `namespaces` (namespaces, cgroups, nonroot commands). + * It is read for every call, so a Sandbox made with one backend is never + * reopened as another. * LAYERFS_DAEMON_HOST is the host this process * dials for a daemon's port, published on the Docker host's 127.0.0.1 only; * set it when the client runs in a container (`host.docker.internal` on @@ -129,13 +137,18 @@ export class LayerFS { volume: env("VOLUME"), daemon: env("DAEMON"), ...(env("CONTAINER_PREFIX") ? { containerPrefix: env("CONTAINER_PREFIX") } : {}), + ...(env("SANDBOX") ? { sandbox: env("SANDBOX") } : {}), + ...(env("SANDBOX_BACKEND") ? { sandboxBackend: env("SANDBOX_BACKEND") } : {}), + ...(env("LOCAL_DIR") ? { localDir: env("LOCAL_DIR") } : {}), + ...(env("IMAGE_ARCHIVE") ? { imageArchive: env("IMAGE_ARCHIVE") } : {}), ...(env("DAEMON_HOST") ? { daemonHost: env("DAEMON_HOST") } : {}), }, storePolicy: env("STORE_POLICY"), storePath: env("STORE_PATH"), }; - // not open until a factory has opened its Store; the binding is chosen now, once - this[STATE] = { closed: null, transport: binding() }; + // not open until a factory has opened its Store; the binding is chosen now, once, + // for this client's project (a VM placement has one worker per project) + this[STATE] = { closed: null, transport: binding(this.projectName) }; this.fs = bind(this, fsMethods); this.project = bind(this, { list_branches, list_sandboxes }); this.branch = bind(this, { list_commits, diff_commits, diff_branches, fork, merge, publish }); diff --git a/api/docs/agent.md b/api/docs/agent.md index 330fcab914..7dc9c7b565 100644 --- a/api/docs/agent.md +++ b/api/docs/agent.md @@ -45,7 +45,7 @@ async and return a ready client, and `new LayerFS()` is refused `InvalidInput`. **There is no options argument:** every setting comes from the environment, read once when the client is made, by its `LAYERFS_` name only: `LAYERFS_HOME`, `_STORE_PATH`, `_DOCKER`, `_IMAGE`, -`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_TRACE`, +`_VOLUME`, `_DAEMON`, `_CONTAINER_PREFIX`, `_DAEMON_HOST`, `_SANDBOX`, `_LOCAL_DIR`, `_TRACE`, `_STORE_POLICY` (and `_TRACE_STEP`, read at each call). An object passed where the options went (`LayerFS.open(name, {…})`, or an object as `LayerFS.init`'s third argument) is refused `InvalidInput`, never ignored. `open` opens the project's Store and makes one project lookup @@ -105,7 +105,9 @@ the Store on a local filesystem the daemons share (ext4, xfs, btrfs); anywhere else opening fails `Unsupported`, and without a `storePath` it fails `InvalidInput`. On macOS and Windows, any process that names its LayerFS home's worker (`LAYERFS_WORKER_ENDPOINT`, or the endpoint `@layerfs/runtime` -hands over with `useWorker`) sends the calls there instead. The route is open to any integration (owner, +hands over with `useWorker` — or that project's endpoint, on a +`vm:` home where each project's Store is in its own VM) sends the +calls there instead. The route is open to any integration (owner, 2026-10-24); LayerFS names none. A client keeps the binding it was made with: the in-process addon, or the worker if one was set when it was made, so a client made before `@layerfs/runtime` hands over its worker stays in-process and @@ -141,10 +143,13 @@ Keep each agent's operations on a Workspace sequential. | `LAYERFS_VOLUME` | Existing named local VM volume shared by the project's daemons. | | `LAYERFS_DAEMON` | Host path to the built Linux daemon executable. | | `LAYERFS_NATIVE` | Optional path of the real addon; defaults to `engine/crates/layerfs-node/layerfs.node`. | -| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows: the endpoint file of the LayerFS home's worker, which then makes every binding call. Any integration may set it. | +| `LAYERFS_WORKER_ENDPOINT` | macOS and Windows, and a `vm:` project VM: the endpoint file of the worker that makes this project's binding calls. Any integration may set it. | | `LAYERFS_STORE_PATH` | The installed Store file this client's Store calls open: `global/store.sqlite` in the project's volume. On Linux this process opens it, so it must be on a local filesystem the daemons share, such as `/layerfs-store/global/store.sqlite` where the volume is mounted at `/layerfs-store`; otherwise Store calls fail `Unsupported`. On macOS and Windows the worker opens it, so it is the path inside the worker's Linux, `/var/lib/docker/volumes//_data/global/store.sqlite`. On any other platform Store calls fail `Unsupported`. Unset, they fail `InvalidInput` everywhere. | | `LAYERFS_CONTAINER_PREFIX` | Optional. Names every container this client creates `-` (default prefix `layerfs-`); another LayerFS home on the same Docker engine sets its own, so names never collide. Docker's name rule applies (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at most 128). A name another container holds refuses the create before any effect. | | `LAYERFS_DAEMON_HOST` | Optional. The host this process dials for a daemon's port. Each daemon publishes its port on the Docker host's `127.0.0.1` only, and that stays so. A client inside a container does not share that loopback, so it names the host that reaches it, `host.docker.internal` on Docker Desktop. One resolution, its first address, one connect; nothing is tried again. Unset, the client dials `127.0.0.1`. | +| `LAYERFS_SANDBOX` | Optional. Unset or `docker`: a Sandbox is a Docker container, exactly as described here. `native` (macOS only): a Sandbox is a daemon process of this user, with no container, no root and no Docker setting read; Workspaces are served over loopback NFS and mounted by this machine's own NFS client, and commands run as this user (see [Native Sandboxes](#native-sandboxes-macos)). Elsewhere `sandbox.create` refuses `native` as `Unsupported`. Any other value is refused as `InvalidInput` before anything is made. | +| `LAYERFS_LOCAL_DIR` | With `LAYERFS_SANDBOX=native`, required: an existing absolute directory, owned by the running user, not group/other writable and traversable by others (0755). Each native Sandbox gets its own directory under it. | +| `LAYERFS_SANDBOX_BACKEND` | With `LAYERFS_SANDBOX=native` on Linux, optional: `native` (unset) or `namespaces`. `namespaces` puts each Sandbox's daemon in its own mount, PID and IPC namespaces and its own cgroup v2 directory, and runs commands as the sandbox's uid:gid with no capabilities and no-new-privs; see [Namespace Sandboxes (Linux)](#namespace-sandboxes-linux). It needs root, cgroup v2, `/dev/fuse` and `user.max_user_namespaces=0`, and is refused `Unsupported` elsewhere or without them. Any other value is refused `InvalidInput` before anything is made. It is read when the client is made and sent with every call, so a Sandbox made with one backend is never reopened as the other. | | `LAYERFS_TRACE` | Optional per-call trace file (see [the call trace](backend.md#the-call-trace)); unset means no trace. | | `LAYERFS_TRACE_STEP` | Optional step label put on each trace line as `step`; read at every call. | | `LAYERFS_TRACE_START` | `1` adds a start line before each traced call; read at every call. | @@ -244,6 +249,125 @@ Daemon/container restart recovery and replay of uncertain operations are exclude Background process ownership is in the native executor, including persisted process identity; the filesystem daemon launches and supervises no commands. +### Native Sandboxes (macOS) + +`LAYERFS_SANDBOX=native` is the native runtime for macOS, with no root, no +Docker, no kernel extension and no macFUSE or FUSE-T. Each Sandbox's daemon +runs as a process of this user and serves its Workspaces over NFSv3 on +127.0.0.1 (`LAYERFS_KERNEL=nfs`); macOS's own client mounts them +(`/sbin/mount_nfs`, as this user) under the Sandbox's `workspaces/`. Commands +run as this user, in this machine's namespaces: **it isolates nothing** +beyond what the caller's own sandbox (for instance Seatbelt) imposes. A +command starts with this process's `PATH` (so `node`, Homebrew's tools and +the like are found as in the caller's own shell) and its own environment, +nothing else of this process's. Docker stays the default, and Linux keeps +the Docker route only. + +It is the in-process route: the client loads the macOS addon and opens the +Store itself, so no worker endpoint (`LAYERFS_WORKER_ENDPOINT`) is set. It +needs the daemon executable (`LAYERFS_DAEMON`), `LAYERFS_LOCAL_DIR` (owned by +this user, 0755; the directories above it need not be traversable by others) +and `LAYERFS_STORE_PATH` on a local apfs or hfs volume; a Store on any other +filesystem is refused `Unsupported`. `LAYERFS_DOCKER`, `_IMAGE`, `_VOLUME` +and `_DAEMON_HOST` are not read, and an `image` given to `sandbox.create` is +ignored. Everything else above holds: numbered Sandboxes of up to 16 +Workspaces, `sandbox.create` as the only start, `sandbox.close`, idle close +after 2 minutes, reclaim, records under `LAYERFS_HOME//sandboxes//` +and start intents and their refusals. + +Sandbox `n` lives in `LAYERFS_LOCAL_DIR/-/`: +`config/daemon.setup` (its daemon setup, 0600 in a 0700 directory), +`overlay/` (0700), `workspaces/` (the Workspace mounts, so a Workspace's +`path` is under this directory), `tmp/` (1777, the commands' `HOME`) and +`daemon.log` (the daemon's output, what `backend.daemon.sandbox.get_log` +reads; a failed start's report carries its last 20 lines). The daemon listens +on 127.0.0.1 only. Its records name the daemon by process id and start time, +so a reused id is never taken for it. `sandbox.create`, the mount result and +`project.list_sandboxes()` give a native Sandbox's `image` as `local`. A +record's `runtime` is `native`; it is used only by a client whose deployment +is `native` with the same `LAYERFS_LOCAL_DIR` and prefix, so it puts that +Sandbox exactly where the record says; any other is refused `InvalidInput` +and nothing is deleted. A record of the former `local` runtime (an earlier, +unmerged build) is never used: it is refused the same way and nothing it +names is deleted. A daemon that ends (idle close, crash, kill) is the stopped +observation: Workspace calls to it are refused, and the next `sandbox.create` +or `runtime.sandbox.reclaim` reclaims it, forcing off any mount the dead +daemon left before deleting its directory. Nothing outside that directory is +removed; the Store is never touched. `@layerfs/runtime`'s Docker-side +`runtime.sandbox.list` and `sweep` do not see native Sandboxes. Before a Commit the daemon +writes back what the client buffered; after a Commit or merge installs +another root, it makes the client drop the replaced files' cached attributes +at once. + +`LAYERFS_NFS_MOUNT`, `_UNMOUNT`, `_FORCE` and `_FLUSH` replace the daemon's +helper commands (`{port}` and `{dir}` are filled in). The default mount is +`/sbin/mount_nfs -o vers=3,tcp,port={port},mountport={port},locallocks,nfc,actimeo=60,rsize=131072,wsize=131072,soft,intr,dumbtimer,timeo=300,retrans=6,mutejukebox,nobrowse 127.0.0.1:/ {dir}`. +`LAYERFS_NFS_REFRESH=0` turns the refresh after an install off (the client +then revalidates within `actimeo`, 60 s). Only these `LAYERFS_` names are +read; an integration that can set only its own prefixed names maps them +itself. + +Limits: background processes still read `/proc` and are not available on +macOS; many small writes cost about six times APFS's time; checked on Apple +silicon with macOS 27.2. + +### Namespace Sandboxes (Linux) + +`LAYERFS_SANDBOX_BACKEND=namespaces` keeps the native deployment (no Docker, no +image) and adds isolation inside the project's VM: each Sandbox's daemon runs +in its own mount, PID and IPC namespaces and its own cgroup v2 directory +(`/sys/fs/cgroup/layerfs/`), so two Sandboxes on one Store cannot +see each other's mounts, processes or files, and closing one ends everything +it started. + +It needs root (mount and PID namespaces, `/dev/fuse`), the unified cgroup v2 +hierarchy with `cgroup.kill`, and `user.max_user_namespaces=0`: without the +last, a command could unshare a user namespace and keep its own copy of the +Workspace mount, outside the daemon's unmount. The runtime never sets that +limit; the VM's owner does (`sysctl -w user.max_user_namespaces=0`). A host +that fails any of these is refused `Unsupported` before anything is made, and +the message names the missing condition. + +Sandbox `n` lives where a native Sandbox does — `LAYERFS_LOCAL_DIR/-/` +with `config/daemon.setup`, `overlay/`, `workspaces/`, `tmp/` and +`daemon.log` — plus `bin/layerfs-daemon` (the daemon executable, copied in +0755) and its sibling `-.root/`, the private root its init +stage mounts a tmpfs on and pivots to. The private root holds: the VM's own +userland (`/usr`, `/bin`, `/sbin`, `/lib`, `/lib64`, `/etc`) bound read-only, +so there is no image and no package restoration; a private `/proc` (only the +Sandbox's processes), `/dev` (the basic devices, `/dev/fuse`, a private +`devpts` and `shm`), `/tmp`; the Sandbox's own directory bound at its own path, +so `config/`, `overlay/` and `workspaces/` keep the same names on both +sides; and the Store's directory bound read-write, so the daemon's SQLite WAL +and the writers' directory lock see the same inode as every other opener on +the kernel. The daemon listens on the VM's `127.0.0.1`, as a native one does, +and `127.0.0.1:0` in its setup means the same thing. + +Commands run through `layerfs-sandbox-helper`, which is looked up at +`LAYERFS_SANDBOX_HELPER` when set and otherwise beside the daemon executable. +Each command enters the Sandbox's namespaces and runs as its uid:gid with no +effective or permitted capabilities and no-new-privs; it cannot mount, cannot +unmount the Workspace and cannot create a user namespace. A background process +it leaves behind (even after `setsid`) is ended by the Sandbox's close, which +is one `cgroup.kill` write followed by waiting for the group to be empty; +Delete then removes the cgroup and the directory, so a closed Sandbox leaves no +process, mount or cgroup behind. No graceful filesystem drain exists. + +A record's `runtime` is `namespaces`, with the same fields as a native record; +it is used only by a client whose `LAYERFS_SANDBOX_BACKEND` is `namespaces` +with the same `LAYERFS_LOCAL_DIR` and prefix. Another backend's record — and a +`namespaces` record seen by a native deployment — is refused `InvalidInput` +and nothing is deleted. Reopen checks the recorded init process (id, start +time, boot) and its cgroup membership, so a reused id, or a process outside +the Sandbox's cgroup, is never taken for it. Everything else above holds: +numbered Sandboxes, up to 16 Workspaces each, one daemon per Sandbox, idle +close after 2 minutes, reclaim, and `image` reported as `local`. + +Limits: the network namespace is shared with the VM, so a command can reach +another Sandbox's daemon port on `127.0.0.1`; `/sys` is not mounted; the +userland is the VM's, not a pinned image; and the private root's bind mounts +are read-only at their top only, as this kernel's remount is not recursive. + ### Exec output `exec` returns `{ kind, exitCode, stdout, stderr, message, truncated }` when the diff --git a/engine/crates/layerfs-api/PROVIDERS.md b/engine/crates/layerfs-api/PROVIDERS.md new file mode 100644 index 0000000000..16bd8b3cf9 --- /dev/null +++ b/engine/crates/layerfs-api/PROVIDERS.md @@ -0,0 +1,13 @@ +# Sandbox runtimes in the SDK + +Status: Current general guide. Native macOS is implemented and host-verified; Linux native is implemented, pending Linux host qualification; the Docker runtime is the default. `layerfs-sandbox` selects between them with the concrete `Runtime` enum; there are no provider traits. + +`layerfs-sandbox::backend` exposes `Runtime` (a deployment's choice: `Docker`, `MacosNative`, `LinuxNative`, `Windows`) and `Sandbox` (the launched owner of one Sandbox). `layerfs_sandbox::{Runtime, Sandbox}` re-export both; `layerfs_sandbox::provider` keeps the inputs and records they share. `ExecRequest` lives in the shared types layer; its Docker re-export remains compatible. `SandboxId` aliases the existing `ContainerId`, preserving encoded identities and records. The runtime layer has no daemon control dependency. + +`Runtime::MacosNative(..)` and `Runtime::LinuxNative(..)` wrap `NativeMacosProvider::new(root)` and `NativeLinuxProvider::new(root)`, the two aliases of the same native implementation whose instances share the existing local runtime owner. Each refuses the wrong platform before root access. macOS uses Darwin process facts, the `mount` table and `umount -f`; Linux uses the retained #402 `/proc` PID/start/boot checks, process groups and lazy `MNT_DETACH`. Linux requires a root daemon and a root directory traversable by command users; ordinary commands keep #402's nonroot identity (Node selects 501:20) and fixed container PATH. macOS keeps the caller identity, inherited PATH and loopback NFS. The provider explicitly selects `LAYERFS_KERNEL=fuse` on Linux and `nfs` on macOS and rejects conflicting launch overrides. Launch preserves the original single-attempt process lifecycle, native identity, inherited PATH, NFS environment and partial owner on failure. `inspect` reports exact process liveness; unknown exit/storage facts stay absent. `Runtime::Windows` refuses every operation with `RuntimeError::Unsupported` before any effect; no Windows host runtime exists. Records remain the Node binding's responsibility; runtimes neither scan directories nor infer project ownership. B8 list aggregation and doctor remain future work. + +`SandboxApi::with_runtime(Arc)` and `create_provider_file(ProviderCreate, store, manifest)` (or `create(ProviderCreate, project)`) launch through the deployment's runtime and return `ProviderSandbox`, whose `runtime` field is the `Sandbox` handle. Both Docker (via `Runtime::Docker(DockerProvider)`) and native runtimes route through this single unified API path. `WorkspaceApi::exec` delegates ordinary execution through `sandbox.runtime.run(...)` with streaming stdout/stderr buffers, and `api.close` drains and removes instances via `sandbox.runtime.stop(1)` and `sandbox.runtime.delete()`. Before launch, API validates authentication identities, selected Store, command identity and local resource ownership: overlay and mount home must belong to the instance, Store must live outside it, and native listen uses dynamic loopback. API encodes daemon configuration, then uses the shared TCP/Noise/Hello and Store-install pipeline in `sandbox/connection.rs`. Failures retain their phase, original cause, runtime, control, observed status, installation acknowledgement and handshake work. No automatic retry or cleanup occurs in API. + +`layerfs-node` builds a `Runtime` from `deploy.sandbox` (`docker` by default, `native` on Linux and macOS), keeps the launched `Sandbox` in its Sandbox state, and reads records, image, scratch directory and background-process tracking through it. Legacy deprecated forwarders (`SandboxApi::new(Docker)`, `SandboxCreate`, `ManagedSandbox`, `from_managed`, `internal::split/exec/control_managed`) remain removed. + +The Store remains explicitly Disposable/WAL/OFF. Native macOS keeps #427's direct Store access without a worker; this refactor does not change persistence authorization or filesystem semantics. diff --git a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs index c2e8f73671..cf2fd9aeed 100644 --- a/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs +++ b/engine/crates/layerfs-api/examples/sandbox_kernel_mount.rs @@ -4,8 +4,8 @@ //! registered with the daemon, read the complete root and mutate it through //! plain syscalls. use layerfs_api::{ - InitRequest, ManagedSandbox, MountedWorkspace, OperationCause, ProjectApi, SandboxApi, - SandboxCreate, WorkspaceApi, + InitRequest, MountedWorkspace, OperationCause, ProjectApi, ProviderCreate, ProviderSandbox, + SandboxApi, WorkspaceApi, }; use layerfs_bridge::{ control::{ControlCode, DaemonPhase, MountPhase, Reply, Request}, @@ -14,7 +14,8 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ @@ -22,6 +23,7 @@ use std::{ fs, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, + sync::Arc, time::{Duration, Instant}, }; @@ -100,59 +102,46 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -type Running = ( - layerfs_sandbox::backend::docker::RuntimeOutput, - layerfs_sandbox::backend::docker::ExecHandle, -); enum Directory<'a> { Mounted(&'a MountedWorkspace), Kernel(&'a str), } -/// Mounted commands use SDK exec; outside-mount commands retain ordinary Sandbox execution. -fn start( - owner: &ManagedSandbox, +fn run_dir( + runtime: &layerfs_sandbox::Sandbox, + identity: layerfs_sandbox::CommandIdentity, script: &str, - directory: Directory<'_>, -) -> Result> { - let arguments = vec!["bash".into(), "-c".into(), script.into()]; - let created = match directory { - Directory::Mounted(mounted) => WorkspaceApi::exec(owner, mounted, arguments, vec![], false), - Directory::Kernel(directory) => layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - arguments, - vec![], - directory.into(), - false, - ), - } - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - Ok((output, handle)) -} -fn finish((output, handle): Running) -> Result, Box> { - let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); - output - .copy_to(&mut stdout, &mut stderr) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - if inspected.known_root_exit() != Some(0) { + directory: &str, +) -> Result, String> { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = runtime + .run( + &layerfs_sandbox::ExecRequest { + identity, + arguments: vec!["bash".into(), "-c".into(), script.into()], + environment: vec![], + directory: directory.into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("exec {f:?}"))?; + if exit.code != 0 { return Err(format!( - "command {inspected:?}: {} {}", + "command exit {exit:?}: {} {}", String::from_utf8_lossy(&stdout), String::from_utf8_lossy(&stderr) - ) - .into()); + )); } Ok(stdout) } -fn run( - owner: &ManagedSandbox, - script: &str, - directory: Directory<'_>, -) -> Result, Box> { - finish(start(owner, script, directory)?) +fn run(owner: &ProviderSandbox, script: &str, directory: Directory<'_>) -> Result, String> { + let dir = match directory { + Directory::Mounted(mounted) => &mounted.ready.directory, + Directory::Kernel(dir) => dir, + }; + run_dir(owner.runtime.as_ref(), owner.command_identity, script, dir) } fn main() -> Result<(), Box> { let began = Instant::now(); @@ -191,22 +180,30 @@ fn main() -> Result<(), Box> { fs::remove_dir_all(directory.join("source"))?; let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; - let api = SandboxApi::new(Docker::new(&args[1], Duration::from_secs(3))?); + let docker = Docker::new(&args[1], Duration::from_secs(3))?; + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); let mut owner = api .create( - SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "kernel-mount-test".into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup: setup(daemon.private, keys.public), - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }, &project, ) @@ -323,45 +320,54 @@ fn main() -> Result<(), Box> { // Reversible normal unmount: a running command's working directory keeps // the kernel busy. The filesystem neither kills nor waits for the command. - let holder = start( - &owner, - "sleep 1.2", - Directory::Kernel(&format!("{root}/node_modules")), - )?; - run( - &owner, - &format!( - "until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{root}/node_modules'; do sleep 0.02; done" - ), - Directory::Kernel("/"), - )?; - let busy = WorkspaceApi::new(&mut owner) - .unmount(mounted.bound.token, false) - .unwrap_err(); - match &busy.cause { - OperationCause::Remote(refusal) => { - assert_eq!( - (refusal.code, refusal.phase.as_str()), - (ControlCode::Busy, "unmount:kernel") - ); + let node_modules_dir = format!("{root}/node_modules"); + let identity = owner.command_identity; + let (runtime, control) = layerfs_api::internal::provider_parts(&mut owner); + std::thread::scope(|s| { + let holder = s.spawn(|| run_dir(runtime, identity, "sleep 1.2", &node_modules_dir)); + run_dir( + runtime, + identity, + &format!("until ls -l /proc/[0-9]*/cwd 2>/dev/null | grep -q '{node_modules_dir}'; do sleep 0.02; done"), + "/", + ) + .unwrap(); + let busy = WorkspaceApi::from_control(control) + .unmount(mounted.bound.token, false) + .unwrap_err(); + match &busy.cause { + OperationCause::Remote(refusal) => { + assert_eq!( + (refusal.code, refusal.phase.as_str()), + (ControlCode::Busy, "unmount:kernel") + ); + } + other => panic!("{other:?}"), } - other => panic!("{other:?}"), - } - let status = WorkspaceApi::new(&mut owner).status(mounted.bound.token)?; - let kernel = status.kernel.expect("kernel status block"); - assert_eq!(kernel.phase, MountPhase::Ready); - assert_eq!( - kernel.ready.as_ref(), - Some(&mounted.ready), - "same connection" - ); - assert_eq!( - run(&owner, "cat .git/index", Directory::Mounted(&mounted))?, - FILES[1].1 - ); - finish(holder)?; - println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); - mark("busy"); + let status = WorkspaceApi::from_control(control) + .status(mounted.bound.token) + .unwrap(); + let kernel = status.kernel.expect("kernel status block"); + assert_eq!(kernel.phase, MountPhase::Ready); + assert_eq!( + kernel.ready.as_ref(), + Some(&mounted.ready), + "same connection" + ); + assert_eq!( + run_dir( + runtime, + identity, + "cat .git/index", + &mounted.ready.directory + ) + .unwrap(), + FILES[1].1 + ); + holder.join().unwrap().unwrap(); + println!("UNMOUNT_BUSY phase=unmount:kernel workspace_ready=true command_untouched=true"); + mark("busy"); + }); // Terminal unmount: detach, loop joins, drain, revocation and Close. WorkspaceApi::new(&mut owner) diff --git a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs index f54c059ff8..f6958361ab 100644 --- a/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs +++ b/engine/crates/layerfs-api/examples/sandbox_lifecycle.rs @@ -1,6 +1,6 @@ //! Positive-path host Init -> protected owned Sandbox -> direct shared Store control proof. //! Typed failure custody is covered separately by public adapter tests; this example reports failures in stderr. -use layerfs_api::{InitRequest, ManagedSandbox, ProjectApi, SandboxApi, SandboxCreate}; +use layerfs_api::{InitRequest, ProjectApi, ProviderCreate, ProviderSandbox, SandboxApi}; use layerfs_bridge::{ control::{DaemonPhase, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, @@ -8,13 +8,15 @@ use layerfs_bridge::{ }; use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId, WorkspaceId}; use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; -use layerfs_sandbox::backend::docker::{Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; use layerfs_storage::StoragePolicy; use layerfs_telemetry::timer::Timing; use std::{ fs, io::Write, path::PathBuf, + sync::Arc, time::{Duration, Instant}, }; fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { @@ -39,25 +41,26 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { existing_store: None, } } -fn ordinary(owner: &ManagedSandbox) -> Result<(), Box> { - let command="set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; - let created = layerfs_api::internal::exec( - &owner.runtime, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, - vec!["bash".into(), "-c".into(), command.into()], - vec![], - "/".into(), - false, - ) - .map_err(|f| format!("create {f:?}"))?; - let attached = created.start().map_err(|f| format!("start {f:?}"))?; - let (_input, output, handle) = attached.into_parts().map_err(|f| format!("split {f:?}"))?; - let progress = output - .copy_to(&mut std::io::stdout(), &mut std::io::stderr()) - .map_err(|f| format!("output {f:?}"))?; - let inspected = handle.inspect().map_err(|f| format!("inspect {f:?}"))?; - println!("ORDINARY_RESULT {:?} {:?}", progress, inspected); - assert_eq!(inspected.known_root_exit(), Some(0)); +fn ordinary(owner: &ProviderSandbox) -> Result<(), Box> { + let command = "set -e; test \"$(id -u)\" = 501; test \"$(id -g)\" = 20; test \"$(id -G)\" = 20; grep -q '^CapEff:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^CapAmb:[[:space:]]*0000000000000000$' /proc/self/status; grep -q '^NoNewPrivs:[[:space:]]*1$' /proc/self/status; for p in /layerfs-local/config/daemon.setup /layerfs-local/overlay/overlay.sqlite /layerfs-store/global/store.sqlite /proc/1/root/layerfs-local/config/daemon.setup /proc/1/root/layerfs-store/global/store.sqlite /proc/1/environ /proc/1/fd; do test ! -r \"$p\"; done; test ! -w /dev/fuse; test ! -w /sys/fs/fuse/connections; printf 'ORDINARY_ACCESS uid=501 gid=20 groups=20 CapEff=0 CapAmb=0 NoNewPrivs=1 private_backing_denied proc_aliases_denied fuse_control_denied\\n'"; + let mut stdout = std::io::stdout(); + let mut stderr = std::io::stderr(); + let exit = owner + .runtime + .run( + &layerfs_sandbox::ExecRequest { + identity: layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), command.into()], + environment: vec![], + directory: "/".into(), + stdin: false, + }, + &mut stdout, + &mut stderr, + ) + .map_err(|f| format!("run {f:?}"))?; + println!("ORDINARY_RESULT {:?}", exit); + assert_eq!(exit.code, 0); Ok(()) } fn main() -> Result<(), Box> { @@ -96,23 +99,30 @@ fn main() -> Result<(), Box> { let keys = transport::generate_keypair()?; let daemon = transport::generate_keypair()?; let docker = Docker::new(&args[1], Duration::from_secs(3))?; - let api = SandboxApi::new(docker); - let make = |setup: DaemonSetup| SandboxCreate { - deployment: SandboxRequest { - image: args[2].clone(), - store_volume: args[3].clone(), - port: 30421, + let provider = Arc::new(layerfs_sandbox::Runtime::Docker(DockerProvider::new( + docker, + ))); + let api = SandboxApi::with_runtime(provider); + let make = |name: &str, setup: DaemonSetup| ProviderCreate { + launch: LaunchRequest { + name: name.into(), + daemon_executable: PathBuf::from(&args[4]), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(3), + options: ProviderLaunchOptions::Docker { + image: args[2].clone(), + store_volume: args[3].clone(), + port: 30421, + connect_host: None, + }, }, setup, - executable: PathBuf::from(&args[4]), controller_private: keys.private, daemon_peer: daemon.public, - startup_wait: Duration::from_secs(3), - connect_host: None, }; let mut first = api - .create(make(setup(daemon.private, keys.public)), &project) + .create(make("first", setup(daemon.private, keys.public)), &project) .map_err(|f| format!("first {f:?}"))?; println!( "OWNED_FIRST_CONTAINER {} phase={:?}", @@ -131,7 +141,7 @@ fn main() -> Result<(), Box> { let mut second_setup = setup(daemon.private, keys.public); second_setup.existing_store = Some(installed.manifest.clone()); let second = api - .create(make(second_setup), &project) + .create(make("second", second_setup), &project) .map_err(|f| format!("second {f:?}"))?; println!( "OWNED_SECOND_CONTAINER {} phase={:?}", @@ -161,7 +171,7 @@ fn main() -> Result<(), Box> { let mut reopened_setup = setup(daemon.private, keys.public); reopened_setup.existing_store = Some(installed.manifest.clone()); let mut reopened = api - .create(make(reopened_setup), &project) + .create(make("reopened", reopened_setup), &project) .map_err(|f| format!("reopen {f:?}"))?; println!( "OWNED_REOPEN_CONTAINER {} phase={:?}", diff --git a/engine/crates/layerfs-api/src/base/internal.rs b/engine/crates/layerfs-api/src/base/internal.rs index 7bffa705c0..108b2f0251 100644 --- a/engine/crates/layerfs-api/src/base/internal.rs +++ b/engine/crates/layerfs-api/src/base/internal.rs @@ -98,59 +98,40 @@ pub fn into_connection(control: Control) -> Connection { /// Original authenticated control owner, without a reconnect or reset. #[cfg(unix)] -pub fn control(sandbox: &mut crate::ManagedSandbox) -> &mut Control { +pub fn control(sandbox: &mut crate::sandbox::ProviderSandbox) -> &mut Control { &mut sandbox.control } -/// The runtime, control owner and command identity of one Sandbox, taken -/// apart for an owner that serves several Workspaces at once: commands run on -/// the runtime while the control owner answers one request at a time. +/// Transfers provider runtime, original channel and admitted command identity. #[cfg(unix)] -pub fn split( - sandbox: crate::ManagedSandbox, +pub fn split_provider( + sandbox: crate::sandbox::ProviderSandbox, ) -> ( - layerfs_sandbox::backend::docker::Sandbox, + Box, Control, layerfs_sandbox::CommandIdentity, ) { (sandbox.runtime, sandbox.control, sandbox.command_identity) } +/// Borrows the same provider instance and original channel for concurrent execution. +#[cfg(unix)] +pub fn provider_parts( + sandbox: &mut crate::sandbox::ProviderSandbox, +) -> (&layerfs_sandbox::Sandbox, &mut Control) { + (sandbox.runtime.as_ref(), &mut sandbox.control) +} /// Original one-attempt startup used by existing-Store deployments and lifecycle proofs. #[cfg(unix)] pub fn start( api: &crate::SandboxApi, - request: crate::SandboxCreate, -) -> Result> { - crate::sandbox::create::create(&api.docker, request) + request: crate::sandbox::ProviderCreate, +) -> Result> { + api.start(request) } /// Existing bounded startup observation for control-channel qualification. #[cfg(unix)] pub fn sandbox_status( - sandbox: &mut crate::ManagedSandbox, + sandbox: &mut crate::sandbox::ProviderSandbox, wait_for_store: bool, ) -> Result> { sandbox.status(wait_for_store) } - -/// Disjoint original runtime/control borrows for concurrent Workspace orchestration. -#[cfg(unix)] -pub fn parts( - sandbox: &mut crate::ManagedSandbox, -) -> (&layerfs_sandbox::backend::docker::Sandbox, &mut Control) { - (&sandbox.runtime, &mut sandbox.control) -} - -/// The same ordinary launcher for external host commands and disjoint concurrent proof borrows. -#[cfg(unix)] -pub fn exec( - sandbox: &layerfs_sandbox::backend::docker::Sandbox, - identity: layerfs_sandbox::CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result< - layerfs_sandbox::backend::docker::ExecCreated, - Box, -> { - crate::sandbox::exec::launch(sandbox, identity, arguments, environment, directory, stdin) -} diff --git a/engine/crates/layerfs-api/src/lib.rs b/engine/crates/layerfs-api/src/lib.rs index e652d32640..1641980cd8 100644 --- a/engine/crates/layerfs-api/src/lib.rs +++ b/engine/crates/layerfs-api/src/lib.rs @@ -27,7 +27,7 @@ pub use layerfs_project::store::{ pub use layerfs_project::{InitCounters, InitCounts, InitStep, ProjectError, SourceRefusal}; #[cfg(unix)] pub use sandbox::{ - connect_address, ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, + connect_address, ProviderCreate, ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, }; pub use sandbox::{BoundWorkspace, MountFailure, MountedWorkspace, Unmounted, WorkspaceApi}; diff --git a/engine/crates/layerfs-api/src/sandbox/close.rs b/engine/crates/layerfs-api/src/sandbox/close.rs index e0a57ec08a..5f55414a9e 100644 --- a/engine/crates/layerfs-api/src/sandbox/close.rs +++ b/engine/crates/layerfs-api/src/sandbox/close.rs @@ -1,10 +1,10 @@ //! Explicit stop then delete of the exact acknowledged Sandbox runtime owner. -use super::{ManagedSandbox, SandboxApi, SandboxCause, SandboxFailure, SandboxPhase}; +use super::{ProviderFailure, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase}; impl SandboxApi { /// Stops once, then deletes once after acknowledged stop. No Workspace drain is inferred. /// A failed attempt retains the original runtime, control and deployment inputs; no replay. - pub fn close(&self, mut sandbox: ManagedSandbox) -> Result<(), Box> { + pub fn close(&self, mut sandbox: ProviderSandbox) -> Result<(), Box> { if let Err(cause) = sandbox.runtime.stop(1) { return Err(failure( sandbox, @@ -23,14 +23,14 @@ impl SandboxApi { } } fn failure( - sandbox: ManagedSandbox, + sandbox: ProviderSandbox, phase: SandboxPhase, cause: SandboxCause, -) -> Box { - Box::new(SandboxFailure { +) -> Box { + Box::new(ProviderFailure { request: sandbox.request, phase, - sandbox: Some(sandbox.runtime), + runtime: Some(sandbox.runtime), control: Some(sandbox.control), handshake: sandbox.handshake, observed: Some(sandbox.startup), diff --git a/engine/crates/layerfs-api/src/sandbox/connection.rs b/engine/crates/layerfs-api/src/sandbox/connection.rs new file mode 100644 index 0000000000..ba3a75df2d --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/connection.rs @@ -0,0 +1,108 @@ +//! One shared TCP/Noise/Hello pipeline, retaining progress through every failure. +use super::create::{hello, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest}, + duplex::{initiate_observed, ChannelWork}, +}; +use std::{ + net::{SocketAddr, TcpStream}, + time::Duration, +}; + +pub(super) struct Startup { + pub phase: SandboxPhase, + pub control: Option, + pub handshake: ChannelWork, + pub observed: Option, +} +impl Startup { + pub fn new() -> Self { + Self { + phase: SandboxPhase::Validate, + control: None, + handshake: ChannelWork::default(), + observed: None, + } + } + pub fn connect( + &mut self, + endpoint: SocketAddr, + host: Option<&str>, + wait: Duration, + secret: &[u8; 32], + peer: [u8; 32], + existing: bool, + ) -> Result { + self.phase = SandboxPhase::Connect; + let address = super::create::connect_address(endpoint, host).map_err(SandboxCause::Io)?; + let stream = TcpStream::connect_timeout(&address, wait).map_err(SandboxCause::Io)?; + stream + .set_read_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + stream + .set_write_timeout(Some(wait)) + .map_err(SandboxCause::Io)?; + self.phase = SandboxPhase::Authenticate; + let start = initiate_observed(stream, secret, peer); + self.handshake = start.work; + self.control = Some(Control::new(start.result.map_err(SandboxCause::Channel)?)); + self.phase = SandboxPhase::Policy; + self.control + .as_ref() + .expect("authenticated") + .clear_io_waits() + .map_err(SandboxCause::Channel)?; + self.phase = SandboxPhase::Hello; + let status = hello( + self.control.as_mut().expect("authenticated"), + HelloRequest { + expected_instance: None, + wait_for_store: existing, + }, + ) + .map_err(SandboxCause::Control)?; + self.observed = Some(status.clone()); + let expected = if existing { + DaemonPhase::ControlReady + } else { + DaemonPhase::InstallPending + }; + if status.phase != expected { + return Err(SandboxCause::Setup(ControlError("original startup phase"))); + } + Ok(status) + } +} +/// Installation and its Ready observation are identical for every runtime. +pub(super) fn install( + control: &mut Control, + startup: &mut DaemonStatus, + installed: &mut Option, + phase: &mut SandboxPhase, + store: &layerfs_persistence::SealedStore, + manifest: &layerfs_bridge::provision::StoreManifest, +) -> Result<(), SandboxCause> { + *phase = SandboxPhase::Install; + *installed = Some( + control + .install_file(store, manifest) + .map_err(SandboxCause::Install)?, + ); + *phase = SandboxPhase::Hello; + let status = hello( + control, + HelloRequest { + expected_instance: Some(startup.instance), + wait_for_store: true, + }, + ) + .map_err(SandboxCause::Control)?; + *startup = status; + if startup.phase != DaemonPhase::ControlReady { + return Err(SandboxCause::Setup(ControlError( + "original installed startup phase", + ))); + } + Ok(()) +} diff --git a/engine/crates/layerfs-api/src/sandbox/create.rs b/engine/crates/layerfs-api/src/sandbox/create.rs index 2146783582..61837c6693 100644 --- a/engine/crates/layerfs-api/src/sandbox/create.rs +++ b/engine/crates/layerfs-api/src/sandbox/create.rs @@ -1,48 +1,10 @@ //! Ordinary Sandbox startup, original deployment inputs and failure custody. use crate::{internal::Control, OperationFailure}; -use layerfs_bridge::{ - control::{ControlError, DaemonPhase, DaemonStatus, HelloRequest, Reply, Request}, - daemon_setup::DaemonSetup, - duplex::{initiate_observed, public_key, ChannelError, ChannelWork}, -}; -use layerfs_sandbox::backend::docker::{ - Docker, ListenerFailure, Sandbox, SandboxCreateFailure, SandboxRequest, UploadFailure, -}; -use std::{fmt, fs::File, io, net::TcpStream, path::PathBuf, time::Duration}; -/// One actual stopped-container deployment, private archive and authenticated startup. -#[derive(Clone)] -pub struct SandboxCreate { - /// Exact immutable image and borrowed shared named volume. - pub deployment: SandboxRequest, - /// Explicit private daemon resources and Disposable install/open selection. - pub setup: DaemonSetup, - /// Sealed Linux executable supplied by the caller; streamed once, never read as Store data. - pub executable: PathBuf, - /// Controller Noise secret; redacted from Debug and never sent in env/argv. - pub controller_private: [u8; 32], - /// Expected actual daemon identity, checked against private deployment config. - pub daemon_peer: [u8; 32], - /// Explicit listener event wait; does not apply to ordinary commands. - pub startup_wait: Duration, - /// Host to dial for the daemon's loopback-published port, when this - /// process does not share the Docker host's loopback (a client in a - /// container: `host.docker.internal`). `None` dials the observed - /// 127.0.0.1 endpoint. The publish and its check are unchanged. - pub connect_host: Option, -} -impl fmt::Debug for SandboxCreate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxCreate") - .field("deployment", &self.deployment) - .field("setup", &self.setup) - .field("executable", &self.executable) - .field("controller_private", &"[redacted]") - .field("daemon_peer", &self.daemon_peer) - .field("startup_wait", &self.startup_wait) - .field("connect_host", &self.connect_host) - .finish() - } -} +use layerfs_bridge::control::{ControlError, DaemonStatus, HelloRequest, Reply, Request}; +use layerfs_bridge::duplex::ChannelError; +use layerfs_sandbox::Runtime; +use std::{io, sync::Arc}; + /// The address to dial for a daemon `endpoint` observed on the Docker host's /// loopback: the endpoint itself, or the port on `host` (one resolution, its /// first address; nothing is tried twice). @@ -62,6 +24,7 @@ pub fn connect_address( ) }) } + /// Concrete boundary reached by one original lifecycle operation. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SandboxPhase { @@ -94,6 +57,7 @@ pub enum SandboxPhase { /// One delete attempt after acknowledged stop. Delete, } + /// Exact original failure, independent of owned resource disposition. #[derive(Debug)] pub enum SandboxCause { @@ -101,14 +65,10 @@ pub enum SandboxCause { Io(io::Error), /// Local config or correlated startup validation. Setup(ControlError), - /// Actual original runtime Create including partial decoded ID. - Create(Box), - /// Actual incremental archive extraction failure and work. - Upload(Box), - /// Original listener wait/progress/fence failure. - Listener(Box), /// Original Engine transfer, no replay. Runtime(Box), + /// Original provider launch cause; partial runtime ownership is retained separately. + Provider(Box), /// Original authentication failure; handshake work remains in the receipt. Channel(ChannelError), /// Original correlated daemon control exchange/refusal. @@ -116,181 +76,13 @@ pub enum SandboxCause { /// Original streaming installation failure and exact publication knowledge. Install(Box), } -/// Caller retains exact acknowledged owner, connection and original deployment inputs. -pub struct SandboxFailure { - /// Original inputs with secrets retained privately. - pub request: SandboxCreate, - /// Original reached boundary. - pub phase: SandboxPhase, - /// Exact acknowledged owned container; no automatically adopted partial ID. - pub sandbox: Option, - /// Original authenticated control owner, if established. - pub control: Option, - /// Actual original handshake work, including failure. - pub handshake: ChannelWork, - /// Original correlated Hello, even if its phase failed selection validation. - pub observed: Option, - /// Installation acknowledgement, retained if the following Ready observation fails. - pub installed: Option, - /// Exact original cause. - pub cause: SandboxCause, -} -impl fmt::Debug for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SandboxFailure") - .field("request", &self.request) - .field("phase", &self.phase) - .field("sandbox", &self.sandbox) - .field("has_control", &self.control.is_some()) - .field("handshake", &self.handshake) - .field("observed", &self.observed) - .field("cause", &self.cause) - .finish() - } -} -impl fmt::Display for SandboxFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) - } -} -impl std::error::Error for SandboxFailure {} + /// Actual local Engine and authenticated daemon lifecycle organization. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct SandboxApi { - pub(crate) docker: Docker, -} -/// Exact acknowledged container and authenticated control owner. -pub struct ManagedSandbox { - /// Owned runtime container; explicit stop/delete only. - pub runtime: Sandbox, - /// Existing sequential control owner; correlation and failed state are never reset. - pub(crate) control: Control, - /// Original acknowledged installation, when this create installed the Project. - pub installed: Option, - pub(crate) request: SandboxCreate, - /// Original correlated startup status; this is not kernel Workspace Ready. - pub startup: DaemonStatus, - /// Actual original duplex handshake work. - pub handshake: ChannelWork, - pub(crate) command_identity: layerfs_sandbox::CommandIdentity, -} -impl SandboxApi { - /// Uses one explicit local Engine endpoint and metadata wait policy. - pub fn new(docker: Docker) -> Self { - Self { docker } - } - /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. - pub fn create( - &self, - request: SandboxCreate, - project: &crate::SealedProject, - ) -> Result> { - self.create_file(request, &project.store, &project.manifest) - } - /// Creates a new daemon using exact persisted sealed-file and manifest facts. - pub fn create_file( - &self, - request: SandboxCreate, - store: &layerfs_persistence::SealedStore, - manifest: &layerfs_bridge::provision::StoreManifest, - ) -> Result> { - let existing = request.setup.existing_store.is_some(); - if let Some(selected) = &request.setup.existing_store { - let mut expected = manifest.clone(); - expected.daemon_sqlite = selected.daemon_sqlite.clone(); - if *selected != expected { - return Err(Box::new(SandboxFailure { - request, - phase: SandboxPhase::Validate, - sandbox: None, - control: None, - handshake: ChannelWork::default(), - observed: None, - installed: None, - cause: SandboxCause::Setup(ControlError("selected existing Project")), - })); - } - } - let mut owner = create(&self.docker, request)?; - if existing { - // The original startup already selected and waited for this Store. - // Installing again would change the existing one-attempt lifecycle. - return Ok(owner); - } - let installed = owner - .control - .install_file(store, manifest) - .map_err(|cause| { - Box::new(SandboxFailure { - request: owner.request.clone(), - phase: SandboxPhase::Install, - sandbox: None, - control: None, - handshake: owner.handshake, - observed: Some(owner.startup.clone()), - installed: None, - cause: SandboxCause::Install(cause), - }) - }); - let installed = match installed { - Ok(installed) => installed, - Err(mut failure) => { - failure.sandbox = Some(owner.runtime); - failure.control = Some(owner.control); - return Err(failure); - } - }; - owner.installed = Some(installed); - match hello( - &mut owner.control, - HelloRequest { - expected_instance: Some(owner.startup.instance), - wait_for_store: true, - }, - ) { - Ok(status) if status.phase == DaemonPhase::ControlReady => { - owner.startup = status; - Ok(owner) - } - Ok(status) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(status), - installed: owner.installed, - cause: SandboxCause::Setup(ControlError("original installed startup phase")), - })), - Err(cause) => Err(Box::new(SandboxFailure { - request: owner.request, - phase: SandboxPhase::Hello, - sandbox: Some(owner.runtime), - control: Some(owner.control), - handshake: owner.handshake, - observed: Some(owner.startup), - installed: owner.installed, - cause: SandboxCause::Control(cause), - })), - } - } -} -impl ManagedSandbox { - /// Explicit startup observation/wait on the same incarnation and Control owner. - /// A later observation never resolves an earlier unknown operation. - pub(crate) fn status( - &mut self, - wait_for_store: bool, - ) -> Result> { - hello( - &mut self.control, - HelloRequest { - expected_instance: Some(self.startup.instance), - wait_for_store, - }, - ) - } + pub(crate) runtime: Arc, } + pub(super) fn hello( control: &mut Control, request: HelloRequest, @@ -301,118 +93,3 @@ pub(super) fn hello( reply => Err(OperationFailure::unexpected(request, reply)), } } -pub(crate) fn create( - docker: &Docker, - request: SandboxCreate, -) -> Result> { - let mut phase = SandboxPhase::Validate; - let mut sandbox = None; - let mut control = None; - let mut handshake = ChannelWork::default(); - let mut observed = None; - let result = (|| { - let config = request.setup.encode().map_err(SandboxCause::Setup)?; - if request.setup.listen != format!("0.0.0.0:{}", request.deployment.port) - || request.setup.store != "/layerfs-store/global/store.sqlite" - || request.setup.overlay != "/layerfs-local/overlay/overlay.sqlite" - || request.setup.mounts != "/workspaces" - || request.startup_wait.is_zero() - || public_key(&request.controller_private).map_err(SandboxCause::Channel)? - != request.setup.control_peer - || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? - != request.daemon_peer - { - return Err(SandboxCause::Setup(ControlError( - "concrete deployment topology/identity", - ))); - } - phase = SandboxPhase::Executable; - let mut file = File::open(&request.executable).map_err(SandboxCause::Io)?; - phase = SandboxPhase::Create; - sandbox = Some( - docker - .create_sandbox(request.deployment.clone()) - .map_err(SandboxCause::Create)?, - ); - let owner = sandbox.as_mut().expect("acknowledged container"); - phase = SandboxPhase::Upload; - owner - .upload(&mut file, &config) - .map_err(SandboxCause::Upload)?; - phase = SandboxPhase::Start; - owner.start().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Listener; - owner - .wait_listener(request.startup_wait) - .map_err(SandboxCause::Listener)?; - phase = SandboxPhase::Endpoint; - let endpoint = owner.endpoint().map_err(SandboxCause::Runtime)?; - phase = SandboxPhase::Connect; - let address = - connect_address(endpoint, request.connect_host.as_deref()).map_err(SandboxCause::Io)?; - let stream = - TcpStream::connect_timeout(&address, request.startup_wait).map_err(SandboxCause::Io)?; - stream - .set_read_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - stream - .set_write_timeout(Some(request.startup_wait)) - .map_err(SandboxCause::Io)?; - phase = SandboxPhase::Authenticate; - let start = initiate_observed(stream, &request.controller_private, request.daemon_peer); - handshake = start.work; - let connection = start.result.map_err(SandboxCause::Channel)?; - // The startup handshake waits are cleared before control/product operations. - // Bridge owns cloned duplex directions; clear policy through that actual owner. - control = Some(Control::new(connection)); - phase = SandboxPhase::Policy; - control - .as_ref() - .expect("authenticated") - .clear_io_waits() - .map_err(SandboxCause::Channel)?; - phase = SandboxPhase::Hello; - let status = hello( - control.as_mut().expect("authenticated"), - HelloRequest { - expected_instance: None, - wait_for_store: request.setup.existing_store.is_some(), - }, - ) - .map_err(SandboxCause::Control)?; - let expected = if request.setup.existing_store.is_some() { - DaemonPhase::ControlReady - } else { - DaemonPhase::InstallPending - }; - observed = Some(status.clone()); - if status.phase != expected { - return Err(SandboxCause::Setup(ControlError("original startup phase"))); - } - Ok(status) - })(); - match result { - Ok(startup) => Ok(ManagedSandbox { - request: request.clone(), - installed: None, - runtime: sandbox.take().expect("owned"), - control: control.take().expect("authenticated"), - startup, - handshake, - command_identity: layerfs_sandbox::CommandIdentity { - uid: request.setup.command_uid, - gid: request.setup.command_gid, - }, - }), - Err(cause) => Err(Box::new(SandboxFailure { - request, - phase, - sandbox, - control, - handshake, - observed, - installed: None, - cause, - })), - } -} diff --git a/engine/crates/layerfs-api/src/sandbox/exec.rs b/engine/crates/layerfs-api/src/sandbox/exec.rs index c0db8c8e5b..40e0a5e449 100644 --- a/engine/crates/layerfs-api/src/sandbox/exec.rs +++ b/engine/crates/layerfs-api/src/sandbox/exec.rs @@ -1,44 +1,30 @@ //! Optional ordinary Sandbox launch at the acknowledged mounted directory. -use crate::{ManagedSandbox, MountedWorkspace, WorkspaceApi}; -use layerfs_sandbox::{ - backend::docker::{ExecCreateFailure, ExecCreated, ExecRequest, Sandbox}, - CommandIdentity, -}; +use super::{MountedWorkspace, ProviderSandbox, WorkspaceApi}; +use layerfs_sandbox::{provider::ExecExit, ExecRequest, WireFailure}; +use std::io::Write; impl WorkspaceApi<'_> { - /// Creates one ordinary command at this mount using the deployment's admitted identity. - /// The returned Sandbox runtime owner owns Start, streams, exit status and cancellation. + /// Runs one ordinary command at this mount using the deployment's admitted identity. + /// The runtime provider owns execution, streams, exit status and cancellation. /// This sends no daemon request, registers no Exec and performs no Commit or unmount. pub fn exec( - sandbox: &ManagedSandbox, + sandbox: &ProviderSandbox, workspace: &MountedWorkspace, arguments: Vec, environment: Vec, - stdin: bool, - ) -> Result> { - launch( - &sandbox.runtime, - sandbox.command_identity, - arguments, - environment, - workspace.ready.directory.clone(), - stdin, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + sandbox.runtime.run( + &ExecRequest { + identity: sandbox.command_identity, + arguments, + environment, + directory: workspace.ready.directory.clone(), + stdin: false, + }, + stdout, + stderr, ) } } -pub(crate) fn launch( - sandbox: &Sandbox, - identity: CommandIdentity, - arguments: Vec, - environment: Vec, - directory: String, - stdin: bool, -) -> Result> { - sandbox.create_exec(ExecRequest { - identity, - arguments, - environment, - directory, - stdin, - }) -} diff --git a/engine/crates/layerfs-api/src/sandbox/mod.rs b/engine/crates/layerfs-api/src/sandbox/mod.rs index 6983a84746..1bf10e271d 100644 --- a/engine/crates/layerfs-api/src/sandbox/mod.rs +++ b/engine/crates/layerfs-api/src/sandbox/mod.rs @@ -1,6 +1,12 @@ //! Actual ordinary Sandbox lifecycle and its Workspaces, with exact logical and kernel readiness. #[cfg(unix)] mod close; +#[cfg(unix)] +mod connection; +#[cfg(unix)] +mod provider; +#[cfg(unix)] +pub use provider::{ProviderCreate, ProviderFailure, ProviderSandbox}; mod commit; #[cfg(unix)] pub(crate) mod create; @@ -12,9 +18,9 @@ mod unmount; pub mod values; pub use commit::CommitReply; #[cfg(unix)] -pub use create::{connect_address, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase}; +pub use create::SandboxApi; #[cfg(unix)] -pub use create::{ManagedSandbox, SandboxApi}; +pub use create::{connect_address, SandboxCause, SandboxPhase}; pub use mount::BoundWorkspace; pub use mount::WorkspaceApi; pub use mount::{MountFailure, MountedWorkspace}; diff --git a/engine/crates/layerfs-api/src/sandbox/mount.rs b/engine/crates/layerfs-api/src/sandbox/mount.rs index 15d2e70aab..4d0414cebc 100644 --- a/engine/crates/layerfs-api/src/sandbox/mount.rs +++ b/engine/crates/layerfs-api/src/sandbox/mount.rs @@ -12,9 +12,14 @@ pub struct WorkspaceApi<'a> { impl<'a> WorkspaceApi<'a> { /// Borrows this Sandbox's original authenticated filesystem control owner. #[cfg(unix)] - pub fn new(sandbox: &'a mut crate::ManagedSandbox) -> Self { + pub fn new(sandbox: &'a mut super::ProviderSandbox) -> Self { + Self::from_control(&mut sandbox.control) + } + /// Borrows an authenticated control channel directly. + #[cfg(unix)] + pub fn from_control(control: &'a mut Control) -> Self { Self { - control: &mut sandbox.control, + control, observation_scope: None, } } diff --git a/engine/crates/layerfs-api/src/sandbox/provider.rs b/engine/crates/layerfs-api/src/sandbox/provider.rs new file mode 100644 index 0000000000..098f66dc8b --- /dev/null +++ b/engine/crates/layerfs-api/src/sandbox/provider.rs @@ -0,0 +1,373 @@ +//! Runtime startup uses the same authenticated connection and Store installation as Docker. +use super::{connection::Startup, SandboxApi, SandboxCause, SandboxPhase}; +use crate::internal::Control; +use layerfs_bridge::{ + control::{ControlError, DaemonStatus}, + daemon_setup::DaemonSetup, + duplex::{public_key, ChannelWork}, + provision::StoreManifest, +}; +use layerfs_sandbox::{ + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, Sandbox, +}; +use std::{fmt, sync::Arc}; + +/// Runtime launch and private daemon authentication inputs. +#[derive(Clone)] +pub struct ProviderCreate { + /// Runtime launch; configuration is encoded from setup after validation. + pub launch: LaunchRequest, + /// Explicit Disposable daemon topology and Store selection. + pub setup: DaemonSetup, + /// Controller secret; never printed or sent in argv/environment. + pub controller_private: [u8; 32], + /// Expected daemon public identity. + pub daemon_peer: [u8; 32], +} +impl fmt::Debug for ProviderCreate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderCreate") + .field("launch", &self.launch) + .field("setup", &self.setup) + .field("daemon_peer", &self.daemon_peer) + .finish_non_exhaustive() + } +} +/// Exact startup failure and all acknowledged ownership, without automatic cleanup. +pub struct ProviderFailure { + /// Original private request. + pub request: ProviderCreate, + /// Last attempted stage. + pub phase: SandboxPhase, + /// Acknowledged runtime, including a partially launched instance. + pub runtime: Option>, + /// Authenticated channel when established. + pub control: Option, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Last correlated startup observation. + pub observed: Option, + /// Installation acknowledgement even when the following Hello fails. + pub installed: Option, + /// Original deciding cause. + pub cause: SandboxCause, +} +impl fmt::Debug for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderFailure") + .field("phase", &self.phase) + .field("runtime", &self.runtime.as_ref().map(|r| r.identity())) + .field("cause", &self.cause) + .finish_non_exhaustive() + } +} +impl fmt::Display for ProviderFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Sandbox {:?}: {:?}", self.phase, self.cause) + } +} +impl std::error::Error for ProviderFailure {} +/// Runtime owner and its original authenticated daemon channel. +pub struct ProviderSandbox { + /// Original private launch and daemon configuration inputs. + pub request: ProviderCreate, + /// Explicit runtime lifecycle owner. + pub runtime: Box, + /// Original correlated startup observation. + pub startup: DaemonStatus, + /// Actual handshake work. + pub handshake: ChannelWork, + /// Store installation acknowledgement, if installed by this create. + pub installed: Option, + pub(crate) control: Control, + /// Command identity admitted for this Sandbox deployment. + pub command_identity: CommandIdentity, +} +impl ProviderSandbox { + /// Explicit startup observation/wait on the same incarnation and Control owner. + pub(crate) fn status( + &mut self, + wait_for_store: bool, + ) -> Result> { + super::create::hello( + &mut self.control, + layerfs_bridge::control::HelloRequest { + expected_instance: Some(self.startup.instance), + wait_for_store, + }, + ) + } +} +impl SandboxApi { + /// Uses the deployment's selected runtime. + pub fn with_runtime(runtime: Arc) -> Self { + Self { runtime } + } + /// Creates, privately configures, starts and authenticates once, preserving partial ownership on failure. + pub fn create( + &self, + request: ProviderCreate, + project: &crate::SealedProject, + ) -> Result> { + self.create_provider_file(request, &project.store, &project.manifest) + } + /// Launches once and authenticates to Hello without Store installation (internal proof only). + pub(crate) fn start( + &self, + request: ProviderCreate, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + ) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed: None, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed: None, + cause, + })), + } + } + /// Launches once, authenticates, selects or installs the Store, then observes Ready. + pub fn create_provider_file( + &self, + request: ProviderCreate, + store: &layerfs_persistence::SealedStore, + manifest: &StoreManifest, + ) -> Result> { + let mut progress = Startup::new(); + let mut launched = None; + let mut installed = None; + let result = (|| { + if let Some(directory) = + self.runtime + .local_directory(&request.launch.name) + .map_err(|cause| { + SandboxCause::Runtime(layerfs_sandbox::WireFailure::local(cause)) + })? + { + if std::path::Path::new(&request.setup.overlay) + != directory.join("overlay/overlay.sqlite") + || std::path::Path::new(&request.setup.mounts) != directory.join("workspaces") + || std::path::Path::new(&request.setup.store).starts_with(&directory) + || request.setup.listen != "127.0.0.1:0" + { + return Err(SandboxCause::Setup(ControlError( + "provider local resource ownership", + ))); + } + } + let mut launch = request.launch.clone(); + launch.daemon_config = request.setup.encode().map_err(SandboxCause::Setup)?; + let local_identity = match &launch.options { + ProviderLaunchOptions::Local { + command_identity, .. + } + | ProviderLaunchOptions::Namespaces { + command_identity, .. + } => Some(command_identity), + ProviderLaunchOptions::Docker { .. } => None, + }; + if let Some(command_identity) = local_identity { + if (command_identity.uid, command_identity.gid) + != (request.setup.command_uid, request.setup.command_gid) + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + } + if request + .setup + .existing_store + .as_ref() + .is_some_and(|m| m.locator != request.setup.store) + || launch.startup_timeout.is_zero() + || public_key(&request.controller_private).map_err(SandboxCause::Channel)? + != request.setup.control_peer + || public_key(&request.setup.private_key).map_err(SandboxCause::Channel)? + != request.daemon_peer + { + return Err(SandboxCause::Setup(ControlError( + "provider topology/identity", + ))); + } + if let Some(selected) = &request.setup.existing_store { + let mut expected = manifest.clone(); + expected.daemon_sqlite = selected.daemon_sqlite.clone(); + if *selected != expected { + return Err(SandboxCause::Setup(ControlError( + "selected existing Project", + ))); + } + } + launched = Some(match self.runtime.launch(launch) { + Ok(owner) => Box::new(owner), + Err(mut failure) => { + progress.phase = match failure.phase { + layerfs_sandbox::provider::LaunchPhase::Validate => SandboxPhase::Validate, + layerfs_sandbox::provider::LaunchPhase::Create => SandboxPhase::Create, + layerfs_sandbox::provider::LaunchPhase::Start => SandboxPhase::Start, + layerfs_sandbox::provider::LaunchPhase::Listener => SandboxPhase::Listener, + }; + launched = failure.instance.take(); + return Err(SandboxCause::Provider(failure)); + } + }); + progress.phase = SandboxPhase::Endpoint; + let endpoint = launched + .as_ref() + .expect("launched") + .endpoint() + .map_err(SandboxCause::Runtime)?; + let connect_host = match &request.launch.options { + ProviderLaunchOptions::Docker { connect_host, .. } => connect_host.as_deref(), + _ => None, + }; + let mut status = progress.connect( + endpoint, + connect_host, + request.launch.startup_timeout, + &request.controller_private, + request.daemon_peer, + request.setup.existing_store.is_some(), + )?; + if request.setup.existing_store.is_none() { + let mut phase = SandboxPhase::Install; + let result = super::connection::install( + progress.control.as_mut().expect("authenticated"), + &mut status, + &mut installed, + &mut phase, + store, + manifest, + ); + progress.phase = phase; + progress.observed = Some(status.clone()); + result?; + } + Ok(status) + })(); + match result { + Ok(startup) => Ok(ProviderSandbox { + request: request.clone(), + runtime: launched.expect("launched"), + startup, + installed, + handshake: progress.handshake, + control: progress.control.expect("authenticated"), + command_identity: CommandIdentity { + uid: request.setup.command_uid, + gid: request.setup.command_gid, + }, + }), + Err(cause) => Err(Box::new(ProviderFailure { + request, + phase: progress.phase, + runtime: launched, + control: progress.control, + handshake: progress.handshake, + observed: progress.observed, + installed, + cause, + })), + } + } +} diff --git a/engine/crates/layerfs-api/tests/provider_ownership.rs b/engine/crates/layerfs-api/tests/provider_ownership.rs new file mode 100644 index 0000000000..3d57397ab7 --- /dev/null +++ b/engine/crates/layerfs-api/tests/provider_ownership.rs @@ -0,0 +1,143 @@ +//! Invalid native topology must refuse before spawning or touching another instance. +#![cfg(target_os = "macos")] +use std::{ + fs, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::{Duration, Instant}, +}; +static NEXT: AtomicUsize = AtomicUsize::new(0); +struct ProjectFixture { + directory: PathBuf, + project: layerfs_api::SealedProject, +} +impl ProjectFixture { + fn new() -> Self { + use layerfs_history::{BranchId, HistoryCatalogConfig, HistoryName, LayerStackId}; + use layerfs_persistence::{PersistenceConfig, SqlitePersistenceProfile}; + let directory = std::env::temp_dir().join(format!( + "sdk-existing-project-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&directory).unwrap(); + let source = directory.join("source"); + fs::create_dir(&source).unwrap(); + fs::write(source.join("payload"), b"original complete project").unwrap(); + let request = layerfs_api::InitRequest { + source, + store: PersistenceConfig::sqlite(directory.join("store.sqlite")) + .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + locator: "/layerfs-store/global/store.sqlite".into(), + policy: layerfs_storage::StoragePolicy::frozen_default(), + catalog: HistoryCatalogConfig { + binding_key: b"sdk-existing-project".to_vec(), + incarnation: 1, + cursor_key: [7; 32], + }, + stack: LayerStackId::from_authority([3; 16]), + stack_name: HistoryName::new("project").unwrap(), + branch: BranchId::from_authority([4; 16]), + branch_name: HistoryName::new("main").unwrap(), + scope_seed: [5; 32], + deadline: Instant::now() + Duration::from_secs(10), + }; + let project = layerfs_telemetry::timer::Timing::disabled("sdk.existing.project", |timer| { + layerfs_api::ProjectApi::new().init(request, timer) + }) + .0 + .unwrap(); + Self { directory, project } + } +} +impl Drop for ProjectFixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +#[test] +fn native_resource_paths_are_validated_before_launch() { + use layerfs_api::{sandbox::ProviderCreate, SandboxApi, SandboxPhase}; + use layerfs_bridge::{ + daemon_setup::{DaemonLimits, DaemonSetup}, + duplex, + }; + use layerfs_sandbox::{ + backend::native::NativeMacosProvider, + provider::{LaunchRequest, ProviderLaunchOptions}, + CommandIdentity, Runtime, + }; + use std::{ + os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, + }; + let fixture = ProjectFixture::new(); + let root = fixture.directory.join("runtime"); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + let provider = NativeMacosProvider::new(&root).unwrap(); + let paths = provider.paths("one").unwrap(); + let metadata = fs::metadata(&root).unwrap(); + let identity = CommandIdentity { + uid: metadata.uid(), + gid: metadata.gid(), + }; + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: [32; 32], + control_peer: duplex::public_key(&[31; 32]).unwrap(), + store: fixture.project.store.path.display().to_string(), + overlay: paths.overlay.display().to_string(), + mounts: paths.mounts.display().to_string(), + command_uid: identity.uid, + command_gid: identity.gid, + existing_store: None, + limits: DaemonLimits { + connections: 2, + handshake_ms: 1000, + cache_bytes: 65536, + namespaces: 2, + pager_kib: 1024, + serial_low_water: 0, + idle_exit_s: 0, + }, + }; + let request = ProviderCreate { + setup, + controller_private: [31; 32], + daemon_peer: duplex::public_key(&[32; 32]).unwrap(), + launch: LaunchRequest { + name: "one".into(), + daemon_executable: "/bin/false".into(), + daemon_config: vec![], + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: identity, + environment: vec![], + }, + }, + }; + let api = SandboxApi::with_runtime(Arc::new(Runtime::MacosNative(provider))); + for field in ["overlay", "mounts", "store", "listen"] { + let mut invalid = request.clone(); + match field { + "overlay" => { + invalid.setup.overlay = root.join("other/overlay.sqlite").display().to_string() + } + "mounts" => invalid.setup.mounts = root.join("other/workspaces").display().to_string(), + "store" => { + invalid.setup.store = paths.directory.join("store.sqlite").display().to_string() + } + _ => invalid.setup.listen = "0.0.0.0:30421".into(), + } + let failure = api + .create_provider_file(invalid, &fixture.project.store, &fixture.project.manifest) + .err() + .unwrap(); + assert_eq!(failure.phase, SandboxPhase::Validate); + assert!(failure.runtime.is_none() && failure.control.is_none()); + assert!(!paths.directory.exists()); + } +} diff --git a/engine/crates/layerfs-api/tests/sandbox_operations.rs b/engine/crates/layerfs-api/tests/sandbox_operations.rs index 64859816e2..507f177de5 100644 --- a/engine/crates/layerfs-api/tests/sandbox_operations.rs +++ b/engine/crates/layerfs-api/tests/sandbox_operations.rs @@ -3,20 +3,26 @@ #[allow(dead_code)] #[path = "../../layerfs-sandbox/tests/support/engine.rs"] mod engine; -use engine::{fixed, frame, Peer}; +use engine::{fixed, frame, upgrade, Peer}; use layerfs_api::{ - ManagedSandbox, SandboxApi, SandboxCause, SandboxCreate, SandboxPhase, WorkspaceApi, + ProviderCreate, ProviderSandbox, SandboxApi, SandboxCause, SandboxPhase, WorkspaceApi, }; use layerfs_bridge::{ control::{Answer, Call, DaemonPhase, DaemonStatus, Reply, Request}, daemon_setup::{DaemonLimits, DaemonSetup}, duplex, }; +use layerfs_sandbox::backend::docker::DockerProvider; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::{ fs, net::TcpListener, path::PathBuf, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, thread::{self, JoinHandle}, time::{Duration, Instant}, }; @@ -52,7 +58,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { struct Fixture { api: SandboxApi, - owner: Option, + owner: Option, peer: Option, daemon: Option>, binary: PathBuf, @@ -124,7 +130,7 @@ impl Fixture { ]; responses.extend(tail); let (docker, peer) = Peer::new(responses, 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let binary = std::env::temp_dir().join(format!( "sdk-operations-{}-{}", std::process::id(), @@ -137,19 +143,23 @@ impl Fixture { manifest.daemon_sqlite = Some("3.51.0".into()); deployment_setup.existing_store = Some(manifest); } - let request = SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + let request = ProviderCreate { + launch: LaunchRequest { + name: "sdk-operations".into(), + daemon_executable: binary.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: binary.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }; // No Store is constructed or opened. This is the original InstallPending // startup used by component proofs, not a fabricated SDK owner. @@ -201,10 +211,8 @@ fn failed_stop_preserves_original_owner_channel_inputs_and_attempt_without_delet let handshake = owner.handshake; let failure = fixture.api.close(owner).unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Stop); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(!runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -227,10 +235,8 @@ fn failed_delete_retains_acknowledged_stop_and_exact_delete_failure() { .close(fixture.owner.take().unwrap()) .unwrap_err(); assert_eq!(failure.phase, SandboxPhase::Delete); - let runtime = failure.sandbox.as_ref().unwrap(); + let runtime = failure.runtime.as_ref().unwrap(); assert_eq!(runtime.identity().to_string(), CID); - assert!(runtime.disposition().stopped); - assert!(!runtime.disposition().deleted); let SandboxCause::Runtime(cause) = &failure.cause else { panic!("original runtime failure") }; @@ -299,39 +305,44 @@ fn workspace_exec_returns_original_runtime_owner_at_the_mount_with_no_daemon_reg const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; let mut fixture = Fixture::new(vec![ fixed(201, format!("{{\"Id\":\"{EID}\"}}").as_bytes()), + upgrade(&frame(1, b"")), + fixed( + 200, + format!( + "{{\"ID\":\"{EID}\",\"ContainerID\":\"{CID}\",\"Running\":false,\"ExitCode\":0,\"Pid\":42}}" + ) + .as_bytes(), + ), fixed(204, b""), fixed(204, b""), ]); let before = layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(); - let created = WorkspaceApi::exec( + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec!["SDK_TEST=1".into()], - false, + &mut stdout, + &mut stderr, ) .unwrap(); - assert_eq!(created.identity().0.to_string(), CID); - assert_eq!(created.identity().1.to_string(), EID); - assert_eq!(created.request().directory, "/workspaces/selected-mount"); - assert_eq!( - created.request().identity, - layerfs_sandbox::CommandIdentity { uid: 501, gid: 20 } - ); + assert_eq!(exit.code, 0); + assert_eq!(exit.pid, Some(42)); assert_eq!( layerfs_api::internal::control(fixture.owner.as_mut().unwrap()).channel_work(), before ); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); let calls = fixture.calls(); - assert_eq!(calls.len(), 9); + assert_eq!(calls.len(), 11); let request = std::str::from_utf8(&calls[6]).unwrap(); assert!(request.starts_with(&format!("POST /v1.54/containers/{CID}/exec "))); assert!(request.contains(r#""WorkingDir":"/workspaces/selected-mount""#)); assert!(request.contains(r#""User":"501:20""#)); - assert!(!calls - .iter() - .any(|call| call.windows(12).any(|bytes| bytes == b"/exec/start "))); + assert!(calls[7].starts_with(format!("POST /v1.54/exec/{EID}/start ").as_bytes())); + assert!(calls[8].starts_with(format!("GET /v1.54/exec/{EID}/json ").as_bytes())); } #[test] fn workspace_exec_failure_keeps_exact_original_request_and_container() { @@ -340,19 +351,22 @@ fn workspace_exec_failure_keeps_exact_original_request_and_container() { fixed(204, b""), fixed(204, b""), ]); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); let failure = WorkspaceApi::exec( fixture.owner.as_ref().unwrap(), &mounted(), vec!["/bin/pwd".into()], vec![], - true, + &mut stdout, + &mut stderr, ) .unwrap_err(); - assert_eq!(failure.container.to_string(), CID); - assert_eq!(failure.request.directory, "/workspaces/selected-mount"); - assert_eq!(failure.request.arguments, ["/bin/pwd"]); - assert!(failure.request.stdin); - assert!(failure.transfer.attempted); + assert_eq!( + failure.requested_container, + Some(layerfs_sandbox::ContainerId::parse(CID).unwrap()) + ); + assert!(failure.attempted); fixture.api.close(fixture.owner.take().unwrap()).unwrap(); assert_eq!(fixture.calls().len(), 9); } @@ -429,25 +443,29 @@ fn create_matching_existing_project_keeps_one_original_hello_and_does_not_reinst fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_effect() { let project = ProjectFixture::new(); let (docker, peer) = Peer::new(vec![], 4096); - let api = SandboxApi::new(docker); + let api = SandboxApi::with_runtime(Arc::new(Runtime::Docker(DockerProvider::new(docker)))); let mut deployment_setup = setup([32; 32], duplex::public_key(&[31; 32]).unwrap()); let mut manifest = project.project.manifest.clone(); manifest.root[0] ^= 1; deployment_setup.existing_store = Some(manifest); let result = api.create( - SandboxCreate { - deployment: layerfs_sandbox::backend::docker::SandboxRequest { - image: IMAGE.into(), - store_volume: "lfs-test-borrowed".into(), - port: 30421, + ProviderCreate { + launch: LaunchRequest { + name: "mismatched-sandbox".into(), + daemon_executable: project.project.store.path.clone(), + daemon_config: vec![], labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "lfs-test-borrowed".into(), + port: 30421, + connect_host: None, + }, }, setup: deployment_setup, - executable: project.project.store.path.clone(), controller_private: [31; 32], daemon_peer: duplex::public_key(&[32; 32]).unwrap(), - startup_wait: Duration::from_secs(2), - connect_host: None, }, &project.project, ); @@ -455,6 +473,6 @@ fn create_mismatched_existing_project_refuses_before_any_deployment_or_control_e panic!("mismatched Project must refuse before effects") }; assert_eq!(failure.phase, SandboxPhase::Validate); - assert!(failure.sandbox.is_none() && failure.control.is_none() && failure.installed.is_none()); + assert!(failure.runtime.is_none() && failure.control.is_none() && failure.installed.is_none()); assert!(peer.finish().is_empty()); } diff --git a/engine/crates/layerfs-node/src/discovery.rs b/engine/crates/layerfs-node/src/discovery.rs index c0e058d87c..95083db61d 100644 --- a/engine/crates/layerfs-node/src/discovery.rs +++ b/engine/crates/layerfs-node/src/discovery.rs @@ -1,4 +1,7 @@ //! Private, permission-authorized project and Sandbox discovery. No mutation replay. +#[cfg(target_os = "linux")] +use crate::runtime::NAMESPACES; +use crate::runtime::{LOCAL, NATIVE}; use crate::{ job::{failed, refused}, registry::{self, Mount, Process, Project, SandboxState}, @@ -9,10 +12,11 @@ use layerfs_bridge::{ provision::StoreManifest, }; use layerfs_persistence::{SealedStore, SqlitePersistenceProfile}; -use layerfs_sandbox::{ - backend::docker::{Docker, Sandbox, SandboxRequest}, - CommandIdentity, ContainerId, -}; +use layerfs_sandbox::backend::docker::{Docker, DockerProvider, SandboxRequest}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::backend::native::{self, LocalProcess}; +use layerfs_sandbox::provider::{LocalRecord, SandboxRecord}; +use layerfs_sandbox::{CommandIdentity, ContainerId, Sandbox, WireFailure}; use std::{ fs::{self, File, OpenOptions}, io::{Read, Write}, @@ -26,6 +30,7 @@ use std::{ }; extern "C" { fn geteuid() -> u32; + fn getegid() -> u32; fn flock(fd: i32, operation: i32) -> i32; } const LOCK_EX: i32 = 2; @@ -45,7 +50,7 @@ pub(crate) fn home(args: &SdkValue) -> Result { directory(&home)?; Ok(home) } -fn private(path: &Path, dir: bool) -> Result<()> { +pub(crate) fn private(path: &Path, dir: bool) -> Result<()> { let meta = fs::symlink_metadata(path).map_err(failed)?; if meta.file_type().is_symlink() || meta.is_dir() != dir @@ -68,6 +73,11 @@ pub(crate) fn directory(path: &Path) -> Result<()> { } private(path, true) } +/// This process's effective uid and gid: a native Sandbox's command +/// identity (its commands run as this user). +pub(crate) fn user() -> (u32, u32) { + unsafe { (geteuid(), getegid()) } +} /// Where a project's Sandboxes live: `LAYERFS_HOME//sandboxes//` /// (#373), numbered from 1 within the project. fn sandboxes(args: &SdkValue) -> Result { @@ -127,6 +137,8 @@ pub(crate) fn project_dir(args: &SdkValue) -> Result { /// Calls take turns on one directory: in this process before its OS lock, /// and between processes on the lock itself. The wait is readiness, bounded, /// and nothing has been attempted before it ends. +// Unmount exchanges run outside this guard; one shared readiness deadline +// covers the in-process turn and the cross-process discovery lock. const IN_PROCESS_WAIT: Duration = Duration::from_secs(60); /// Between two looks at another process's lock. const LOCK_PAUSE: Duration = Duration::from_millis(2); @@ -328,6 +340,20 @@ pub(crate) fn project(args: &SdkValue) -> Result { installed, }) } +/// The labels a record keeps, as the value map its reader expects. +fn label_values(labels: &[(String, String)]) -> SdkValue { + SdkValue::List( + (labels.iter()) + .map(|(key, value)| { + SdkValue::map([ + ("key", SdkValue::text(key)), + ("value", SdkValue::text(value)), + ]) + }) + .collect(), + ) +} + pub(crate) fn save_sandbox( directory: &Path, state: &SandboxState, @@ -339,41 +365,97 @@ pub(crate) fn save_sandbox( let deploy = args .get("deploy") .ok_or_else(|| refused("InvalidInput", "deploy"))?; + let mut fields = vec![ + ("id", SdkValue::text(&state.id)), + ("project", SdkValue::text(&state.project)), + ("name", SdkValue::text(&state.name)), + ( + "container", + SdkValue::text(state.runtime.identity().to_string()), + ), + ]; + match state.runtime.as_ref() { + Sandbox::Docker(runtime) => { + let record = runtime.request(); + fields.extend([ + ("docker", SdkValue::text(registry::text(deploy, "docker")?)), + ("image", SdkValue::text(&record.image)), + ("volume", SdkValue::text(&record.store_volume)), + ("labels", label_values(&record.labels)), + ]); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Sandbox::Native(runtime) => { + let record = runtime + .local_record() + .ok_or_else(|| refused("Failed", "local Sandbox has no daemon process"))?; + fields.extend(local_fields(args, state, &record, NATIVE, LOCAL)?); + } + #[cfg(target_os = "linux")] + Sandbox::Namespaces(runtime) => { + let Some(SandboxRecord::Namespaces(record)) = runtime.record() else { + return Err(refused("Failed", "namespace Sandbox has no init process")); + }; + let image = record.image().as_str().to_owned(); + fields.extend(local_fields( + args, + state, + record.local(), + NAMESPACES, + &image, + )?); + } + } + fields.extend([ + ("secret", SdkValue::Bytes(secret.to_vec())), + ("peer", SdkValue::Bytes(peer.to_vec())), + ("instance", SdkValue::Bytes(instance.to_vec())), + ]); save( &directory.join("sandbox.record"), - SdkValue::map([ - ("id", SdkValue::text(&state.id)), - ("project", SdkValue::text(&state.project)), - ("name", SdkValue::text(&state.name)), - ( - "container", - SdkValue::text(state.runtime.identity().to_string()), - ), - ("docker", SdkValue::text(registry::text(deploy, "docker")?)), - ("image", SdkValue::text(&state.runtime.request().image)), - ( - "volume", - SdkValue::text(&state.runtime.request().store_volume), - ), - ( - "labels", - SdkValue::List( - (state.runtime.request().labels.iter()) - .map(|(key, value)| { - SdkValue::map([ - ("key", SdkValue::text(key)), - ("value", SdkValue::text(value)), - ]) - }) - .collect(), - ), - ), - ("secret", SdkValue::Bytes(secret.to_vec())), - ("peer", SdkValue::Bytes(peer.to_vec())), - ("instance", SdkValue::Bytes(instance.to_vec())), - ]), + SdkValue::Map( + (fields.into_iter()) + .map(|(name, value)| (name.to_owned(), value)) + .collect(), + ), ) } +/// The fields a local Sandbox record keeps: its directory, its daemon or init +/// process identity, its listener port and the image it runs, under `runtime` +/// `native` (whose image is `local`) or `namespaces` (whose image is the +/// digest its root is an overlay of). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_fields( + args: &SdkValue, + state: &SandboxState, + record: &LocalRecord, + runtime: &'static str, + image: &str, +) -> Result> { + let boot = record + .boot + .ok_or_else(|| refused("Failed", "local Sandbox daemon has no boot id"))?; + Ok(vec![ + ("runtime", SdkValue::text(runtime)), + ("uid", SdkValue::Int(i64::from(state.identity.uid))), + ("gid", SdkValue::Int(i64::from(state.identity.gid))), + ( + "directory", + SdkValue::text( + crate::local::directory(args, &state.id)? + .display() + .to_string(), + ), + ), + ("pid", SdkValue::Int(i64::from(record.pid))), + ("start", SdkValue::text(record.start_time.to_string())), + ("boot", SdkValue::text(format!("{boot:032x}"))), + ("port", SdkValue::Int(i64::from(record.port))), + ("image", SdkValue::text(image)), + ("labels", label_values(&record.labels)), + ]) +} + /// A Sandbox whose container no longer runs it: the Engine no longer has the /// container, or has it stopped (it closed when idle, or Docker restarted), /// which lost its daemon and every mount in it. Only its records, and a @@ -406,30 +488,260 @@ pub(crate) fn gone_container(refusal: &SdkRefusal) -> Option<(&str, bool)> { _ => None, } } -/// The Engine, container and deployment a Sandbox record names. -pub(crate) fn runtime(value: &SdkValue) -> Result<(Docker, ContainerId, SandboxRequest)> { +/// The labels a Sandbox record keeps; a record written before #373 has none. +fn labels(value: &SdkValue) -> Result> { + match value.get("labels") { + Some(SdkValue::List(items)) => items + .iter() + .map(|item| { + Ok(( + registry::text(item, "key")?.to_owned(), + registry::text(item, "value")?.to_owned(), + )) + }) + .collect(), + _ => Ok(Vec::new()), + } +} +/// The Engine, container and deployment a Docker Sandbox record names. +fn runtime(value: &SdkValue) -> Result<(DockerProvider, ContainerId, SandboxRecord)> { let docker = Docker::new(registry::text(value, "docker")?, Duration::from_secs(30)).map_err(failed)?; let request = SandboxRequest { image: registry::text(value, "image")?.to_owned(), store_volume: registry::text(value, "volume")?.to_owned(), port: 30421, - // a record written before #373 has no labels, nor has its container - labels: match value.get("labels") { - Some(SdkValue::List(items)) => items - .iter() - .map(|item| { - Ok(( - registry::text(item, "key")?.to_owned(), - registry::text(item, "value")?.to_owned(), - )) - }) - .collect::>()?, - _ => Vec::new(), - }, + labels: labels(value)?, }; let container = ContainerId::parse(registry::text(value, "container")?).map_err(failed)?; - Ok((docker, container, request)) + let provider = DockerProvider::new(docker); + Ok((provider, container, SandboxRecord::Docker(request.into()))) +} +/// What a Sandbox record names: a Docker container, a local daemon process +/// (`runtime` `native`) with its directory and port, or a namespace Sandbox +/// (`runtime` `namespaces`) with its own runtime, record and directory. +pub(crate) enum Recorded { + Docker(DockerProvider, ContainerId, SandboxRecord), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Local { + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + }, + #[cfg(target_os = "linux")] + Namespaces(layerfs_sandbox::Runtime, ContainerId, SandboxRecord), +} +/// Reads Sandbox `id`'s record's runtime. +pub(crate) fn recorded(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + #[cfg(target_os = "linux")] + if value.get_text("runtime") == Some(NAMESPACES) { + return namespaces_record(args, id, value); + } + if matches!(value.get_text("runtime"), Some(LOCAL | NATIVE)) { + return local_record(args, id, value); + } + let (docker, container, request) = runtime(value)?; + Ok(Recorded::Docker(docker, container, request)) +} +/// A local record is used only where this deployment puts that Sandbox: +/// its directory must be exactly `/-` +/// and named so, so a damaged, edited or stale record (another +/// LAYERFS_LOCAL_DIR or prefix) never points a delete elsewhere. A record +/// without `boot` was written before boot ids were kept: its daemon is +/// matched by id and start time as then, and never signalled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, record) = local_record_of(args, id, value)?; + // with no placement (#428), a mount names its Sandbox: a record is used + // only by a native deployment, and only a native record is. A record of + // the `local` runtime (an earlier, unmerged build) is never used, so + // nothing it names is deleted + let native = args.get("deploy").is_some_and(crate::runtime::native); + if !native || value.get_text("runtime") != Some(NATIVE) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a {} Sandbox, but only a native deployment \ + (LAYERFS_SANDBOX=native) uses a native record and none uses another; \ + Sandboxes of other runtimes are never mixed, so the record is neither used \ + nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + value.get_text("runtime").unwrap_or_default(), + registry::text(args, "project")? + ), + )); + } + Ok(Recorded::Local { + directory, + id: ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + process: LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + }, + port: record.port, + labels: record.labels, + }) +} +/// A record's directory and physical facts, checked against where this +/// deployment puts that Sandbox. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn local_record_of(args: &SdkValue, id: &str, value: &SdkValue) -> Result<(PathBuf, LocalRecord)> { + let directory = PathBuf::from(registry::text(value, "directory")?); + let expected = crate::local::directory(args, id)?; + // the Sandbox's own directory name, exactly as this deployment names it + let name = registry::text(value, "name")?; + if directory != expected || expected.file_name().and_then(|n| n.to_str()) != Some(name) { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id}'s local record names {} (as {name}), but this deployment puts it at \ + {}: the record is damaged or from another LAYERFS_LOCAL_DIR or prefix, so it is \ + neither used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + directory.display(), + expected.display(), + registry::text(args, "project")? + ), + )); + } + let number = |key: &str| { + value + .get_int(key) + .ok_or_else(|| refused("InvalidInput", format!("local Sandbox record {key}"))) + }; + let boot = match value.get_text("boot") { + None => None, + Some(hex) => Some( + (hex.len() == 32) + .then(|| u128::from_str_radix(hex, 16).ok()) + .flatten() + .ok_or_else(|| refused("InvalidInput", "local Sandbox record boot"))?, + ), + }; + let record = LocalRecord { + name: name.to_owned(), + pid: u32::try_from(number("pid")?).map_err(failed)?, + start_time: registry::text(value, "start")? + .parse::() + .map_err(failed)?, + boot, + port: u16::try_from(number("port")?).map_err(failed)?, + labels: labels(value)?, + }; + Ok((directory, record)) +} +/// A namespaces record is used only where this backend puts that Sandbox and +/// only by a deployment that selects it: another backend never reopens or +/// deletes what this one made. +#[cfg(target_os = "linux")] +fn namespaces_record(args: &SdkValue, id: &str, value: &SdkValue) -> Result { + let (directory, local) = local_record_of(args, id, value)?; + let image = + layerfs_sandbox::Image::parse(registry::text(value, "image")?).map_err(|error| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox whose record names no image digest \ + ({error}); it cannot be given a root, so it is neither reopened nor deleted" + ), + ) + })?; + let record = layerfs_sandbox::provider::NamespaceRecord { local, image }; + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + if crate::runtime::backend(deploy)? != crate::runtime::Backend::Namespaces { + return Err(refused( + "InvalidInput", + format!( + "Sandbox {id} is a namespaces Sandbox, but this deployment runs the other local \ + backend; Sandboxes of other runtimes are never mixed, so the record is neither \ + used nor deleted. Remove what it left by hand, then delete \ + LAYERFS_HOME/{}/sandboxes/{id}/sandbox.record", + registry::text(args, "project")? + ), + )); + } + let root = (directory.parent()) + .ok_or_else(|| refused("InvalidInput", "namespaces Sandbox directory"))?; + let provider = + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root).map_err(|error| { + refused( + "Unsupported", + format!( + "namespaces Sandbox at {} cannot run on this host: {error}", + directory.display() + ), + ) + })?; + Ok(Recorded::Namespaces( + layerfs_sandbox::Runtime::Namespaces(provider), + ContainerId::parse(registry::text(value, "container")?).map_err(failed)?, + SandboxRecord::Namespaces(record), + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn local_record(_: &SdkValue, id: &str, _: &SdkValue) -> Result { + Err(refused( + "Unsupported", + format!( + "Sandbox {id} is a native Sandbox (LAYERFS_SANDBOX=native), which needs Linux or macOS" + ), + )) +} +impl Recorded { + /// The runtime, observed running once (it starts nothing). + pub(crate) fn reopen(self) -> std::result::Result, Box> { + match self { + Self::Docker(provider, id, record) => provider + .reopen(id, &record) + .map(|s| Box::new(Sandbox::Docker(s))), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => native::Sandbox::reopen(directory, id, process, port, labels) + .map(|s| Box::new(Sandbox::Native(s))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => runtime.reopen(id, &record).map(Box::new), + } + } + /// The runtime found stopped: only its one Delete is admitted. + pub(crate) fn stopped(self) -> Box { + match self { + Self::Docker(provider, id, record) => { + let instance = provider + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"); + Box::new(Sandbox::Docker(instance)) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Local { + directory, + id, + process, + port, + labels, + } => Box::new(Sandbox::Native(native::Sandbox::stopped( + directory, id, process, port, labels, + ))), + #[cfg(target_os = "linux")] + Self::Namespaces(runtime, id, record) => Box::new( + runtime + .reopen_stopped(id, &record) + .map_err(failed) + .expect("validated record"), + ), + } + } } pub(crate) fn reopen(args: &SdkValue) -> Result> { let id = registry::text(args, "sandbox")?; @@ -447,8 +759,16 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { } let project = registry::text(&value, "project")?; let container = registry::text(&value, "container")?; - let (docker, selector, request) = runtime(&value)?; - let runtime = Sandbox::reopen(docker, selector, request).map_err(|failure| { + let recorded = recorded(args, id, &value)?; + // a local daemon listens on this kernel's loopback; no other host reaches it + let host = match recorded { + Recorded::Docker(..) => crate::pool::daemon_host(args), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Recorded::Local { .. } => None, + #[cfg(target_os = "linux")] + Recorded::Namespaces(..) => None, + }; + let runtime = recorded.reopen().map_err(|failure| { if failure.container_gone() || failure.container_stopped() { gone(id, container, failure.container_stopped()) } else { @@ -457,7 +777,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { })?; let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, - host: crate::pool::daemon_host(args), + host, secret: bytes(&value, "secret")?, peer: bytes(&value, "peer")?, instance: bytes(&value, "instance")?, @@ -470,7 +790,7 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { name: value.get_text("name").unwrap_or_default().to_owned(), directory, runtime, - identity: CommandIdentity { uid: 501, gid: 20 }, + identity: recorded_identity(&value), controls: crate::pool::Controls::new(control, dial), mounts: Default::default(), processes: Default::default(), @@ -478,8 +798,23 @@ pub(crate) fn reopen(args: &SdkValue) -> Result> { refresh(&state)?; Ok(state) } +/// The command identity a Sandbox record names: a native Sandbox's is the +/// user who started it (`uid`, `gid`); every other runs as 501:20. +fn recorded_identity(value: &SdkValue) -> CommandIdentity { + let id = |key: &str| value.get_int(key).and_then(|n| u32::try_from(n).ok()); + match (id("uid"), id("gid")) { + (Some(uid), Some(gid)) => CommandIdentity { uid, gid }, + _ => CommandIdentity { uid: 501, gid: 20 }, + } +} /// Refresh shared executor/mount metadata without repeating an operation. pub(crate) fn refresh(state: &SandboxState) -> Result<()> { + refresh_except(state, &[]) +} +/// Explicit Sandbox close may dispose retained unmounts without asking their +/// daemon for a Ready receipt that no longer exists. Other records still +/// undergo the ordinary refresh and MountsActive admission. +pub(crate) fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { let mut mounts = std::collections::BTreeMap::new(); let mut processes = std::collections::BTreeMap::new(); for item in fs::read_dir(&state.directory).map_err(failed)? { @@ -490,6 +825,9 @@ pub(crate) fn refresh(state: &SandboxState) -> Result<()> { continue; }; let path = registry::text(&value, "path")?; + if retained.iter().any(|kept| kept == path) { + continue; + } let token = layerfs_bridge::control::WorkspaceToken { workspace: layerfs_history::WorkspaceId::from_slice(&bytes::<32>( &value, diff --git a/engine/crates/layerfs-node/src/exec.rs b/engine/crates/layerfs-node/src/exec.rs index 80a3120b2a..3982d74648 100644 --- a/engine/crates/layerfs-node/src/exec.rs +++ b/engine/crates/layerfs-node/src/exec.rs @@ -5,46 +5,38 @@ //! daemon is not involved. A background command is started in its own session //! with its output in a log file, and this binding keeps its process id. The //! Engine offers no per-exec cancel, so `cancel` signals the process group -//! from a second command and waits (bounded) until it is gone. -use crate::job::{failed, refused, Outcome}; +//! from a second command and waits (bounded) until it is gone. A local +//! Sandbox (`layerfs_sandbox::Sandbox::Native`) runs the same commands as +//! processes of this kernel, so the same probes and signals apply. +use crate::job::{refused, Outcome}; use crate::output::Spool; use crate::registry::{lock, naming, sandbox, sandbox_live, text, Process, SandboxState}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use std::time::Duration; -/// Home of the command identity, which has no account in the image. -const HOME: &str = "HOME=/tmp"; /// Checks that a cancelled process group is gone. const GONE_CHECKS: u32 = 50; /// Pause between those checks. const GONE_PAUSE: Duration = Duration::from_millis(100); /// One command to its end: exit code, stdout, stderr. `spool` sends every -/// non-empty stream to its file instead of keeping it inline. +/// non-empty stream to its file instead of keeping it inline. Its home is +/// the Sandbox's scratch directory (`/tmp` in a container), since the +/// command identity has no account there. fn run( state: &SandboxState, directory: &str, arguments: Vec, spool: bool, ) -> Result<(i32, Spool, Spool), SdkRefusal> { - let created = layerfs_api::internal::exec( + crate::runtime::run( &state.runtime, state.identity, arguments, - vec![HOME.to_owned()], - directory.to_owned(), - false, + vec![format!("HOME={}", crate::runtime::scratch(&state.runtime))], + directory, + spool, ) - .map_err(failed)?; - let attached = created.start().map_err(failed)?; - let (_input, output, handle) = attached.into_parts().map_err(failed)?; - let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); - output.copy_to(&mut stdout, &mut stderr).map_err(failed)?; - let inspected = handle.inspect().map_err(failed)?; - let code = inspected - .known_root_exit() - .ok_or_else(|| refused("Unknown", "the command's exit code is not known"))?; - Ok((code, stdout, stderr)) } /// `workspace.exec(ws, command, { background?, spool? })`; with @@ -91,11 +83,17 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { state.id, layerfs_bridge::sdk_values::hex(&crate::registry::random()?[..12]) ); - let script = format!("setsid bash -c \"$0\" >/tmp/layerfs-{id}.log 2>&1 \"$1\" 2>&1 Outcome { "background process exited before its identity was observed", ) })?; + // its own session leaves the command's process group; named, it still + // ends with a local Sandbox. Unnamed it would outlive it: it ends here, once. + if let Err(refusal) = crate::runtime::track(&state.runtime, pid, &start) { + let end = format!("kill -KILL -- -{pid} 2>/dev/null; true"); + let _ = run( + &state, + &directory, + vec!["bash".into(), "-c".into(), end], + false, + ); + return Err(refusal); + } let process = Process { path: path.to_owned(), command: command.to_owned(), @@ -136,7 +146,7 @@ pub(crate) fn exec(args: &SdkValue) -> Outcome { /// The container's first process is the daemon, which does not reap orphans, /// so a finished member can stay a zombie that `kill -0` still finds. fn start_time(state: &SandboxState, pid: u32) -> Result, SdkRefusal> { - let (code, out, _) = run(state, "/tmp", vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; + let (code, out, _) = run(state, &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), format!("s=$(cat /proc/{pid}/stat 2>/dev/null) || exit 1; set -- ${{s##*) }}; shift 19; printf '%s' \"$1\"")], false)?; if code == 0 { Ok(Some(String::from_utf8_lossy(out.kept()).into_owned())) } else { @@ -151,7 +161,7 @@ fn alive(state: &SandboxState, pid: u32) -> Result { ); let (code, _, _) = run( state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec!["bash".into(), "-c".into(), probe], false, )?; @@ -283,7 +293,7 @@ pub(crate) fn cancel(args: &SdkValue) -> Outcome { } run( &state, - "/tmp", + &crate::runtime::scratch(&state.runtime), vec![ "bash".into(), "-c".into(), diff --git a/engine/crates/layerfs-node/src/lib.rs b/engine/crates/layerfs-node/src/lib.rs index 298d5020af..f7f89ae788 100644 --- a/engine/crates/layerfs-node/src/lib.rs +++ b/engine/crates/layerfs-node/src/lib.rs @@ -5,12 +5,16 @@ mod discovery; mod dispatch; mod exec; mod job; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local; mod output; mod pool; mod project; mod registry; +mod runtime; mod sandbox; mod trace; +mod unmount; mod workspace; use job::Job; diff --git a/engine/crates/layerfs-node/src/local.rs b/engine/crates/layerfs-node/src/local.rs new file mode 100644 index 0000000000..1556ed7efa --- /dev/null +++ b/engine/crates/layerfs-node/src/local.rs @@ -0,0 +1,440 @@ +//! Native Sandboxes (`deploy.sandbox` `native`, LAYERFS_SANDBOX=native): +//! the project's daemon runs as a process of this user instead of in a +//! container. It isolates nothing beyond the caller's own sandbox. +//! +//! Each Sandbox gets a directory `/-` +//! with what a container held at fixed paths: its setup +//! (`/layerfs-local/config`), overlay template (`/layerfs-local/overlay`) and +//! Workspace mounts (`/workspaces`). The daemon opens the installed Store +//! where this process opens it (`storePath`), so the daemon setup and the +//! Store's locator name that path instead of `/layerfs-store`. The daemon +//! listens on 127.0.0.1 only. macOS serves loopback NFS as this user; Linux +//! keeps the root FUSE daemon and nonroot commands (501:20) from #402. Start, +//! failure and reclaim follow the Docker route: one attempt, a creation intent while the outcome is open, and a failed start's +//! daemon stopped and its directory deleted, each once. +use crate::job::{failed, refused}; +use crate::registry::{lock, text, SandboxState, SANDBOXES}; +use layerfs_api::{sandbox::ProviderCreate, SandboxApi}; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::provision::StoreManifest; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::{CommandIdentity, Runtime, Sandbox}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +/// Starts the project's next Sandbox as a native daemon. Linux requires +/// root before any effect. Called by +/// [`crate::sandbox::start`] under the project's guard, with the project +/// and its installed manifest already read. +pub(crate) fn start( + args: &SdkValue, + directory: &Path, + project: (&crate::registry::Project, StoreManifest), +) -> Result, SdkRefusal> { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + // Inputs are read first, so a refused request leaves no creation intent. + if cfg!(target_os = "linux") && crate::discovery::user().0 != 0 { + return Err(refused( + "Unsupported", + "LAYERFS_SANDBOX=native on Linux needs root for FUSE and commands as uid 501, gid 20; run inside the VM or CI box that isolates it", + )); + } + let root = deploy.get_text("localDir").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.localDir is not configured (LAYERFS_LOCAL_DIR); a native Sandbox needs it", + ) + })?; + let backend = crate::runtime::backend(deploy)?; + let runtime = native_runtime(root, backend) + .map_err(|error| refused("InvalidInput", format!("LAYERFS_LOCAL_DIR {root}: {error}")))?; + let executable = PathBuf::from(text(deploy, "daemon")?); + let store = match args.get_text("storePath") { + Some(path) if Path::new(path).is_absolute() => path.to_owned(), + _ => { + return Err(refused( + "InvalidInput", + "a local Sandbox's daemon opens the client's storePath (LAYERFS_STORE_PATH), an absolute path", + )) + } + }; + crate::sandbox::unresolved( + directory, + &format!("a directory under {root} named for it, and its daemon process"), + )?; + let (id, saved_directory) = crate::discovery::claim_next(args)?; + let made = start_claimed( + args, + directory, + &saved_directory, + &id, + project, + (runtime, backend, executable, store), + ); + if made.is_err() { + let _ = std::fs::remove_dir(&saved_directory); + } + made +} + +/// Starts the claimed Sandbox `id`: directory and daemon on the installed +/// Store, its first control connection, then its records. +fn start_claimed( + args: &SdkValue, + directory: &Path, + saved_directory: &Path, + id: &str, + (project, installed): (&crate::registry::Project, StoreManifest), + (runtime, backend, executable, store): (Runtime, crate::runtime::Backend, PathBuf, String), +) -> Result, SdkRefusal> { + let name = text(args, "project")?; + let sandbox_name = sandbox_name(args, id)?; + let paths = runtime.native_paths(&sandbox_name).map_err(failed)?; + let path_text = |path: &Path| { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| refused("InvalidInput", "LAYERFS_LOCAL_DIR must be UTF-8")) + }; + let controller = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let daemon = layerfs_bridge::duplex::generate_keypair().map_err(failed)?; + let mut setup = crate::sandbox::setup(daemon.private, controller.public); + // macOS uses the caller; Linux keeps #402's separate nonroot identity. + #[cfg(target_os = "macos")] + { + (setup.command_uid, setup.command_gid) = crate::discovery::user(); + } + setup.listen = "127.0.0.1:0".into(); + setup.store = store.clone(); + setup.overlay = path_text(&paths.overlay)?; + setup.mounts = path_text(&paths.mounts)?; + let mut existing = project.manifest.clone(); + existing.daemon_sqlite = installed.daemon_sqlite; + // the Store's path in the daemon's filesystem, which is this one + existing.locator = store; + setup.existing_store = Some(existing); + let identity = CommandIdentity { + uid: setup.command_uid, + gid: setup.command_gid, + }; + setup.encode().map_err(failed)?; + // Read only if this process ends before the intent is resolved. + crate::discovery::save( + &directory.join("create.intent"), + SdkValue::text(format!("creation of Sandbox {id} began and never finished")), + )?; + let labels = crate::sandbox::labels(name, id); + let api = SandboxApi::with_runtime(Arc::new(runtime)); + let options = launch_options(backend, identity, &setup.store, args)?; + let mut manifest = project.manifest.clone(); + manifest.locator = setup.store.clone(); + let created = api + .create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: sandbox_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), + labels, + startup_timeout: crate::sandbox::STARTUP, + options, + }, + setup, + controller_private: controller.private, + daemon_peer: daemon.public, + }, + &project.store, + &manifest, + ) + .map_err(|mut failure| { + drop(failure.control.take()); + abandon( + directory, + id, + format!( + "native sandbox stopped at {:?}: {:?}", + failure.phase, failure.cause + ), + failure.runtime.take(), + ) + })?; + let instance = created.startup.instance; + let (sandbox, control, identity) = layerfs_api::internal::split_provider(created); + let endpoint = match sandbox.endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + drop(control); + return Err(abandon( + directory, + id, + format!("native endpoint after startup: {error:?}"), + Some(sandbox), + )); + } + }; + let dial = crate::pool::Dial { + endpoint, + host: None, + secret: controller.private, + peer: daemon.public, + instance, + }; + let state = SandboxState { + id: id.to_owned(), + project: name.to_owned(), + name: sandbox_name, + directory: saved_directory.to_owned(), + runtime: sandbox, + identity, + controls: crate::pool::Controls::new(control, dial), + mounts: Default::default(), + processes: Default::default(), + }; + let recorded = (|| { + crate::discovery::save_sandbox( + &state.directory, + &state, + args, + controller.private, + daemon.public, + instance, + )?; + crate::discovery::save_text( + &state.directory.join("container"), + &state.runtime.identity().to_string(), + )?; + std::fs::remove_file(directory.join("create.intent")).map_err(failed) + })(); + if let Err(refusal) = recorded { + // The daemon runs and is this start's own: it is stopped and its + // directory deleted like any failed start's, and the records written + // so far are removed, so its number is free again. + let SandboxState { runtime, .. } = state; + for record in ["sandbox.record", "container"] { + let _ = std::fs::remove_file(saved_directory.join(record)); + } + return Err(abandon( + directory, + id, + format!("local sandbox records: {}", refusal.message), + Some(runtime), + )); + } + let state = Arc::new(state); + lock(&SANDBOXES)?.insert(state.directory.clone(), state.clone()); + Ok(state) +} + +/// The native runtime this host offers, over `root`: macOS loopback NFS as +/// this user, or Linux FUSE with a root daemon. The other platform's +/// provider, and another OS, is refused before root access. +#[cfg(target_os = "macos")] +fn native_runtime( + root: &str, + _: crate::runtime::Backend, +) -> Result { + layerfs_sandbox::backend::native::NativeMacosProvider::new(root).map(Runtime::MacosNative) +} + +#[cfg(target_os = "linux")] +fn native_runtime( + root: &str, + backend: crate::runtime::Backend, +) -> Result { + match backend { + crate::runtime::Backend::Native => { + layerfs_sandbox::backend::native::NativeLinuxProvider::new(root) + .map(Runtime::LinuxNative) + } + crate::runtime::Backend::Namespaces => { + layerfs_sandbox::backend::namespaces::NamespaceProvider::new(root) + .map(Runtime::Namespaces) + } + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn native_runtime( + _: &str, + _: crate::runtime::Backend, +) -> Result { + Err(layerfs_sandbox::RuntimeError::Unsupported( + "native Sandbox runtime platform", + )) +} + +/// How the daemon of `backend` is launched: the same local options, and for +/// the namespace backend the Store file its private root binds and the image +/// archive its root is unpacked from. +#[cfg(target_os = "linux")] +fn launch_options( + backend: crate::runtime::Backend, + identity: CommandIdentity, + store: &str, + args: &SdkValue, +) -> Result { + Ok(match backend { + crate::runtime::Backend::Native => ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }, + crate::runtime::Backend::Namespaces => { + let deploy = args + .get("deploy") + .ok_or_else(|| refused("InvalidInput", "missing `deploy`"))?; + let (image, archive) = crate::runtime::namespaces_image(deploy)?; + ProviderLaunchOptions::Namespaces { + command_identity: identity, + environment: native_environment(), + store: PathBuf::from(store), + image, + archive, + } + } + }) +} + +#[cfg(not(target_os = "linux"))] +fn launch_options( + _: crate::runtime::Backend, + identity: CommandIdentity, + _: &str, + _: &SdkValue, +) -> Result { + Ok(ProviderLaunchOptions::Local { + command_identity: identity, + environment: native_environment(), + }) +} + +/// What a native Sandbox's daemon starts with besides `PATH`: the NFS +/// route, and the mount helper and reference overrides this process was +/// given (`LAYERFS_NFS_*`), so a test host can name its own client. +#[cfg(target_os = "macos")] +fn native_environment() -> Vec<(String, String)> { + let mut environment = vec![("LAYERFS_KERNEL".to_owned(), "nfs".to_owned())]; + for name in [ + "LAYERFS_NFS_MOUNT", + "LAYERFS_NFS_UNMOUNT", + "LAYERFS_NFS_FORCE", + "LAYERFS_NFS_FLUSH", + "LAYERFS_NFS_REFRESH", + ] { + if let Ok(value) = std::env::var(name) { + environment.push((name.to_owned(), value)); + } + } + environment +} + +#[cfg(target_os = "linux")] +fn native_environment() -> Vec<(String, String)> { + vec![("LAYERFS_KERNEL".to_owned(), "fuse".to_owned())] +} + +/// A local Sandbox's directory name: `-`, the name its +/// container would have. +fn sandbox_name(args: &SdkValue, id: &str) -> Result { + let prefix = args + .get("deploy") + .and_then(|d| d.get_text("containerPrefix")) + .filter(|prefix| !prefix.is_empty()) + .unwrap_or(crate::sandbox::PREFIX); + Ok(format!("{prefix}{}-{id}", text(args, "project")?)) +} + +/// Where this deployment puts Sandbox `id`: +/// `/-`, computed from the +/// configuration alone, as a record's directory must be. +pub(crate) fn directory(args: &SdkValue, id: &str) -> Result { + let root = (args.get("deploy")) + .and_then(|deploy| deploy.get_text("localDir")) + .map(Path::new) + .ok_or_else(|| { + refused( + "InvalidInput", + format!( + "Sandbox {id} is a local Sandbox; its record is used only with \ + LAYERFS_LOCAL_DIR (deploy.localDir) configured" + ), + ) + })?; + let name = sandbox_name(args, id)?; + let plain = |path: &Path| { + !path + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + }; + let single = matches!( + Path::new(&name).components().collect::>()[..], + [Component::Normal(_)] + ); + if !root.is_absolute() || !plain(root) || !single { + return Err(refused( + "InvalidInput", + format!("local Sandbox directory {}/{name}", root.display()), + )); + } + Ok(root.join(name)) +} + +/// The refusal for a failed start. Everything a local start makes is its +/// own (a directory and a child process), so each effect is known: the +/// daemon's last log lines are read once, then it is stopped and its +/// directory deleted, each once, and the intent removed. If that cleanup +/// fails, the intent stays and says what is left. +fn abandon( + directory: &Path, + id: &str, + mut cause: String, + sandbox: Option>, +) -> SdkRefusal { + let container = sandbox.as_ref().map(|s| s.identity().to_string()); + let mut log = None; + let mut left = None; + if let Some(mut sandbox) = sandbox { + match sandbox.log_tail(crate::sandbox::LOG_LINES, crate::sandbox::LOG_BYTES) { + Ok(bytes) => { + let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); + cause = format!("{cause}; its daemon's last log lines:\n{text}"); + log = Some(text); + } + Err(error) => cause = format!("{cause}; its daemon log was unreadable: {error:?}"), + } + let removed = sandbox.stop(1).and_then(|()| sandbox.delete()); + left = removed.err().map(|error| { + format!( + "its local Sandbox {} could not be removed: {error:?}", + sandbox.identity() + ) + }); + } + let intent = directory.join("create.intent"); + let (message, removed) = match left { + None => match std::fs::remove_file(&intent) { + Ok(()) => (cause, container.is_some()), + Err(error) => ( + format!("{cause}; its creation intent stays: {error}"), + container.is_some(), + ), + }, + Some(left) => { + let state = format!("creation of Sandbox {id} failed ({cause}); {left}"); + let kept = crate::discovery::save(&intent, SdkValue::text(&state)) + .err() + .map(|error| format!(" (recording it failed: {})", error.message)) + .unwrap_or_default(); + (format!("{state}; its creation intent stays{kept}"), false) + } + }; + SdkRefusal { + code: "Failed".to_owned(), + message, + detail: SdkValue::map([ + ("container", SdkValue::opt_text(container)), + ("removed", SdkValue::Bool(removed)), + ("log", SdkValue::opt_text(log)), + ]), + } +} diff --git a/engine/crates/layerfs-node/src/registry.rs b/engine/crates/layerfs-node/src/registry.rs index aeeb25bc47..8bbbc2081c 100644 --- a/engine/crates/layerfs-node/src/registry.rs +++ b/engine/crates/layerfs-node/src/registry.rs @@ -2,7 +2,8 @@ //! project itself is only its home entry (`//`), so a project //! removed there can be initialized again without a restart. //! -//! A Sandbox is one container and its daemon. Its runtime runs commands +//! A Sandbox is one container and its daemon (or, with the local runtime, +//! one daemon process: `runtime`). Its runtime runs commands //! directly; its control connections (`pool`) each carry one exchange at a //! time, and concurrent calls take separate connections. The Sandbox runtime owner (this //! binding) also keeps the background processes it started; the daemon holds @@ -11,7 +12,7 @@ use crate::job::refused; use layerfs_api::MountedWorkspace; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::{backend::docker::Sandbox, CommandIdentity}; +use layerfs_sandbox::CommandIdentity; use std::collections::BTreeMap; use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; @@ -42,10 +43,11 @@ pub(crate) struct Process { pub(crate) struct SandboxState { pub(crate) id: String, pub(crate) project: String, - /// Its container's name, `-`. + /// Its container's name, `-` (a local Sandbox's + /// directory name). pub(crate) name: String, pub(crate) directory: std::path::PathBuf, - pub(crate) runtime: Sandbox, + pub(crate) runtime: Box, pub(crate) identity: CommandIdentity, pub(crate) controls: crate::pool::Controls, pub(crate) mounts: Mutex>, @@ -76,7 +78,7 @@ pub(crate) fn text<'a>(args: &'a SdkValue, name: &str) -> Result<&'a str, SdkRef /// tables as they stand: no daemon call (#352 F-029). A call that names a /// mount checks just that mount (`SandboxState::token`, `directory`); one /// that needs every mount or process another process may have changed takes -/// [`sandbox_refreshed`]. +/// `discovery::refresh`. pub(crate) fn sandbox(args: &SdkValue) -> Result, SdkRefusal> { let directory = crate::discovery::sandbox_dir(args, text(args, "sandbox")?)?; let existing = lock(&SANDBOXES)?.get(&directory).cloned(); @@ -129,14 +131,6 @@ pub(crate) fn naming(args: &SdkValue, id: &str) -> SdkValue { SdkValue::Map(fields) } -/// The Sandbox an operation names, its tables read again from the shared -/// records and the daemon (one status call per mount). -pub(crate) fn sandbox_refreshed(args: &SdkValue) -> Result, SdkRefusal> { - let state = sandbox(args)?; - crate::discovery::refresh(&state)?; - Ok(state) -} - impl SandboxState { /// The token of the Workspace mounted at `path`. pub(crate) fn token(&self, path: &str) -> Result { @@ -154,6 +148,7 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { + crate::unmount::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/engine/crates/layerfs-node/src/runtime.rs b/engine/crates/layerfs-node/src/runtime.rs new file mode 100644 index 0000000000..a4757f26ab --- /dev/null +++ b/engine/crates/layerfs-node/src/runtime.rs @@ -0,0 +1,156 @@ +//! A Sandbox's runtime: Docker (default), or `LAYERFS_SANDBOX=native` on +//! Linux (root FUSE daemon, nonroot commands) and macOS (loopback NFS, +//! caller identity). Both use the same authenticated daemon control pipeline +//! and the same concrete `layerfs_sandbox::Sandbox` handle. +use crate::job::{failed, refused}; +use crate::output::Spool; +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::provider::SandboxRecord; +#[cfg(target_os = "linux")] +use layerfs_sandbox::Image; +use layerfs_sandbox::{CommandIdentity, Sandbox}; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// The image a native Sandbox records and reports, and the `runtime` of a +/// record from the former `local` runtime, which is never used (a Docker +/// record has no `runtime`). +pub(crate) const LOCAL: &str = "local"; +/// A native Sandbox record's `runtime` (LAYERFS_SANDBOX=native): a local +/// daemon serving FUSE on Linux or loopback NFS on macOS. +pub(crate) const NATIVE: &str = "native"; +/// A namespace Sandbox record's `runtime` (LAYERFS_SANDBOX_BACKEND=namespaces, +/// Linux): a local daemon in its own mount, PID and IPC namespaces and cgroup. +pub(crate) const NAMESPACES: &str = "namespaces"; + +/// Which local backend a native deployment runs inside its Sandbox directory: +/// the plain native one, or the Linux namespace one. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Backend { + Native, + Namespaces, +} + +/// The local backend the deployment selects: `deploy.sandboxBackend` +/// (LAYERFS_SANDBOX_BACKEND), `native` when unset. The namespace backend is +/// Linux only; anywhere else selecting it is refused before any effect. +pub(crate) fn backend(deploy: &SdkValue) -> Result { + match deploy.get_text("sandboxBackend") { + None | Some(NATIVE) => Ok(Backend::Native), + Some(NAMESPACES) if cfg!(target_os = "linux") => Ok(Backend::Namespaces), + Some(NAMESPACES) => Err(refused( + "Unsupported", + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) namespaces is Linux only; use native", + )), + Some(other) => Err(refused( + "InvalidInput", + format!( + "deploy.sandboxBackend (LAYERFS_SANDBOX_BACKEND) is native or namespaces, not {other}" + ), + )), + } +} + +/// Whether the deployment selects a local daemon: `deploy.sandbox` unset or +/// `docker` is Docker, `native` is a local daemon; anything else (the +/// former `local` included) is refused before any effect. +pub(crate) fn local(deploy: &SdkValue) -> Result { + match deploy.get_text("sandbox") { + None | Some("docker") => Ok(false), + Some(NATIVE) => Ok(true), + Some(other) => Err(refused( + "InvalidInput", + format!("deploy.sandbox (LAYERFS_SANDBOX) is docker or native (Linux or macOS), not {other}"), + )), + } +} + +/// Whether the deployment selects the native route: FUSE on Linux, +/// loopback NFS on macOS, with no Docker. +pub(crate) fn native(deploy: &SdkValue) -> bool { + deploy.get_text("sandbox") == Some(NATIVE) +} + +/// The image a Sandbox reports: its container's, the digest a namespaces +/// Sandbox's root is an overlay of, or `local` for a native one. +pub(crate) fn image(runtime: &Sandbox) -> String { + match runtime.record() { + Some(SandboxRecord::Docker(docker)) => docker.image, + #[cfg(target_os = "linux")] + Some(SandboxRecord::Namespaces(record)) => record.image().as_str().to_owned(), + _ => LOCAL.to_owned(), + } +} + +/// The image a namespaces deployment's Sandbox root is an overlay of: +/// `deploy.image` (LAYERFS_IMAGE), the same `sha256:<64 hex>` digest the +/// Docker backend uses, and `deploy.imageArchive` (LAYERFS_IMAGE_ARCHIVE), +/// the docker-archive in the bundle that holds it. Both are read before any +/// effect, so a bundle without its image refuses the start. +#[cfg(target_os = "linux")] +pub(crate) fn namespaces_image(deploy: &SdkValue) -> Result<(Image, PathBuf), SdkRefusal> { + let digest = deploy.get_text("image").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.image is not configured (LAYERFS_IMAGE); a namespaces Sandbox's root is the image that digest names", + ) + })?; + let image = Image::parse(digest) + .map_err(|error| refused("InvalidInput", format!("deploy.image: {error}")))?; + let archive = deploy.get_text("imageArchive").ok_or_else(|| { + refused( + "InvalidInput", + "deploy.imageArchive is not configured (LAYERFS_IMAGE_ARCHIVE); a namespaces Sandbox needs the docker-archive the bundle carries", + ) + })?; + let archive = PathBuf::from(archive); + if !archive.is_absolute() { + return Err(refused( + "InvalidInput", + "deploy.imageArchive (LAYERFS_IMAGE_ARCHIVE) must be an absolute path", + )); + } + Ok((image, archive)) +} + +/// Where its commands keep scratch files: `/tmp` in a container, the +/// Sandbox's own `tmp` directory for a native one (deleted with it), so a +/// native Sandbox leaves nothing in this machine's `/tmp`. +pub(crate) fn scratch(runtime: &Sandbox) -> String { + runtime.scratch_directory() +} + +/// Names a background process a command started in a session of its own +/// (`pid`, started at `start`), so it ends with a native Sandbox as it ends +/// with a container. A container's teardown already owns its descendants. +pub(crate) fn track(runtime: &Sandbox, pid: u32, start: &str) -> Result<(), SdkRefusal> { + if matches!(runtime, Sandbox::Docker(_)) { + return Ok(()); + } + let start = start.parse::().map_err(failed)?; + runtime.track(pid, start).map_err(failed) +} + +/// One command to its end: exit code, stdout, stderr. `spool` sends every +/// non-empty stream to its file instead of keeping it inline. +pub(crate) fn run( + runtime: &Sandbox, + identity: CommandIdentity, + arguments: Vec, + environment: Vec, + directory: &str, + spool: bool, +) -> Result<(i32, Spool, Spool), SdkRefusal> { + let (mut stdout, mut stderr) = (Spool::new(spool), Spool::new(spool)); + let request = layerfs_sandbox::ExecRequest { + identity, + arguments, + environment, + directory: directory.to_owned(), + stdin: false, + }; + let exit = runtime + .run(&request, &mut stdout, &mut stderr) + .map_err(failed)?; + Ok((exit.code, stdout, stderr)) +} diff --git a/engine/crates/layerfs-node/src/sandbox.rs b/engine/crates/layerfs-node/src/sandbox.rs index edf23889a2..45e164f3b3 100644 --- a/engine/crates/layerfs-node/src/sandbox.rs +++ b/engine/crates/layerfs-node/src/sandbox.rs @@ -14,15 +14,15 @@ //! the caller's behalf. use crate::job::{failed, refused, Outcome}; use crate::registry::{lock, text, SandboxState, SANDBOXES}; -use layerfs_api::internal::split; -use layerfs_api::sandbox::values; -use layerfs_api::{ - OperationFailure, SandboxApi, SandboxCause, SandboxCreate, SandboxFailure, SandboxPhase, -}; +use layerfs_api::internal::split_provider; +use layerfs_api::sandbox::{values, ProviderCreate, ProviderFailure}; +use layerfs_api::{OperationFailure, SandboxApi, SandboxCause, SandboxPhase}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_bridge::daemon_setup::{DaemonLimits, DaemonSetup}; use layerfs_bridge::provision::StoreManifest; -use layerfs_sandbox::backend::docker::{CreatePhase, Docker, SandboxRequest}; +use layerfs_sandbox::backend::docker::{CreatePhase, Docker, DockerProvider}; +use layerfs_sandbox::provider::{LaunchRequest, ProviderLaunchOptions}; +use layerfs_sandbox::Runtime; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -30,12 +30,12 @@ use std::time::Duration; /// Port the daemon listens on inside its container. const PORT: u16 = 30421; /// Wait for the daemon's listener after Start. -const STARTUP: Duration = Duration::from_secs(30); +pub(crate) const STARTUP: Duration = Duration::from_secs(30); /// Wait for one Engine control response. const ENGINE: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. -const LOG_LINES: u16 = 20; -const LOG_BYTES: usize = 2048; +pub(crate) const LOG_LINES: u16 = 20; +pub(crate) const LOG_BYTES: usize = 2048; pub(crate) fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: u32 = 16; /// for this process and the rest for others (#352 F-017). pub(crate) const DAEMON_CONNECTIONS: u16 = 8; -fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { +pub(crate) fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { DaemonSetup { listen: format!("0.0.0.0:{PORT}"), private_key: private, @@ -84,7 +84,7 @@ fn setup(private: [u8; 32], peer: [u8; 32]) -> DaemonSetup { } /// A container name prefix when the deployment sets none. -const PREFIX: &str = "layerfs-"; +pub(crate) const PREFIX: &str = "layerfs-"; /// The container labels of Sandbox `id` of `project`: what it is, whose, /// and which. Saved in its record, so a reopen expects the same. @@ -115,6 +115,15 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result state, - _ => "its record is unreadable".to_owned(), - }; - return Err(refused( - "Unknown", - format!( - "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ - is unknown, so it is neither retried nor taken as done. Remove any container it \ - left (docker ps -a --filter volume={store_volume}), then delete {}", - intent.display() - ), - )); - } + unresolved( + directory, + &format!("docker ps -a --filter volume={store_volume}"), + )?; // The number is claimed before anything is made; a failed start gives it // back (its directory holds no record yet), so the number is used again. let (id, saved_directory) = crate::discovery::claim_next(args)?; @@ -174,6 +170,29 @@ pub(crate) fn start(args: &SdkValue, directory: &Path) -> Result Result<(), SdkRefusal> { + let intent = directory.join("create.intent"); + if intent.exists() { + let state = match crate::discovery::load(&intent) { + Ok(SdkValue::Text(state)) => state, + _ => "its record is unreadable".to_owned(), + }; + return Err(refused( + "Unknown", + format!( + "a prior Sandbox creation has an unresolved discovery intent: {state}. Its outcome \ + is unknown, so it is neither retried nor taken as done. Remove any container it \ + left ({left}), then delete {}", + intent.display() + ), + )); + } + Ok(()) +} + /// Starts the claimed Sandbox `id`: container, daemon on the installed /// Store, then its records. fn start_claimed( @@ -204,25 +223,30 @@ fn start_claimed( .map_err(failed)? .naming(&container_name) .map_err(|_| refused("InvalidInput", format!("container name {container_name}")))?; + let provider = Arc::new(Runtime::Docker(DockerProvider::new(docker))); // Read only if this process ends before the intent is resolved. crate::discovery::save( &directory.join("create.intent"), SdkValue::text(format!("creation of Sandbox {id} began and never finished")), )?; - let created = SandboxApi::new(docker).create_file( - SandboxCreate { - deployment: SandboxRequest { - image, - store_volume, - port: PORT, + let created = SandboxApi::with_runtime(provider).create_provider_file( + ProviderCreate { + launch: LaunchRequest { + name: container_name.clone(), + daemon_executable: executable, + daemon_config: Vec::new(), labels: labels(name, id), + startup_timeout: STARTUP, + options: ProviderLaunchOptions::Docker { + image, + store_volume, + port: PORT, + connect_host: crate::pool::daemon_host(args), + }, }, setup: daemon_setup, - executable, controller_private: controller.private, daemon_peer: daemon.public, - startup_wait: STARTUP, - connect_host: crate::pool::daemon_host(args), }, &project.store, &project.manifest, @@ -230,7 +254,7 @@ fn start_claimed( let owner = created.map_err(|failure| abandon(directory, id, failure))?; let container = owner.runtime.identity().to_string(); let instance = owner.startup.instance; - let (runtime, control, identity) = split(owner); + let (runtime, control, identity) = split_provider(owner); let dial = crate::pool::Dial { endpoint: runtime.endpoint().map_err(failed)?, host: crate::pool::daemon_host(args), @@ -267,33 +291,46 @@ fn start_claimed( /// Whether a failed create's every effect is known: nothing reached the /// Store volume, and any container it made is acknowledged by identity. -fn settled(failure: &SandboxFailure) -> bool { +fn settled(failure: &ProviderFailure) -> bool { // A Create the Engine refused (409: the container name is taken) made no container. - let unknown_container = matches!( - &failure.cause, - SandboxCause::Create(create) if create.phase == CreatePhase::Create - && !matches!(create.transfer.cause, layerfs_sandbox::RuntimeError::Http(409)) - ); + let unknown_container = match &failure.cause { + SandboxCause::Provider(launch) => match &launch.cause { + layerfs_sandbox::RuntimeError::Docker(docker_failure) => { + match docker_failure.as_ref() { + layerfs_sandbox::backend::docker::ProviderFailure::Create(create) => { + create.phase == CreatePhase::Create + && !matches!( + create.transfer.cause, + layerfs_sandbox::RuntimeError::Http(409) + ) + } + _ => false, + } + } + _ => false, + }, + _ => false, + }; failure.phase != SandboxPhase::Install && failure.installed.is_none() && !unknown_container } /// The refusal for a failed create. A settled one reads its container's log /// tail once, removes the container (stop, then delete, each once) and its /// intent; otherwise the intent stays, rewritten to say what is left. -fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { +fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkRefusal { let mut cause = format!( "sandbox create stopped at {:?}: {:?}", failure.phase, failure.cause ); let container = failure - .sandbox + .runtime .as_ref() .map(|sandbox| sandbox.identity().to_string()); drop(failure.control.take()); let mut log = None; let left = if !settled(&failure) { Some("its effect on the Store volume or the Engine is unknown".to_owned()) - } else if let Some(sandbox) = failure.sandbox.as_mut() { + } else if let Some(sandbox) = failure.runtime.as_mut() { match sandbox.log_tail(LOG_LINES, LOG_BYTES) { Ok(bytes) => { let text = String::from_utf8_lossy(&bytes).trim_end().to_owned(); @@ -344,7 +381,8 @@ fn abandon(directory: &Path, id: &str, mut failure: Box) -> SdkR /// `sandbox.create({ image })`: under the project's guard, reclaims the /// project's idle-closed Sandboxes, then starts the lowest free number on /// the call's image (`deploy.image`) and waits for its daemon: -/// `{ sandbox, name, image }`. The only call that starts a Sandbox. Failures +/// `{ sandbox, name, image }`. With `deploy.sandbox` `native` (macOS) it +/// starts a daemon process instead of a container; its `image` is `local`. The only call that starts a Sandbox. Failures /// are `Failed` or `Unknown`, never retried. pub(crate) fn create(args: &SdkValue) -> Outcome { let project = crate::discovery::project_dir(args)?; @@ -356,7 +394,7 @@ pub(crate) fn create(args: &SdkValue) -> Outcome { ("name", SdkValue::text(&state.name)), ( "image", - SdkValue::text(state.runtime.request().image.as_str()), + SdkValue::text(crate::runtime::image(&state.runtime)), ), ])) } @@ -381,8 +419,16 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { } let state = crate::registry::sandbox_live(args)?; let _guard = crate::discovery::guard(&state.directory)?; - crate::discovery::refresh(&state)?; - let mounted = lock(&state.mounts)?.keys().cloned().collect::>(); + let recoverable = crate::unmount::check_all(&state)?; + crate::discovery::refresh_except(&state, &recoverable)?; + // Explicit close can take over an orphaned/retained unmount, but must + // still refuse every unrelated live Workspace. Stop detaches before + // killing the daemon; Delete independently checks physical detach. + let mounted = lock(&state.mounts)? + .keys() + .filter(|path| !recoverable.contains(path)) + .cloned() + .collect::>(); if !mounted.is_empty() { return Err(SdkRefusal { code: "MountsActive".to_owned(), @@ -397,8 +443,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { }); } let value = crate::discovery::load(&record)?; - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let mut owner = layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request) + let mut owner = crate::discovery::recorded(args, &id, &value)? + .reopen() .map_err(failed)?; owner.stop(STOP_S).map_err(failed)?; owner.delete().map_err(failed)?; @@ -413,7 +459,8 @@ pub(crate) fn close(args: &SdkValue) -> Outcome { /// `backend.daemon.sandbox.get_log({ sandbox, lines })`: the last `lines` /// lines (1..=2000) of Sandbox `sandbox`'s daemon log, at most -/// [`GET_LOG_BYTES`], read once through the Engine: `{ sandbox, container, +/// [`GET_LOG_BYTES`], read once through the Engine (a local Sandbox's from +/// its log file): `{ sandbox, container, /// text, bytes }`. A stopped container's log is readable until its records /// are reclaimed. Refused `NoSuchSandbox` with no record or no container. /// Read-only: it starts, stops and reclaims nothing. @@ -437,8 +484,7 @@ pub(crate) fn get_log(args: &SdkValue) -> Outcome { crate::discovery::load(&record)? }; let container = text(&value, "container")?.to_owned(); - let (docker, selector, request) = crate::discovery::runtime(&value)?; - let sandbox = layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request); + let sandbox = crate::discovery::recorded(args, &id, &value)?.stopped(); let bytes = match sandbox.log_tail(lines, GET_LOG_BYTES) { Ok(bytes) => bytes, Err(failure) if failure.container_gone() => { @@ -460,7 +506,7 @@ const GET_LOG_BYTES: usize = 64 * 1024; /// Seconds a closing Sandbox's daemon has to exit before the Engine kills it. const STOP_S: u16 = 5; -/// Reclaims Sandbox `id`, whose container a reopen found gone or stopped +/// Reclaims Sandbox `id`, whose container (or local daemon) a reopen found gone or stopped /// (`container`, `stopped`): one unforced Delete of a stopped container, then /// its records, so its number is free again. Called under the project's /// guard; the Sandbox's own guard is taken too. The record is read again @@ -485,8 +531,9 @@ pub(crate) fn reclaim(args: &SdkValue, id: &str, container: &str, stopped: bool) )); } if stopped { - let (docker, selector, request) = crate::discovery::runtime(&value)?; - match layerfs_sandbox::backend::docker::Sandbox::stopped(docker, selector, request).delete() + match crate::discovery::recorded(args, id, &value)? + .stopped() + .delete() { Err(failure) if !failure.container_gone() => return Err(failed(failure)), _ => {} diff --git a/engine/crates/layerfs-node/src/unmount.rs b/engine/crates/layerfs-node/src/unmount.rs new file mode 100644 index 0000000000..71ca6dba05 --- /dev/null +++ b/engine/crates/layerfs-node/src/unmount.rs @@ -0,0 +1,45 @@ +//! Workspace-scoped unmount ownership slot; implemented by the native unmount lease. +use crate::{job::Outcome, registry::SandboxState}; +use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; + +pub(crate) struct Attempt; + +pub(crate) fn check(_state: &SandboxState, _path: &str) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn find(_state: &SandboxState, _path: &str) -> Result, SdkRefusal> { + Ok(None) +} + +pub(crate) fn check_all(_state: &SandboxState) -> Result, SdkRefusal> { + Ok(Vec::new()) +} + +pub(crate) fn begin( + _state: &SandboxState, + _token: WorkspaceToken, + _path: &str, +) -> Result { + Ok(Attempt) +} + +pub(crate) fn finish( + _attempt: &Attempt, + _path: &str, + _outcome: &Outcome, +) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn cancel(_attempt: &Attempt) -> Result<(), SdkRefusal> { + Ok(()) +} + +pub(crate) fn cleanup_failure(outcome: Outcome, _error: SdkRefusal) -> Outcome { + outcome +} + +pub(crate) fn repeat(value: &SdkValue) -> Outcome { + Ok(value.clone()) +} diff --git a/engine/crates/layerfs-node/src/workspace.rs b/engine/crates/layerfs-node/src/workspace.rs index 829df67f71..8dcb0c4ca9 100644 --- a/engine/crates/layerfs-node/src/workspace.rs +++ b/engine/crates/layerfs-node/src/workspace.rs @@ -8,9 +8,7 @@ //! by a mount and reclaimed by the next `sandbox.create` or //! `runtime.sandbox.reclaim`. use crate::job::{failed, Outcome}; -use crate::registry::{ - lock, naming, random, sandbox, sandbox_live, sandbox_refreshed, text, Mount, -}; +use crate::registry::{lock, naming, random, sandbox, sandbox_live, text, Mount}; use crate::sandbox::{operation, WORKSPACES}; use layerfs_api::internal::workspace; use layerfs_api::sandbox::{values, Unmounted}; @@ -96,8 +94,10 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { if let SdkValue::Map(fields) = &mut value { fields.push(("sandbox".to_owned(), SdkValue::text(state.id.as_str()))); fields.push(("container".to_owned(), SdkValue::text(state.name.as_str()))); - let image = state.runtime.request().image.as_str(); - fields.push(("image".to_owned(), SdkValue::text(image))); + fields.push(( + "image".to_owned(), + SdkValue::text(crate::runtime::image(&state.runtime)), + )); } crate::discovery::save_workspace(&state, &mounted, branch)?; lock(&state.mounts)?.insert( @@ -112,15 +112,21 @@ pub(crate) fn mount(args: &SdkValue) -> Outcome { /// One recorded Sandbox of the project: running (`Ok(true)`), or found /// gone or stopped (`Ok(false)`, with its container and whether it stopped). -/// Its container is inspected once; no daemon is dialed. -fn observed(record: &SdkValue, directory: &Path) -> Result, SdkRefusal> { +/// Its container (or local daemon process) is inspected once; no daemon +/// is dialed. +fn observed( + args: &SdkValue, + id: &str, + record: &SdkValue, + directory: &Path, +) -> Result, SdkRefusal> { let cached = lock(&crate::registry::SANDBOXES)?.get(directory).cloned(); let observed = match cached { Some(state) => state.runtime.endpoint().map(drop), - None => { - let (docker, selector, request) = crate::discovery::runtime(record)?; - layerfs_sandbox::backend::docker::Sandbox::reopen(docker, selector, request).map(drop) - } + // a container, or with the local runtime a daemon process + None => crate::discovery::recorded(args, id, record)? + .reopen() + .map(drop), }; match observed { Ok(()) => Ok(None), @@ -152,7 +158,7 @@ fn records(args: &SdkValue) -> Result Result<(), SdkRefusal> { for (id, directory, value) in records(args)? { - if let Some((container, stopped)) = observed(&value, &directory)? { + if let Some((container, stopped)) = observed(args, &id, &value, &directory)? { crate::sandbox::reclaim(args, &id, &container, stopped)?; } } @@ -168,7 +174,7 @@ pub(crate) fn reclaim_gone(args: &SdkValue) -> Result<(), SdkRefusal> { pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { let mut items = Vec::new(); for (id, directory, value) in records(args)? { - if observed(&value, &directory)?.is_some() { + if observed(args, &id, &value, &directory)?.is_some() { continue; } items.push(SdkValue::map([ @@ -198,7 +204,7 @@ pub(crate) fn sandboxes(args: &SdkValue) -> Outcome { pub(crate) fn reclaim(args: &SdkValue) -> Outcome { let mut reclaimed = Vec::new(); for (id, directory, value) in records(args)? { - let Some((container, stopped)) = observed(&value, &directory)? else { + let Some((container, stopped)) = observed(args, &id, &value, &directory)? else { continue; }; let _placing = crate::discovery::guard(&crate::discovery::project_dir(args)?)?; @@ -252,10 +258,19 @@ pub(crate) fn commit(args: &SdkValue) -> Outcome { /// `workspace.unmount(ws, { discard?, force? })`: `Busy` while a background /// process this binding started still runs there, then the daemon's answer. pub(crate) fn unmount(args: &SdkValue) -> Outcome { - let state = sandbox_refreshed(args)?; - let _guard = crate::discovery::guard(&state.directory)?; + let state = sandbox(args)?; let path = text(args, "path")?; + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } + let guard = crate::discovery::guard(&state.directory)?; + // Another process may have registered while this call waited for the + // short admission section. Check again before sending anything. + if let Some(value) = crate::unmount::find(&state, path)? { + return crate::unmount::repeat(&value); + } let token = state.token(path)?; + crate::discovery::refresh(&state)?; // The runtime owner knows its own processes; the daemon's kernel probe // still answers Busy for any other holder. let running = crate::exec::running_in(&state, path)?; @@ -273,22 +288,59 @@ pub(crate) fn unmount(args: &SdkValue) -> Outcome { )]), }); } - let value = { + let attempt = crate::unmount::begin(&state, token, path)?; + drop(guard); + // Only this Workspace's lease remains held. Other Workspaces and + // processes can take Sandbox ownership while the daemon helper waits. + let mut sent = false; + let outcome = (|| { let mut control = crate::trace::control(&state.controls)?; + sent = true; let mut api = workspace(&mut control); // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. if args.get_bool("force") { match api.unmount(token, true).map_err(operation)? { - Some(outcome) => values::force_unmounted(&outcome), - None => values::unmounted(&Unmounted::Closed)?, + Some(outcome) => Ok(values::force_unmounted(&outcome)), + None => values::unmounted(&Unmounted::Closed), } } else { - let discard = args.get_bool("discard"); - values::unmounted(&api.unmount_checked(token, discard).map_err(operation)?)? + match api.unmount_checked(token, args.get_bool("discard")) { + Ok(answer) => values::unmounted(&answer), + Err(error) => Err(operation(error)), + } } + })(); + let outcome = match outcome { + Err(error) if error.code == "Missing" => { + Ok(SdkValue::map([("kind", SdkValue::text("Unmounted"))])) + } + other => other, }; - std::fs::remove_file(crate::discovery::workspace_file(&state, token)).map_err(failed)?; - lock(&state.mounts)?.remove(path); - Ok(value) + // Preserve the daemon's known outcome before any discovery-lock wait. + // A file/cleanup failure is separate from an already successful detach. + let publication = if !sent { + crate::unmount::cancel(&attempt) + } else { + crate::unmount::finish(&attempt, path, &outcome) + }; + if let Err(error) = publication { + return crate::unmount::cleanup_failure(outcome, error); + } + if outcome.is_ok() { + let cleanup = (|| { + let _guard = crate::discovery::guard(&state.directory)?; + match std::fs::remove_file(crate::discovery::workspace_file(&state, token)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(failed(error)), + } + lock(&state.mounts)?.remove(path); + Ok(()) + })(); + if let Err(error) = cleanup { + return crate::unmount::cleanup_failure(outcome, error); + } + } + outcome } diff --git a/engine/crates/layerfs-node/tests/native_discovery.rs b/engine/crates/layerfs-node/tests/native_discovery.rs index f0afde037f..8498ae9424 100644 --- a/engine/crates/layerfs-node/tests/native_discovery.rs +++ b/engine/crates/layerfs-node/tests/native_discovery.rs @@ -32,11 +32,19 @@ fn another_404_is_a_failure_that_removes_nothing() { let home = Home::new(); let engine = engine(&home.0, 404, r#"{"message":"page not found"}"#); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(404\)/); + assert.match(answer.refused.message, /page not found/); + "#, None, Duration::from_secs(5), ); - engine.join().unwrap(); + assert!(engine + .join() + .unwrap() + .starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } @@ -113,11 +121,21 @@ fn a_stopped_container_the_engine_will_not_delete_keeps_its_records() { ], ); home.node( - "assert.equal((await reclaim()).refused.code, 'Failed');", + r#" + const answer = await reclaim(); + assert.equal(answer.refused.code, 'Failed'); + assert.match(answer.refused.message, /status: Some\(409\)/); + assert.match(answer.refused.message, /cannot remove container: container is running/); + "#, None, Duration::from_secs(5), ); - assert_eq!(engine.join().unwrap().len(), 2); + let requests = engine.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with(&format!("GET /v1.54/containers/{CID}/json HTTP/1.1\r\n"))); + assert!(requests[1].starts_with(&format!( + "DELETE /v1.54/containers/{CID}?force=false&v=false HTTP/1.1\r\n" + ))); assert!(home.0.join("proj/sandboxes/1/sandbox.record").exists()); } diff --git a/engine/crates/layerfs-node/tests/support/engine.rs b/engine/crates/layerfs-node/tests/support/engine.rs index 6fb656c2d4..302c2f8e38 100644 --- a/engine/crates/layerfs-node/tests/support/engine.rs +++ b/engine/crates/layerfs-node/tests/support/engine.rs @@ -1,5 +1,5 @@ use super::*; -/// The Engine at `home/docker.sock`, answering one request with `status` and `body`. +/// The Engine at `home/docker.sock`, answering one business request with `status` and `body`. pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { let engine = engines(home, vec![(status, body.to_owned())]); thread::spawn(move || engine.join().unwrap().remove(0)) @@ -7,6 +7,7 @@ pub fn engine(home: &Path, status: u16, body: &str) -> JoinHandle { /// The Engine at `home/docker.sock`, answering each request, one connection /// each, with the next `(status, body)`; it returns their request heads. +/// Version probes get API 1.54 without consuming a response or calling `before`. pub fn engines(home: &Path, responses: Vec<(u16, String)>) -> JoinHandle> { engines_before(home, responses, || {}) } @@ -22,11 +23,9 @@ pub fn engines_before( let mut before = Some(before); let deadline = Instant::now() + Duration::from_secs(3); let mut requests = Vec::new(); - for (status, body) in responses { - let response = format!( - "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", - body.len() - ); + let mut responses = responses.into_iter().peekable(); + while responses.peek().is_some() { + assert!(Instant::now() < deadline, "bounded engine request deadline"); let stream = loop { match listener.accept() { Ok((stream, _)) => break stream, @@ -43,6 +42,9 @@ pub fn engines_before( stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); let mut reader = BufReader::new(stream.try_clone().unwrap()); let mut request = String::new(); loop { @@ -52,11 +54,23 @@ pub fn engines_before( } request.push_str(&line); } - if let Some(before) = before.take() { - before(); - } + let probe = request.split_whitespace().nth(1) == Some("/version"); + let (status, body) = if probe { + (200, r#"{"ApiVersion":"1.54"}"#.to_owned()) + } else { + if let Some(before) = before.take() { + before(); + } + responses.next().unwrap() + }; + let response = format!( + "HTTP/1.1 {status} test\r\nConnection: close\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); (&stream).write_all(response.as_bytes()).unwrap(); - requests.push(request); + if !probe { + requests.push(request); + } } requests }) diff --git a/engine/crates/layerfs-sandbox/Cargo.toml b/engine/crates/layerfs-sandbox/Cargo.toml index e75b3fd4ce..a1f1141f86 100644 --- a/engine/crates/layerfs-sandbox/Cargo.toml +++ b/engine/crates/layerfs-sandbox/Cargo.toml @@ -7,4 +7,13 @@ rust-version.workspace = true license.workspace = true [dependencies] -nix = { version = "=0.31.3", features = ["poll", "socket", "fs"] } +nix = { version = "=0.31.3", features = [ + "poll", + "socket", + "fs", + "signal", + "mount", + "user", + "sched", + "process", +] } diff --git a/engine/crates/layerfs-sandbox/PROVIDERS.md b/engine/crates/layerfs-sandbox/PROVIDERS.md new file mode 100644 index 0000000000..85c740797c --- /dev/null +++ b/engine/crates/layerfs-sandbox/PROVIDERS.md @@ -0,0 +1,134 @@ +# Sandbox runtimes + +Status: Current general guide. + +`backend::runtime` holds the concrete runtime choice and the launched owner. +There are no provider or instance traits: each backend is an ordinary concrete +type and the only dispatch is one `match` per method. + +- `Runtime` is one deployment's selection and is kept for every Sandbox it + starts: `Docker(DockerProvider)` (the default), `MacosNative(NativeMacosProvider)`, + `LinuxNative(NativeLinuxProvider)`, `Namespaces(NamespaceProvider)` (Linux + only) and `Windows`, a placeholder with no backend. + `Runtime::{capabilities, local_directory, native_paths, launch, reopen, + reopen_stopped}` dispatch to them. +- `Sandbox` is what `Runtime::launch`/`reopen` return: `Docker(docker::Sandbox)`, + `Native(native::Sandbox)` or `Namespaces(Box)`, with + `identity`, `endpoint`, `run`, `track`, `log_tail`, `inspect`, `stop`, + `delete`, `scratch_directory`, `local_record` and `record`. + +`layerfs_sandbox::{Runtime, Sandbox}` re-export both. `provider` keeps the +inputs and observations they share: `LaunchRequest`, `ProviderLaunchOptions`, +`LocalRecord`, `DockerRecord`, `SandboxRecord`, `ExecExit`, +`SandboxInspection`, `ProviderCapabilities` and the `LaunchPhase`/`LaunchFailure` +pair. The runtime layer has no daemon control dependency. `backend::HelperSpec` +is the namespace helper's line format; it is compiled and tested on every +platform, while only the helper that reads it is Linux only. + +## Docker + +`Runtime::Docker(DockerProvider::new(docker))` adapts the existing Engine client +and `SandboxRequest`. It shares its lifecycle admission, API 1.48–1.54 +negotiation cache, streamed private archive, logs, exact deployment inspection, +and explicit stop/delete. Call `runtime.launch(LaunchRequest { .. })` with the +executable, private configuration, name, explicit startup wait, and +`ProviderLaunchOptions::Docker { image, store_volume, port, connect_host }`; +Docker validates the immutable image digest, existing volume name, port, +connect_host format and labels before any Engine effect. The runtime does not +decode daemon configuration or perform authentication. + +`Sandbox::run` streams stdout/stderr to caller sinks with bounded windows and +returns `ExecExit` only after one inspection knows the root process exit. It +rejects `stdin=true` before effects: interactive users keep the existing +concrete `create_exec/start/into_parts` stream API. EOF alone is not an exit and +no automatic poll follows an unknown exit. `track` needs no host registration +because container teardown owns descendants; scratch is `/tmp` and +`local_record()` is absent. `inspect` checks the recorded deployment but neither +adopts it nor changes lifecycle admission; unknown PID/exit/storage fields stay +absent. + +## Native + +`Runtime::MacosNative(NativeMacosProvider::new(root))` runs the daemon as this +user and serves loopback NFS; `Runtime::LinuxNative(NativeLinuxProvider::new(root))` +runs a root daemon serving FUSE with ordinary commands as a nonroot identity. +Each refuses the wrong platform before root access, and Linux refuses a non-root +caller the same way. `backend::native` owns the one lifecycle both use; only the +platform facts differ: + +- `backend::macos`: `darwin.rs` (`proc_pidinfo`, `sysctl`, the `mount` table and + `umount -f`) and `mounts.rs` (force detach, deepest first). +- `backend::linux`: `process.rs` (`/proc//stat`, the boot id and the + process table) and `mounts.rs` (`/proc/self/mountinfo`, lazy `MNT_DETACH`). + +The provider validates the command identity, the environment it is given and +the daemon executable, then starts one daemon in a private directory with the +platform's kernel marker (`LAYERFS_KERNEL=fuse` on Linux, `nfs` on macOS); it +rejects conflicting overrides. Launch preserves the original single-attempt +process lifecycle and partial owner on failure. `inspect` reports exact process +liveness; unknown exit/storage facts stay absent. Records remain the Node +binding's responsibility; runtimes neither scan directories nor infer project +ownership. + +## Namespaces (Linux) + +`Runtime::Namespaces(NamespaceProvider::new(root))` is the same local root and +daemon control as `LinuxNative`, with each Sandbox's daemon in its own mount, +PID and IPC namespaces and its own cgroup v2 directory. The VM that holds this +kernel is the boundary (one per Store); the namespaces keep Sandboxes on one +Store out of each other. + +- `backend::namespaces::create` makes the Sandbox's private directory, copies + the daemon to `/bin/layerfs-daemon` (0755), writes the setup (0600) + and a `HelperSpec`, makes `/sys/fs/cgroup/layerfs/`, then spawns the + helper's init stage. +- `backend::namespaces::sandbox` owns the launched handle: the init process's + identity, `cgroup.kill` for stop (a `SIGKILL` to everything the Sandbox + started, `populated 0` awaited), cgroup and directory removal on delete, and + reopen by the recorded process plus its cgroup membership. +- `backend::namespaces::exec` runs one command through the helper's exec stage. +- `src/bin/layerfs-sandbox-helper` performs the calls that act on the calling + process, so the product crate stays `unsafe`-free: the init stage creates the + namespaces, joins the cgroup and starts the inner stage (PID 1), which mounts + the private root (the VM's userland read-only, a private `/proc`, `/dev`, + `/dev/pts`, `/dev/shm`, `/tmp`, plus the Sandbox's directory and the Store's + directory at their own paths), pivots, starts the daemon and reaps; the exec + stage enters the namespaces, sets no-new-privs and runs the command as the + requested uid:gid (its own exit is a signal, never a command's status). + +Preconditions are checked before any effect and refused with the existing +`RuntimeError::Unsupported`, never replaced by a weaker Sandbox: root, the +unified cgroup v2 hierarchy (and `cgroup.kill`), `/dev/fuse`, and +`user.max_user_namespaces=0` — without the last one a command could unshare a +user namespace and hold its own copy of the Workspace mount. The runtime never +sets that limit itself. There is no image: the private root binds the VM's own +userland, so the Sandbox runs the daemon and libraries the VM already has. +`ProviderLaunchOptions::Namespaces { command_identity, environment, store }` +names the Store file whose directory is bound in; `SandboxRecord::Namespaces` +keeps the same physical facts as `LocalRecord`. + +## Windows + +`Runtime::Windows` is a placeholder: `capabilities`, `local_directory`, +`native_paths`, `launch`, `reopen` and `reopen_stopped` all refuse with the +existing `RuntimeError::Unsupported` before any effect, and no `Sandbox` variant +exists for it. No Windows host runtime is implemented; see +[`src/backend/windows/README.md`](src/backend/windows/README.md) for the planned +routes. + +## Failure and records + +A `LaunchFailure` retains the acknowledged `Sandbox`, when any. Its cause may be +`RuntimeError::Docker(Box)`: match the typed variant for +the original Create/Upload/Start/Listener receipt, partial IDs, upload work and +pending source bytes. Upload is within existing `LaunchPhase::Create`; inspect +the typed cause to distinguish it. A partially decoded Create ID is never +adopted, and retaining a handle permits an explicit later observation, never a +replay of creation or Start. + +`Runtime::reopen(id, &SandboxRecord)` validates the exact existing deployment or +process and endpoint without starting it; `reopen_stopped` requires the caller's +prior exact stopped observation and admits only non-forcing Delete. Both accept +`SandboxRecord`, which represents `LocalRecord`, `Namespaces(LocalRecord)` or +`DockerRecord`; each runtime refuses another runtime's record before effects. `Sandbox::record()` exposes the persisted deployment facts. No disk +format changes occur: a record without a `runtime` field is Docker's. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs index 0b1ceaa8a6..88ba0e4eb1 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/container.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/container.rs @@ -1,11 +1,11 @@ //! One-attempt create/start/stop/delete, with no implicit adoption or volume deletion. use super::{ container_types::{CreatePhase, Sandbox, SandboxCreateFailure, SandboxRequest}, - json::{once, Json}, - strings::quoted, topology::SECCOMP, Docker, }; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; use crate::{ContainerId, RuntimeError, WireFailure}; use std::io::{self, Read, Write}; impl SandboxRequest { diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs index 0d207f1049..16ddfe2e62 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/endpoint.rs @@ -1,8 +1,6 @@ //! Structurally observed exact container and loopback-published daemon endpoint. -use super::{ - json::{once, Json}, - Sandbox, -}; +use super::Sandbox; +use crate::backend::json::{once, Json}; use crate::{ContainerId, RuntimeError, WireFailure}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; impl Sandbox { @@ -61,6 +59,20 @@ impl Sandbox { if !self.started || self.stop_attempted || self.delete_attempted { return Err(self.invalid("running endpoint admission")); } + let (_, endpoint) = self.observe(true)?; + endpoint.ok_or_else(|| self.invalid("missing daemon endpoint")) + } + /// One exact deployment/state observation; absent optional facts remain absent. + pub fn inspect(&self) -> Result> { + self.observe(false).map(|(inspection, _)| inspection) + } + fn observe( + &self, + require_running: bool, + ) -> Result<(crate::provider::SandboxInspection, Option), Box> { + if self.deleted { + return Err(self.invalid("deleted container observation")); + } let mut r = self .docker .exchange( @@ -80,6 +92,8 @@ impl Sandbox { let mut topology = true; let mut running = None; let mut endpoint = None; + let mut exit_code = None; + let mut process_id = None; let mut j = Json::new(&mut r.body); j.object(|j, key| { if key.equals("Id") { @@ -108,6 +122,20 @@ impl Sandbox { if key.equals("Running") { once(&mut bits, 1)?; running = Some(j.boolean()?); + } else if key.equals("ExitCode") { + once(&mut bits, 16)?; + if !j.null()? { + exit_code = + Some(i32::try_from(j.integer()?).map_err(|_| { + RuntimeError::Protocol("container exit width") + })?); + } + } else if key.equals("Pid") { + once(&mut bits, 32)?; + process_id = Some( + u32::try_from(j.integer()?) + .map_err(|_| RuntimeError::Protocol("container PID width"))?, + ); } else if key.equals("Paused") || key.equals("Restarting") || key.equals("Dead") @@ -126,7 +154,7 @@ impl Sandbox { } Ok(()) })?; - topology &= bits == 15; + topology &= bits & 15 == 15; } else if key.equals("NetworkSettings") { once(&mut fields, 4)?; let mut bits = 0; @@ -137,6 +165,9 @@ impl Sandbox { j.object(|j, key| { if key.equals(&format!("{}/tcp", self.request.port)) { once(&mut selected, 1)?; + if j.null()? { + return Ok(()); + } let mut count = 0; j.array(|j| { count += 1; @@ -213,27 +244,36 @@ impl Sandbox { Ok(()) })?; j.finish()?; - // the exact recorded container, intact but not running: its daemon - // and every mount in it are gone (Docker restarted, for instance) - if observed == Some(self.id) - && fields == 127 - && image - && topology - && running == Some(false) - { - return Err(RuntimeError::Protocol(crate::types::STOPPED)); - } if observed != Some(self.id) || fields != 127 || !image || !topology - || running != Some(true) + || running.is_none() { return Err(RuntimeError::Protocol( "exact running container observation", )); } - endpoint.ok_or(RuntimeError::Protocol("missing daemon endpoint")) + if require_running && running == Some(false) { + return Err(RuntimeError::Protocol(crate::types::STOPPED)); + } + if running == Some(true) && endpoint.is_none() { + return Err(RuntimeError::Protocol("missing daemon endpoint")); + } + Ok(( + crate::provider::SandboxInspection { + identity: self.id, + running: running.expect("required"), + exit_code: if running == Some(false) { + exit_code + } else { + None + }, + process_id: process_id.filter(|pid| *pid != 0), + scratch_bytes: None, + }, + endpoint, + )) })(); result.map_err(|e| { let mut f = self.selected(r.fail(e, None)); diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs index f4d5e88b4a..1a81fc2baf 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/inspect.rs @@ -1,8 +1,6 @@ //! One exact runtime observation, structurally decoded without retaining Arguments. -use super::{ - json::{once, Json}, - Docker, -}; +use super::Docker; +use crate::backend::json::{once, Json}; use crate::{ContainerId, ExecId, ExecInspection, RuntimeError, WireFailure}; impl Docker { /// Reads one coherent runtime snapshot. PID is diagnostic, never a signal target. diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs index e0eedb87b1..e02b4a03b2 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/mod.rs @@ -7,13 +7,14 @@ mod endpoint; mod headers; mod http; mod inspect; -mod json; mod listener; mod mux; +mod provider; +mod provider_exec; +mod provider_failure; mod request; mod socket; mod streams; -mod strings; mod topology; mod version; pub use archive::{UploadFailure, UploadWork}; @@ -22,6 +23,9 @@ pub use container_types::{ }; pub use http::Docker; pub use listener::{ListenerFailure, ListenerObservation}; +pub use provider::{DockerInstance, DockerProvider}; +pub(crate) use provider_exec::run as run_command; +pub use provider_failure::ProviderFailure; pub use request::{ExecCreateFailure, ExecCreated, ExecRequest}; pub use streams::{ Attached, ExecHandle, InputCloseFailure, InputEnd, OutputFailure, OutputProgress, RuntimeInput, diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs new file mode 100644 index 0000000000..d4171e4002 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider.rs @@ -0,0 +1,124 @@ +//! Docker's existing one-attempt runtime as a concrete backend. +use super::{Docker, Sandbox, SandboxRequest}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::fs::File; + +/// Negotiation and all transfers use the supplied client's original policy and version cache. +#[derive(Clone, Debug)] +pub struct DockerProvider { + docker: Docker, +} +/// The original container owner, including its exact attempted/acknowledged lifecycle state. +pub type DockerInstance = Sandbox; +impl DockerProvider { + /// Negotiation and all transfers use the supplied client's original policy and version cache. + pub fn new(docker: Docker) -> Self { + Self { docker } + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "docker", + isolation: IsolationModel::Container, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let mut phase = LaunchPhase::Validate; + let mut instance = None; + let result = (|| { + let ProviderLaunchOptions::Docker { + image, + store_volume, + port, + connect_host, + } = request.options + else { + return Err(RuntimeError::Unsupported( + "Docker requires Docker launch options", + )); + }; + if let Some(host) = &connect_host { + if host.is_empty() || host.contains(['\0', ' ', '\t', '\r', '\n']) { + return Err(RuntimeError::Protocol("Docker connect_host format")); + } + } + let deployment = SandboxRequest { + image, + store_volume, + port, + labels: request.labels, + }; + deployment.check()?; + if request.startup_timeout.is_zero() + || request.daemon_config.is_empty() + || request.daemon_config.len() > 8192 + { + return Err(RuntimeError::Protocol("Docker startup wait/private config")); + } + let docker = self.docker.clone().naming(&request.name)?; + // Opening the executable precedes any Engine effect, as in the original API route. + let mut file = File::open(request.daemon_executable)?; + phase = LaunchPhase::Create; + instance = + Some(docker.create_sandbox(deployment).map_err(|f| { + RuntimeError::Docker(Box::new(super::ProviderFailure::Create(f))) + })?); + let owner = instance.as_mut().expect("acknowledged container"); + owner + .upload(&mut file, &request.daemon_config) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Upload(f))))?; + phase = LaunchPhase::Start; + owner + .start() + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Start(f))))?; + phase = LaunchPhase::Listener; + owner + .wait_listener(request.startup_timeout) + .map_err(|f| RuntimeError::Docker(Box::new(super::ProviderFailure::Listener(f))))?; + Ok(()) + })(); + match result { + Ok(()) => Ok(instance.expect("launched")), + Err(cause) => Err(Box::new(LaunchFailure { + phase, + instance: instance + .map(|sandbox| Box::new(crate::backend::Sandbox::Docker(sandbox))), + cause, + })), + } + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + Sandbox::reopen(self.docker.clone(), id, record.clone().into()) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.docker() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "Docker requires a Docker deployment record", + ))); + }; + let request: SandboxRequest = record.clone().into(); + request.check().map_err(WireFailure::local)?; + Ok(Sandbox::stopped(self.docker.clone(), id, request)) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs new file mode 100644 index 0000000000..59a415d3a5 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_exec.rs @@ -0,0 +1,100 @@ +//! Ordinary noninteractive provider execution, with streamed output and exact failure custody. +use super::{ProviderFailure, Sandbox}; +use crate::{provider::ExecExit, ExecRequest, RuntimeError, WireFailure}; +use std::io::Write; +fn failure(container: crate::ContainerId, cause: ProviderFailure) -> Box { + // Scalar facts describe only the deciding exchange. Owned pending bytes, + // diagnostics and fence errors remain in the original nested receipt. + let wire = match &cause { + ProviderFailure::ExecCreate(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecStart(f) => Some(f.transfer.as_ref()), + ProviderFailure::ExecInspect { failure, .. } => Some(failure.as_ref()), + _ => None, + }; + let mut outer = WireFailure::local(RuntimeError::Protocol("provider execution")); + if let Some(wire) = wire { + outer.attempted = wire.attempted; + outer.sent_bytes = wire.sent_bytes; + outer.request_header_bytes = wire.request_header_bytes; + outer.sent_body_bytes = wire.sent_body_bytes; + outer.status = wire.status; + outer.observed_exec = wire.observed_exec; + outer.observed_container = wire.observed_container; + outer.requested_exec = wire.requested_exec; + } else { + // Attachment/splitting, output and exit observation follow an attempted Start. + outer.attempted = true; + } + outer.requested_container = Some(container); + outer.requested_exec = outer.requested_exec.or_else(|| match &cause { + ProviderFailure::ExecStart(f) => Some(f.created.identity().1), + ProviderFailure::ExecOutput { handle, .. } + | ProviderFailure::ExecInspect { handle, .. } + | ProviderFailure::ExecExitUnknown { handle, .. } => Some(handle.identity().1), + _ => None, + }); + outer.cause = RuntimeError::Docker(Box::new(cause)); + outer +} +pub(crate) fn run( + sandbox: &Sandbox, + request: &ExecRequest, + mut stdout: &mut (dyn Write + Send), + mut stderr: &mut (dyn Write + Send), +) -> Result> { + if request.stdin { + return Err( + sandbox.invalid("provider run has no stdin source; use ordinary Docker streams") + ); + } + let id = sandbox.identity(); + let created = sandbox + .create_exec(request.clone()) + .map_err(|f| failure(id, ProviderFailure::ExecCreate(f)))?; + let attached = created + .start() + .map_err(|f| failure(id, ProviderFailure::ExecStart(f)))?; + let (input, output, handle) = attached + .into_parts() + .map_err(|f| failure(id, ProviderFailure::ExecSplit(f)))?; + // Stdin was disabled in the creation request; dropping it sends no cancellation. + drop(input); + let progress = match output.copy_to(&mut stdout, &mut stderr) { + Ok(progress) => progress, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecOutput { failure: f, handle }, + )) + } + }; + let inspection = match handle.inspect() { + Ok(inspection) => inspection, + Err(f) => { + return Err(failure( + id, + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + )) + } + }; + let Some(code) = inspection.known_root_exit() else { + return Err(failure( + id, + ProviderFailure::ExecExitUnknown { + inspection, + handle, + progress, + }, + )); + }; + Ok(ExecExit { + code, + pid: inspection.pid, + stdout_bytes: progress.stdout, + stderr_bytes: progress.stderr, + }) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs new file mode 100644 index 0000000000..c97c9a596d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/docker/provider_failure.rs @@ -0,0 +1,32 @@ +//! Original Docker stage failures retained across the provider's common error port. +use super::{ + ExecCreateFailure, ExecHandle, ListenerFailure, OutputFailure, OutputProgress, + SandboxCreateFailure, SplitFailure, StartFailure, UploadFailure, +}; +use crate::{ExecInspection, WireFailure}; +/// No failed creation/Start/stream owner is reconstructed, replayed or silently cleaned up. +#[derive(Debug)] +pub enum ProviderFailure { + Create(Box), + Upload(Box), + Start(Box), + Listener(Box), + ExecCreate(Box), + ExecStart(Box), + ExecSplit(Box), + ExecOutput { + failure: Box, + handle: ExecHandle, + }, + ExecInspect { + failure: Box, + handle: ExecHandle, + progress: OutputProgress, + }, + /// EOF is not proof of process exit. The original handle permits explicit observation. + ExecExitUnknown { + inspection: ExecInspection, + handle: ExecHandle, + progress: OutputProgress, + }, +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs index b7622d7efc..5b283c9c85 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/request.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/request.rs @@ -1,58 +1,11 @@ //! One ordinary non-TTY Exec creation; exact acknowledged runtime custody. -use super::{ - json::{once, Json}, - strings::quoted, - Docker, -}; -use crate::{CommandIdentity, ContainerId, ExecId, RuntimeError, WireFailure}; -use std::{ - fmt, - io::{self, Write}, -}; -/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. -#[derive(Clone)] -pub struct ExecRequest { - pub identity: CommandIdentity, - pub arguments: Vec, - pub environment: Vec, - pub directory: String, - pub stdin: bool, -} -impl fmt::Debug for ExecRequest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ExecRequest") - .field("identity", &self.identity) - .field("arguments_count", &self.arguments.len()) - .field("environment_count", &self.environment.len()) - .field("directory", &self.directory) - .field("stdin", &self.stdin) - .finish() - } -} +use super::Docker; +use crate::backend::json::{once, Json}; +use crate::backend::strings::quoted; +pub use crate::ExecRequest; +use crate::{ContainerId, ExecId, RuntimeError, WireFailure}; +use std::io::{self, Write}; impl ExecRequest { - fn check(&self) -> Result<(), RuntimeError> { - self.identity.check()?; - if self.arguments.is_empty() - || self.arguments[0].is_empty() - || !std::path::Path::new(&self.directory).is_absolute() - || self - .arguments - .iter() - .chain(&self.environment) - .chain(std::iter::once(&self.directory)) - .any(|v| v.as_bytes().contains(&0)) - { - return Err(RuntimeError::Protocol("ordinary command fields")); - } - if self - .environment - .iter() - .any(|v| !v.contains('=') || v.starts_with('=')) - { - return Err(RuntimeError::Protocol("ordinary environment field")); - } - Ok(()) - } fn encode(&self, out: &mut dyn Write) -> io::Result<()> { out.write_all(b"{\"AttachStdout\":true,\"AttachStderr\":true,\"AttachStdin\":")?; out.write_all(if self.stdin { b"true" } else { b"false" })?; diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs index 9ee9ae628d..3626ab3bbe 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/topology.rs @@ -1,8 +1,6 @@ //! Exact observed daemon identity, security and kernel-access settings, and volume placement. -use super::{ - json::{once, Json}, - SandboxRequest, -}; +use super::SandboxRequest; +use crate::backend::json::{once, Json}; use crate::RuntimeError; use std::io::Read; /// The container's whole system-call filter. It denies creation of a user diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs index ae3504b098..973623a478 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/version.rs +++ b/engine/crates/layerfs-sandbox/src/backend/docker/version.rs @@ -1,9 +1,9 @@ //! The Engine API version a client speaks: the daemon's, up to the newest this client targets. use super::{ http::{failure, Mode}, - json::{once, Json}, Docker, }; +use crate::backend::json::{once, Json}; use crate::{ErrorDiagnostic, RuntimeError, WireFailure}; /// The oldest Engine API minor version (1.N) this client speaks: Engine 28.0. /// Before it, uploading the daemon fails (`getent unable to find entry "0:0"`: diff --git a/engine/crates/layerfs-sandbox/src/backend/identity.rs b/engine/crates/layerfs-sandbox/src/backend/identity.rs new file mode 100644 index 0000000000..8e1c174c73 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/identity.rs @@ -0,0 +1,16 @@ +//! A fresh random identity in the Engine's form (64 lowercase hex, not all +//! zero), so records, names and the cgroup directory treat a local Sandbox +//! like a container. +use crate::{ContainerId, RuntimeError}; +use std::{fs::File, io::Read}; + +pub(crate) fn random() -> Result { + let mut bytes = [0u8; 32]; + File::open("/dev/urandom")?.read_exact(&mut bytes)?; + const HEX: &[u8; 16] = b"0123456789abcdef"; + let text: String = (bytes.iter()) + .flat_map(|b| [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]]) + .map(char::from) + .collect(); + ContainerId::parse(&text) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/json.rs b/engine/crates/layerfs-sandbox/src/backend/json.rs similarity index 100% rename from engine/crates/layerfs-sandbox/src/backend/docker/json.rs rename to engine/crates/layerfs-sandbox/src/backend/json.rs diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs new file mode 100644 index 0000000000..46c121526d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mod.rs @@ -0,0 +1,7 @@ +//! Linux platform facts for the native runtime: the daemon runs as root and +//! serves FUSE, so its process facts come from `/proc` and its mounts are +//! detached lazily. Only facts live here; the Sandbox lifecycle is shared +//! with macOS in [`super::native`]. +pub(crate) mod mounts; +mod process; +pub(crate) use process::{boot, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs new file mode 100644 index 0000000000..35f67d00ed --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/mounts.rs @@ -0,0 +1,20 @@ +//! Linux mount table and lazy detach slot; implemented by the Linux native backend. +use std::{ + io, + path::{Path, PathBuf}, +}; + +const UNSUPPORTED: &str = "Linux native mount table"; + +pub(crate) fn under(_directory: &Path) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn filesystem(_path: &Path) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + let _ = filesystem(directory); + under(directory).map(drop) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/linux/process.rs b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs new file mode 100644 index 0000000000..53b1661565 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/linux/process.rs @@ -0,0 +1,17 @@ +//! Linux `/proc` process facts slot; implemented by the Linux native backend. +use super::super::native::process::Stat; +use std::io; + +const UNSUPPORTED: &str = "Linux native process table"; + +pub(crate) fn boot() -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn stat(_pid: u32) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn processes() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/listener.rs b/engine/crates/layerfs-sandbox/src/backend/listener.rs new file mode 100644 index 0000000000..c8fe639ce1 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/listener.rs @@ -0,0 +1,68 @@ +//! The daemon's startup line: a bounded wait for the loopback address it +//! prints once it listens. Both local backends spawn a daemon whose output is +//! a log file, so both read that same marker from it, each look taking only +//! what was appended since the last one. Nothing is retried: the wait ends at +//! the marker, at the daemon's exit, or at the caller's deadline. +use crate::RuntimeError; +use std::{ + fs::File, + io::{self, Read, Seek, SeekFrom}, + path::Path, + time::{Duration, Instant}, +}; + +/// The daemon's startup line; the address follows it. +pub(crate) const MARKER: &[u8] = b"LAYERFS_DAEMON_LISTEN "; +/// Pause between two reads of a starting daemon's output. +const PAUSE: Duration = Duration::from_millis(10); + +/// Waits until `log` names the daemon's loopback address, the daemon is no +/// longer alive (as `alive` reports), or `wait` ends. +pub(crate) fn await_listener( + log: &Path, + wait: Duration, + alive: &dyn Fn() -> bool, +) -> Result { + let deadline = Instant::now() + wait; + let mut file = File::open(log)?; + let mut offset = 0u64; + let mut line = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + file.seek(SeekFrom::Start(offset))?; + loop { + let n = file.read(&mut chunk)?; + if n == 0 { + break; + } + offset += n as u64; + for &byte in &chunk[..n] { + if byte != b'\n' { + // a startup line is short; a longer one is not the marker + if line.len() < 256 { + line.push(byte); + } + continue; + } + if let Some(address) = line.strip_prefix(MARKER) { + let address = std::str::from_utf8(address) + .ok() + .and_then(|text| text.parse::().ok()) + .filter(|a| a.ip().is_loopback() && a.port() != 0) + .ok_or(RuntimeError::Protocol("local daemon loopback listener"))?; + return Ok(address.port()); + } + line.clear(); + } + } + if !alive() { + return Err(RuntimeError::Protocol( + "local daemon exited before its listener line", + )); + } + if Instant::now() >= deadline { + return Err(io::Error::from(io::ErrorKind::TimedOut).into()); + } + std::thread::sleep(PAUSE); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs new file mode 100644 index 0000000000..23b45ea94b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs @@ -0,0 +1,28 @@ +//! macOS Darwin process and mount facts slot; implemented by the macOS native backend. +use crate::backend::native::process::Stat; +use std::{ + io, + path::{Path, PathBuf}, +}; + +const UNSUPPORTED: &str = "macOS native process and mount table"; + +pub(crate) fn stat(_pid: u32) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn processes() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn boot() -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn mount_points() -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} + +pub(crate) fn force_unmount(_point: &Path) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, UNSUPPORTED)) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs new file mode 100644 index 0000000000..f9bf26868c --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mod.rs @@ -0,0 +1,7 @@ +//! macOS platform facts for the native runtime: the daemon serves loopback +//! NFS as this user, so its process facts come from `proc_pidinfo`/`sysctl` +//! and its mounts from `mount`/`umount -f`. Only facts live here; the Sandbox +//! lifecycle is shared with Linux in [`super::native`]. +mod darwin; +pub(crate) mod mounts; +pub(crate) use darwin::{boot, force_unmount, mount_points, processes, stat}; diff --git a/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs new file mode 100644 index 0000000000..2b05e21e2d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs @@ -0,0 +1,16 @@ +//! macOS mount table and forced detach slot; implemented by the macOS native backend. +use std::{ + io, + path::{Path, PathBuf}, +}; + +pub(crate) fn under(_directory: &Path) -> io::Result> { + super::mount_points() +} + +pub(crate) fn detach(directory: &Path) -> io::Result<()> { + for point in under(directory)? { + super::force_unmount(&point)?; + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/mod.rs b/engine/crates/layerfs-sandbox/src/backend/mod.rs index 509d7e29fa..602a842926 100644 --- a/engine/crates/layerfs-sandbox/src/backend/mod.rs +++ b/engine/crates/layerfs-sandbox/src/backend/mod.rs @@ -1,2 +1,20 @@ -//! Concrete ordinary runtime providers. +//! Concrete ordinary runtime backends and the enum that selects one. pub mod docker; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod identity; +mod json; +#[cfg(target_os = "linux")] +pub(crate) mod linux; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod listener; +#[cfg(target_os = "macos")] +pub(crate) mod macos; +#[cfg(target_os = "linux")] +pub mod namespaces; +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub mod native; +pub mod runtime; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod streams; +mod strings; +pub use runtime::{Runtime, Sandbox}; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs new file mode 100644 index 0000000000..95e8e27e0b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/mod.rs @@ -0,0 +1,6 @@ +//! Linux namespace Sandboxes slot; implemented by the namespaces backend. +mod provider; +mod sandbox; + +pub use provider::NamespaceProvider; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs new file mode 100644 index 0000000000..b57cd94a16 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/provider.rs @@ -0,0 +1,74 @@ +//! Linux namespace runtime slot; implemented by the namespaces backend. +use super::sandbox::Sandbox; +use crate::backend::native::{Local, LocalPaths}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, MountDriver, ProviderCapabilities, + SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +const UNSUPPORTED: &str = "the Linux namespaces Sandbox backend"; + +/// A Linux namespace runtime over one local root directory. +#[derive(Clone, Debug)] +pub struct NamespaceProvider { + local: Local, +} + +impl NamespaceProvider { + /// Refuses with `Unsupported` before any effect in the interface-only build. + pub fn new(root: impl AsRef) -> Result { + let _ = ( + Local::new(root), + crate::backend::json::Json::<&[u8]>::text_into, + ); + Err(RuntimeError::Unsupported(UNSUPPORTED)) + } + + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + + /// What this backend supports. + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: "namespaces-linux", + isolation: IsolationModel::MicroVm, + mount_driver: MountDriver::Fuse, + requires_image: true, + search_path_policy: SearchPathPolicy::FixedContainer, + supports_resource_limits: false, + direct_store_access: true, + } + } + + /// Starts one Sandbox once; a failed start keeps the acknowledged owner. + pub fn launch(&self, _request: LaunchRequest) -> Result> { + Err(Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(UNSUPPORTED), + })) + } + + /// Reopen starts nothing: it checks the recorded init process, its cgroup + /// and its listener. + pub fn reopen( + &self, + _id: ContainerId, + _record: &SandboxRecord, + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + _id: ContainerId, + _record: &SandboxRecord, + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs new file mode 100644 index 0000000000..c2f85776d9 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/namespaces/sandbox.rs @@ -0,0 +1,75 @@ +//! Linux namespace Sandbox handle slot; implemented by the namespaces backend. +use crate::backend::native::LocalProcess; +use crate::provider::{ExecExit, SandboxRecord}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +const UNSUPPORTED: &str = "the Linux namespaces Sandbox backend"; + +/// One acknowledged namespace Sandbox. +#[derive(Debug)] +pub struct Sandbox { + id: ContainerId, +} + +impl Sandbox { + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + PathBuf::new() + } + + /// Its init stage process, once started. + pub fn process(&self) -> Option { + None + } + + /// Its record, naming this backend so another one never reopens it. + pub fn record(&self) -> Option { + None + } + + /// The daemon's endpoint while its init stage runs. + pub fn endpoint(&self) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Runs one command to its end inside the Sandbox's namespaces. + pub fn run( + &self, + _request: &ExecRequest, + _stdout: &mut (dyn Write + Send), + _stderr: &mut (dyn Write + Send), + ) -> Result> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Its cgroup owns every process the Sandbox started. + pub fn track(&self, _pid: u32, _start_time: u64) -> Result<(), Box> { + Ok(()) + } + + /// The daemon output's last `lines` lines, keeping only the final `cap` bytes. + pub fn log_tail(&self, _lines: u16, _cap: usize) -> Result, Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Reaps the init stage this process started, if it exited. + pub(crate) fn reap(&self) -> bool { + true + } + + /// Ends the Sandbox once. + pub fn stop(&mut self, _timeout_seconds: u16) -> Result<(), Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } + + /// Deletes its directory and cgroup once, after a known stop. + pub fn delete(&mut self) -> Result<(), Box> { + Err(WireFailure::local(RuntimeError::Unsupported(UNSUPPORTED))) + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/create.rs b/engine/crates/layerfs-sandbox/src/backend/native/create.rs new file mode 100644 index 0000000000..c814869456 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/create.rs @@ -0,0 +1,284 @@ +//! One local daemon start: the Sandbox's private directory, its 0600 setup, +//! one spawn and its listener line. Nothing is retried. +use super::{ + process::LocalProcess, + sandbox::{Sandbox, LOG}, + search::search_path, +}; +use crate::backend::{identity, listener}; +use crate::RuntimeError; +use nix::fcntl::OFlag; +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{self, Write}, + os::unix::{ + fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + process::CommandExt, + }, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, + time::Duration, +}; + +/// A local runtime root: one directory per Sandbox under it. +#[derive(Clone, Debug)] +pub struct Local { + root: PathBuf, + name: Option, + /// Variables the daemon starts with besides `PATH`. + environment: Vec<(String, String)>, +} + +/// Where one Sandbox keeps its parts, all under its directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalPaths { + /// The Sandbox directory, `/`, 0755. + pub directory: PathBuf, + /// Its daemon setup, `config/daemon.setup`, 0600 in a 0700 directory. + pub config: PathBuf, + /// Its daemon overlay template, `overlay/overlay.sqlite`, in a 0700 directory. + pub overlay: PathBuf, + /// Its Workspace mount home, `workspaces`, made 0755 by the daemon. + pub mounts: PathBuf, + /// Its daemon's output, `daemon.log`, 0600. + pub log: PathBuf, + /// Its commands' scratch directory, `tmp`, 1777 like a container's + /// `/tmp`, so what they leave there goes with the Sandbox. + pub tmp: PathBuf, +} + +/// The boundary one failed create reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LocalPhase { + /// Local checks; nothing made. + Validate, + /// The directory, setup and log file. + Create, + /// The daemon spawn. + Start, + /// The wait for the daemon's listener line. + Listener, +} + +/// A failed create. `sandbox` holds what it made (its directory, and its +/// daemon once spawned), for the caller to stop and delete once. +#[derive(Debug)] +pub struct LocalCreateFailure { + pub phase: LocalPhase, + pub sandbox: Option, + pub cause: RuntimeError, +} + +impl Local { + /// Selects `root`: an existing absolute directory owned by this user, + /// writable by it alone and traversable by others, as is every directory + /// above it (commands enter their Workspace below it only after taking + /// another identity, so one closed directory on the way, such as a 0700 + /// `/root`, would fail every command at spawn). + pub fn new(root: impl AsRef) -> Result { + let local = Self::native(root.as_ref())?; + let root = root.as_ref(); + // every directory on the path as given and on the path it resolves to + let resolved = fs::canonicalize(root)?; + for directory in root.ancestors().chain(resolved.ancestors()) { + let metadata = fs::metadata(directory)?; + if !metadata.is_dir() || metadata.mode() & 0o001 == 0 { + return Err(RuntimeError::Io(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "{} is not traversable by others (o+x); local Sandbox commands run as \ + another identity and must pass it to reach their Workspace", + directory.display() + ), + ))); + } + } + Ok(local) + } + /// Selects `root` for the native route, whose commands run as this user + /// (no other identity): the root's own checks as in [`Self::new`], but + /// the directories above it need not be traversable by others (a macOS + /// home is 0750). + pub fn native(root: impl AsRef) -> Result { + let root = root.as_ref(); + if !root.is_absolute() + || root + .components() + .any(|c| matches!(c, Component::ParentDir | Component::CurDir)) + { + return Err(RuntimeError::Protocol("absolute local runtime root")); + } + let metadata = fs::symlink_metadata(root)?; + if !metadata.is_dir() + || metadata.uid() != nix::unistd::Uid::effective().as_raw() + || metadata.mode() & 0o022 != 0 + || metadata.mode() & 0o001 == 0 + { + return Err(RuntimeError::Protocol( + "local runtime root must be a directory of this user, not group/other writable, traversable by others (0755)", + )); + } + Ok(Self { + root: root.to_owned(), + name: None, + environment: Vec::new(), + }) + } + /// Starts the next daemon with `environment` besides its search path (the + /// native route's `LAYERFS_KERNEL=nfs` and its mount helpers). Nothing + /// else of this process's environment reaches it. + pub fn environment(mut self, environment: Vec<(String, String)>) -> Self { + self.environment = environment; + self + } + /// Names the next Sandbox's directory `name`, under the Docker name rule + /// (an ASCII letter or digit, then letters, digits, `_`, `.`, `-`, at + /// most 128), so it is one plain path component. + pub fn naming(mut self, name: &str) -> Result { + let bytes = name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 128 + || !bytes[0].is_ascii_alphanumeric() + || !bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) + { + return Err(RuntimeError::Protocol("local Sandbox name")); + } + self.name = Some(name.to_owned()); + Ok(self) + } + /// The named Sandbox's paths, for its daemon setup. + pub fn paths(&self) -> Result { + let name = self + .name + .as_deref() + .ok_or(RuntimeError::Protocol("local Sandbox name"))?; + let directory = self.root.join(name); + Ok(LocalPaths { + config: directory.join("config").join("daemon.setup"), + overlay: directory.join("overlay").join("overlay.sqlite"), + mounts: directory.join("workspaces"), + log: directory.join(LOG), + tmp: directory.join("tmp"), + directory, + }) + } + /// Makes the named Sandbox's directory, writes `config` (the encoded + /// daemon setup naming [`Self::paths`]) 0600, spawns `executable + /// --config ` in its own process group with no inherited + /// environment but the search path, and waits up to `wait` for its + /// listener line. Once anything is made, a failure returns it. + pub fn create( + &self, + executable: &Path, + config: &[u8], + wait: Duration, + labels: Vec<(String, String)>, + ) -> Result> { + let fail = |phase, sandbox, cause| { + Box::new(LocalCreateFailure { + phase, + sandbox, + cause, + }) + }; + let paths = self + .paths() + .map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let checked = (|| { + let metadata = fs::metadata(executable)?; + if !executable.is_absolute() || !metadata.is_file() || metadata.len() == 0 { + return Err(RuntimeError::Protocol("local daemon executable")); + } + if config.is_empty() || config.len() > 8192 || wait.is_zero() { + return Err(RuntimeError::Protocol("local daemon setup/startup wait")); + } + match fs::symlink_metadata(&paths.directory) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + Ok(_) => Err(RuntimeError::Protocol("local Sandbox directory exists")), + } + })(); + checked.map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + let id = identity::random().map_err(|cause| fail(LocalPhase::Validate, None, cause))?; + // one create of the directory: a name another start holds fails here + fs::DirBuilder::new() + .mode(0o755) + .create(&paths.directory) + .map_err(|error| fail(LocalPhase::Create, None, error.into()))?; + let mut sandbox = Sandbox::unstarted(id, paths.directory.clone(), labels); + let log = match prepare(&paths, config) { + Ok(log) => log, + Err(error) => return Err(fail(LocalPhase::Create, Some(sandbox), error.into())), + }; + let spawned = log.try_clone().and_then(|output| { + Command::new(executable) + .arg("--config") + .arg(&paths.config) + .env_clear() + .env("PATH", search_path()) + .envs(self.environment.iter().map(|(k, v)| (k, v))) + .current_dir(&paths.directory) + .stdin(Stdio::null()) + .stdout(output) + .stderr(log) + .process_group(0) + .spawn() + }); + let mut child = match spawned { + Ok(child) => child, + Err(error) => return Err(fail(LocalPhase::Start, Some(sandbox), error.into())), + }; + let process = match LocalProcess::observe(child.id()) { + Ok(process) => process, + Err(error) => { + // spawned but unobservable: end it here, it has no identity to record + let _ = child.kill(); + let _ = child.wait(); + return Err(fail(LocalPhase::Start, Some(sandbox), error.into())); + } + }; + sandbox.process = Some(process); + *sandbox + .child + .get_mut() + .unwrap_or_else(|error| error.into_inner()) = Some(child); + match listener::await_listener(&paths.log, wait, &|| { + sandbox.reap(); + sandbox.process.is_some_and(LocalProcess::alive) + }) { + Ok(port) => { + sandbox.port = port; + Ok(sandbox) + } + Err(cause) => Err(fail(LocalPhase::Listener, Some(sandbox), cause)), + } + } +} + +/// The setup and overlay directories (0700), the scratch directory (1777), +/// the setup file (0600, written once) and the empty log file (0600) it +/// returns. +fn prepare(paths: &LocalPaths, config: &[u8]) -> io::Result { + // the process umask may narrow the create mode + fs::set_permissions(&paths.directory, Permissions::from_mode(0o755))?; + for file in [&paths.config, &paths.overlay] { + let parent = file + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "local path"))?; + fs::DirBuilder::new().mode(0o700).create(parent)?; + } + fs::DirBuilder::new().mode(0o700).create(&paths.tmp)?; + fs::set_permissions(&paths.tmp, Permissions::from_mode(0o1777))?; + let private = |path: &Path| { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(path) + }; + private(&paths.config)?.write_all(config)?; + private(&paths.log) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/exec.rs b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs new file mode 100644 index 0000000000..1478054771 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/exec.rs @@ -0,0 +1,133 @@ +//! One ordinary command to its end in a local Sandbox: the command identity, +//! the Workspace directory and a search path (on macOS this process's own); +//! nothing else of this process's environment, no shell of its own and no +//! implicit timeout. Each command leads a process group of its own, named in +//! the Sandbox until it is empty, so what it leaves running ends with the +//! Sandbox. +use super::{groups, process, process::LocalProcess, sandbox::Sandbox, search::search_path}; +use crate::backend::streams; +use crate::{ExecRequest, RuntimeError, WireFailure}; +use nix::sys::signal::{killpg, Signal}; +use std::{ + io::{self, Write}, + os::unix::process::{CommandExt, ExitStatusExt}, + path::Path, + process::{Command, Stdio}, +}; + +/// One command's end: its exit code (128 + the signal for one a signal +/// ended, as the Engine reports it), its process id (diagnostic only) and the +/// bytes each stream delivered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalExit { + pub code: i32, + pub pid: u32, + pub stdout: u64, + pub stderr: u64, +} + +impl Sandbox { + /// Runs `request` once while this Sandbox's daemon runs: as its uid/gid + /// with no supplementary groups, in its directory, its stdout and stderr + /// streamed to the sinks as they arrive, then waits for its exit. A sink + /// failure closes the command's pipes, still waits for its exit, and + /// returns the sink's error. Standard input is not offered. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + check(request).map_err(|cause| self.failure(cause))?; + self.endpoint()?; + let mut command = Command::new(&request.arguments[0]); + command + .args(&request.arguments[1..]) + .env_clear() + .env("PATH", search_path()) + .current_dir(Path::new(&request.directory)) + // std drops supplementary groups (setgroups 0) when it sets a uid as root + .uid(request.identity.uid) + .gid(request.identity.gid) + .process_group(0) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + for entry in &request.environment { + if let Some((name, value)) = entry.split_once('=') { + command.env(name, value); + } + } + let mut child = command.spawn().map_err(|error| { + let mut failure = self.io(error); + failure.attempted = true; + failure + })?; + let pid = child.id(); + // named before anything is waited for: the leader is not reaped + // before the wait below, so it is observable even once it exited + let named = LocalProcess::observe(pid) + .and_then(|leader| groups::name(self.directory(), leader).map(|()| leader)); + let leader = match named { + Ok(leader) => leader, + Err(error) => { + // an unnamed group would outlive the Sandbox: it ends here, once + if let Ok(group) = process::pid(pid) { + let _ = killpg(group, Signal::SIGKILL); + } + let _ = child.wait(); + let mut failure = self.io(error); + failure.attempted = true; + return Err(failure); + } + }; + let out = child.stdout.take(); + let err = child.stderr.take(); + let copied = std::thread::scope(|scope| { + let errors = scope.spawn(move || streams::copy(err, stderr)); + let outs = streams::copy(out, stdout); + let errs = errors + .join() + .unwrap_or_else(|_| Err(io::Error::other("stderr copy panicked"))); + Ok::<_, io::Error>((outs?, errs?)) + }); + // the pipes are closed here either way; the exit is still waited for + let status = child.wait(); + // a name left behind is harmless: its group is checked again when ended + let _ = groups::forget_if_empty(self.directory(), leader); + let (out_bytes, err_bytes) = copied.map_err(|error| self.io(error))?; + let status = status.map_err(|error| self.io(error))?; + let code = status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .ok_or_else(|| self.invalid("local command exit status"))?; + Ok(LocalExit { + code, + pid, + stdout: out_bytes, + stderr: err_bytes, + }) + } +} + +/// The Engine's own checks of an ordinary command, and no stdin. +fn check(request: &ExecRequest) -> Result<(), RuntimeError> { + request.identity.check()?; + if request.arguments.is_empty() + || request.arguments[0].is_empty() + || !Path::new(&request.directory).is_absolute() + || (request.arguments.iter()) + .chain(&request.environment) + .chain(std::iter::once(&request.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if (request.environment.iter()).any(|v| !v.contains('=') || v.starts_with('=')) { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + if request.stdin { + return Err(RuntimeError::Unsupported("local command standard input")); + } + Ok(()) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/groups.rs b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs new file mode 100644 index 0000000000..6f18869381 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/groups.rs @@ -0,0 +1,149 @@ +//! The process groups a local Sandbox's processes run in. A container's +//! processes end with it; a local Sandbox's would outlive it on this kernel, +//! so each command starts its own process group, named in the Sandbox's +//! `groups` directory while it may have members, and every named group (and +//! the daemon's own) is killed when the Sandbox stops or is deleted. +//! +//! A group is named by its leader: id, start time and boot. Its id stays the +//! group's while any member lives, so a group whose leader is gone is still +//! known by it; a leader found with another start time means the id was +//! reused and the group is not this Sandbox's. A process that leaves its +//! group (`setsid`) is not followed unless it is named here too +//! ([`super::Sandbox::track`]). +use super::platform; +use super::process::{self, LocalProcess}; +use nix::{fcntl::OFlag, sys::signal::Signal}; +use std::{ + fs::{self, OpenOptions}, + io, + os::unix::fs::{DirBuilderExt, OpenOptionsExt}, + path::Path, + time::{Duration, Instant}, +}; + +/// The directory of a Sandbox that names its groups, one empty file each. +pub(super) const GROUPS: &str = "groups"; +/// Pause between two looks at a killed group. +const PAUSE: Duration = Duration::from_millis(20); + +/// Names the group `leader` leads (its id is the group's), once. +pub(super) fn name(directory: &Path, leader: LocalProcess) -> io::Result<()> { + let groups = directory.join(GROUPS); + // a Sandbox started before groups were named has no directory yet + match fs::DirBuilder::new().mode(0o700).create(&groups) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC).bits()) + .open(groups.join(entry(leader)?)) + .map(drop) +} + +/// Forgets the group `leader` led once no member of it is left. Called after +/// the leader was reaped; a group that still has members stays named. +pub(super) fn forget_if_empty(directory: &Path, leader: LocalProcess) -> io::Result<()> { + match nix::sys::signal::killpg(process::pid(leader.pid)?, None) { + Err(nix::errno::Errno::ESRCH) => {} + _ => return Ok(()), + } + match fs::remove_file(directory.join(GROUPS).join(entry(leader)?)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Kills every group the Sandbox at `directory` named, and the group of +/// `daemon` (its leader), once with KILL, as a container's end kills what +/// ran in it, then waits up to `wait` for their members to be gone. Groups +/// of another boot ended with it. +pub(super) fn end( + directory: &Path, + daemon: Option, + wait: Duration, +) -> io::Result<()> { + let mut leaders: Vec = daemon.into_iter().collect(); + match fs::read_dir(directory.join(GROUPS)) { + Ok(items) => { + for item in items { + let name = item?.file_name(); + let leader = name.to_str().and_then(parse).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "local Sandbox group name") + })?; + leaders.push(leader); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let this_boot = process::boot()?; + leaders.retain(|leader| leader.boot == Some(this_boot)); + let table = processes()?; + let mut killed = Vec::new(); + for leader in leaders { + if !members(&table, leader).is_empty() { + match nix::sys::signal::killpg(process::pid(leader.pid)?, Signal::SIGKILL) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => killed.push(leader), + Err(errno) => return Err(errno.into()), + } + } + } + let deadline = Instant::now() + wait; + loop { + let table = processes()?; + if killed + .iter() + .all(|leader| members(&table, *leader).is_empty()) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::other( + "a process of the local Sandbox still runs after KILL", + )); + } + std::thread::sleep(PAUSE); + } +} + +/// The running members of `leader`'s group: none when the leader's id now +/// names another process (the id was reused, so the group is another's). +fn members(table: &[(u32, process::Stat)], leader: LocalProcess) -> Vec { + let reused = (table.iter()).any(|(pid, stat)| *pid == leader.pid && stat.start != leader.start); + if reused { + return Vec::new(); + } + (table.iter()) + .filter(|(_, stat)| stat.group == leader.pid && !stat.ended()) + .map(|(pid, _)| *pid) + .collect() +} + +/// Every process of this kernel now, with its stat. One that ends while +/// being read is left out. +fn processes() -> io::Result> { + platform::processes() +} + +/// A group's file name: `--`. +fn entry(leader: LocalProcess) -> io::Result { + let boot = leader + .boot + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "a group leader's boot"))?; + Ok(format!("{boot:032x}-{}-{}", leader.pid, leader.start)) +} + +fn parse(name: &str) -> Option { + let mut parts = name.split('-'); + let boot = parts.next().filter(|b| b.len() == 32)?; + let process = LocalProcess { + boot: Some(u128::from_str_radix(boot, 16).ok()?), + pid: parts.next()?.parse().ok()?, + start: parts.next()?.parse().ok()?, + }; + parts.next().is_none().then_some(process) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/mod.rs b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs new file mode 100644 index 0000000000..1f1fb4ad81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/mod.rs @@ -0,0 +1,40 @@ +//! Native process runtime: each Sandbox is one daemon process on this kernel, +//! started from a private directory, with no container, image or Engine. +//! +//! It is for a machine that is itself the isolation boundary (a microVM, a +//! CI job, a developer's own Mac). It isolates nothing: the daemon and its +//! commands share this kernel's processes, network, users and filesystem with +//! everything else on it. Commands still run as their own nonroot identity +//! where the platform allows it, and what they leave running ends with the +//! Sandbox. One attempt per operation, as in the Docker runtime; nothing is +//! retried. +//! +//! The lifecycle here is one implementation for both platforms. What differs +//! is read where it lives: [`super::macos`] supplies Darwin process facts, the +//! `mount` table and `umount -f` for the loopback NFS the macOS daemon serves; +//! [`super::linux`] supplies `/proc` facts, the `mountinfo` table and a lazy +//! `MNT_DETACH` for the FUSE mount the root daemon serves. +//! +//! The daemon and its commands start with one search path and nothing else of +//! this process's environment: on Linux a container's, whatever this process +//! has; on macOS this process's own `PATH`, since a native command is this +//! user's own tool (Homebrew's `/opt/homebrew/bin`, `~/.local/bin`, a version +//! manager's `node`) and no image supplies it. +mod create; +mod exec; +mod groups; +pub(crate) mod process; +mod provider; +mod sandbox; +pub(crate) mod search; +#[cfg(target_os = "linux")] +pub(crate) use super::linux as platform; +#[cfg(target_os = "macos")] +pub(crate) use super::macos as platform; +pub use create::{Local, LocalCreateFailure, LocalPaths, LocalPhase}; +pub use exec::LocalExit; +pub use process::LocalProcess; +pub use provider::{ + NativeLinuxInstance, NativeLinuxProvider, NativeMacosInstance, NativeMacosProvider, +}; +pub use sandbox::Sandbox; diff --git a/engine/crates/layerfs-sandbox/src/backend/native/process.rs b/engine/crates/layerfs-sandbox/src/backend/native/process.rs new file mode 100644 index 0000000000..74d58bf34d --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/process.rs @@ -0,0 +1,85 @@ +//! A local process named by its id, kernel start time and boot, so a reused +//! id, or the same id and start time after a reboot or snapshot restore, is +//! never taken for the process that was started. The three facts come from +//! `/proc` on Linux and from `proc_pidinfo`/`sysctl` on macOS (see +//! [`super::macos`]). +pub(crate) use super::platform::{boot, stat}; +use nix::{sys::signal::Signal, unistd::Pid}; +use std::io; + +/// One started process: its id, its start time in clock ticks since boot +/// (`/proc//stat` field 22) and the boot it started in +/// (`/proc/sys/kernel/random/boot_id`). All must match for it to count as +/// alive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct LocalProcess { + pub pid: u32, + pub start: u64, + /// The boot id as a number. `None` only for a record written before boot + /// ids were kept: it is matched by id and start time alone, as it was + /// then, and never signalled. + pub boot: Option, +} + +impl LocalProcess { + /// The process running now under `pid`, read once from this kernel, in + /// this boot. + pub fn observe(pid: u32) -> io::Result { + let stat = stat(pid)?; + Ok(Self { + pid, + start: stat.start, + boot: Some(boot()?), + }) + } + /// True while this exact process exists and has not exited: a zombie + /// (or a dead entry) is not alive, and another process under the same id + /// has another start time or another boot. + pub fn alive(self) -> bool { + let this_boot = self + .boot + .is_none_or(|recorded| boot().ok() == Some(recorded)); + match stat(self.pid) { + Ok(stat) => this_boot && stat.start == self.start && !stat.ended(), + Err(_) => false, + } + } + /// Sends `signal` once, only after this exact process was observed alive. + pub(super) fn signal(self, signal: Signal) -> io::Result<()> { + if self.boot.is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "a local process recorded without its boot is never signalled", + )); + } + if !self.alive() { + return Ok(()); + } + match nix::sys::signal::kill(pid(self.pid)?, signal) { + Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), + Err(errno) => Err(errno.into()), + } + } +} + +/// What this kernel says of one process. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Stat { + pub(crate) state: u8, + pub(crate) group: u32, + pub(crate) start: u64, +} + +impl Stat { + /// A zombie or a dead entry: it runs nothing any more. + pub(crate) fn ended(self) -> bool { + matches!(self.state, b'Z' | b'X' | b'x') + } +} + +/// A process id as the kernel's signed type. +pub(super) fn pid(id: u32) -> io::Result { + i32::try_from(id) + .map(Pid::from_raw) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id width")) +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/provider.rs b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs new file mode 100644 index 0000000000..b5e48d9916 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/provider.rs @@ -0,0 +1,178 @@ +//! The existing native runtime as a concrete backend; no alternate process engine. +use super::{Local, LocalPhase, LocalProcess, Sandbox}; +use crate::provider::{ + IsolationModel, LaunchFailure, LaunchPhase, LaunchRequest, LocalRecord, MountDriver, + ProviderCapabilities, ProviderLaunchOptions, SandboxRecord, SearchPathPolicy, +}; +use crate::{ContainerId, RuntimeError, WireFailure}; +use std::path::Path; + +#[derive(Clone, Debug)] +pub struct NativeProvider { + local: Local, +} +/// Native macOS processes with loopback NFS and the caller's identity. +pub type NativeMacosProvider = NativeProvider; +/// Native Linux processes with FUSE and a separate nonroot command identity. +pub type NativeLinuxProvider = NativeProvider; +/// The existing local owner, using Darwin process facts. +pub type NativeMacosInstance = Sandbox; +/// The existing local owner, using Linux `/proc` process facts. +pub type NativeLinuxInstance = Sandbox; +impl NativeProvider { + /// Refuses other platforms before accessing the root or launching anything. + pub fn new(root: impl AsRef) -> Result { + if (LINUX && !cfg!(target_os = "linux")) || (!LINUX && !cfg!(target_os = "macos")) { + return Err(RuntimeError::Unsupported("native provider platform")); + } + if LINUX && !nix::unistd::Uid::effective().is_root() { + return Err(RuntimeError::Unsupported( + "native Linux provider requires root", + )); + } + Ok(Self { + local: if LINUX { + Local::new(root)? + } else { + Local::native(root)? + }, + }) + } + pub fn paths(&self, name: &str) -> Result { + self.local.clone().naming(name)?.paths() + } + pub fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + name: if LINUX { + "native-linux" + } else { + "native-macos" + }, + isolation: IsolationModel::HostProcess, + mount_driver: if LINUX { + MountDriver::Fuse + } else { + MountDriver::LoopbackNfs + }, + requires_image: false, + search_path_policy: if LINUX { + SearchPathPolicy::FixedContainer + } else { + SearchPathPolicy::InheritHost + }, + supports_resource_limits: false, + direct_store_access: true, + } + } + pub fn launch(&self, request: LaunchRequest) -> Result> { + let fail = |cause| { + Box::new(LaunchFailure { + phase: LaunchPhase::Validate, + instance: None, + cause, + }) + }; + let ProviderLaunchOptions::Local { + command_identity, + mut environment, + } = request.options + else { + return Err(fail(RuntimeError::Unsupported( + "native provider requires local launch options", + ))); + }; + command_identity.check().map_err(fail)?; + if (!LINUX + && (command_identity.uid != nix::unistd::Uid::effective().as_raw() + || command_identity.gid != nix::unistd::Gid::effective().as_raw())) + || environment.iter().any(|(key, value)| { + key.is_empty() + || key.contains(['=', '\0']) + || value.contains('\0') + || key == "PATH" + || (key == "LAYERFS_KERNEL" && value != if LINUX { "fuse" } else { "nfs" }) + }) + { + return Err(fail(RuntimeError::Protocol( + "native command identity/environment", + ))); + } + environment.retain(|(key, _)| key != "LAYERFS_KERNEL"); + environment.push(( + "LAYERFS_KERNEL".into(), + if LINUX { "fuse" } else { "nfs" }.into(), + )); + let local = self + .local + .clone() + .naming(&request.name) + .map_err(fail)? + .environment(environment); + local + .create( + &request.daemon_executable, + &request.daemon_config, + request.startup_timeout, + request.labels, + ) + .map_err(|failure| { + Box::new(LaunchFailure { + phase: match failure.phase { + LocalPhase::Validate => LaunchPhase::Validate, + LocalPhase::Create => LaunchPhase::Create, + LocalPhase::Start => LaunchPhase::Start, + LocalPhase::Listener => LaunchPhase::Listener, + }, + instance: failure + .sandbox + .map(|sandbox| Box::new(crate::backend::Sandbox::Native(sandbox))), + cause: failure.cause, + }) + }) + } + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Sandbox::reopen( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + ) + } + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + let Some(record) = record.local() else { + return Err(WireFailure::local(RuntimeError::Unsupported( + "native provider requires a local record", + ))); + }; + let paths = self.paths(&record.name).map_err(WireFailure::local)?; + Ok(Sandbox::stopped( + paths.directory, + id, + process(record), + record.port, + record.labels.clone(), + )) + } +} +fn process(record: &LocalRecord) -> LocalProcess { + LocalProcess { + pid: record.pid, + start: record.start_time, + boot: record.boot, + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs new file mode 100644 index 0000000000..8c67717882 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/sandbox.rs @@ -0,0 +1,335 @@ +//! One local Sandbox: its directory, its daemon process and its exact +//! attempts. No Drop cleanup or replay, as for a container; a Drop only +//! reaps a daemon this process started that has already exited. A stop +//! returns only once a daemon this process started is reaped, so it never +//! stays a zombie of this process. +use super::{groups, platform, process::LocalProcess}; +use crate::provider::LocalRecord; +use crate::{ContainerId, RuntimeError, WireFailure}; +use nix::sys::signal::Signal; +use std::{ + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + net::{Ipv4Addr, SocketAddr}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + process::Child, + sync::Mutex, + time::{Duration, Instant}, +}; + +/// The daemon's output file in a Sandbox directory, stdout and stderr together. +pub(super) const LOG: &str = "daemon.log"; +/// Pause between two looks at a stopping process. +const PAUSE: Duration = Duration::from_millis(20); +/// Wait for a killed process to be gone. +const KILL_WAIT: Duration = Duration::from_secs(5); + +/// One acknowledged local Sandbox directory and, once started, its daemon. +#[derive(Debug)] +pub struct Sandbox { + pub(super) id: ContainerId, + pub(super) directory: PathBuf, + pub(super) labels: Vec<(String, String)>, + pub(super) process: Option, + pub(super) port: u16, + /// The daemon this process started, kept so its exit is reaped here and + /// it does not stay a zombie of this process. `None` after a reopen. + pub(super) child: Mutex>, + pub(super) stop_attempted: bool, + pub(super) stopped: bool, + /// Its process groups were killed and found empty (by its stop). + pub(super) ended: bool, + pub(super) delete_attempted: bool, + pub(super) deleted: bool, +} + +impl Sandbox { + /// A directory made by a start that has not spawned its daemon yet. + pub(super) fn unstarted( + id: ContainerId, + directory: PathBuf, + labels: Vec<(String, String)>, + ) -> Self { + Self { + id, + directory, + labels, + process: None, + port: 0, + child: Mutex::new(None), + stop_attempted: false, + stopped: false, + ended: false, + delete_attempted: false, + deleted: false, + } + } + /// Reopens a recorded Sandbox whose daemon still runs: the same process + /// (id and start time) and its loopback port. It starts nothing. + pub fn reopen( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Result> { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + if !owner.directory.is_absolute() || port == 0 { + return Err(owner.invalid("saved local Sandbox")); + } + owner.endpoint()?; + Ok(owner) + } + /// A recorded Sandbox that [`Self::reopen`] found stopped + /// ([`WireFailure::container_stopped`]): only its one Delete is admitted. + pub fn stopped( + directory: PathBuf, + id: ContainerId, + process: LocalProcess, + port: u16, + labels: Vec<(String, String)>, + ) -> Self { + let mut owner = Self::unstarted(id, directory, labels); + owner.process = Some(process); + owner.port = port; + owner.stop_attempted = true; + owner.stopped = true; + owner + } + /// Its identity: random at create, recorded, never an Engine's. + pub fn identity(&self) -> ContainerId { + self.id + } + /// Its private directory (setup, overlay, Workspace mounts, daemon log). + pub fn directory(&self) -> &Path { + &self.directory + } + /// Its commands' scratch directory (1777), deleted with it. + pub fn tmp(&self) -> PathBuf { + self.directory.join("tmp") + } + /// Its daemon process, once started. + pub fn process(&self) -> Option { + self.process + } + /// The loopback port its daemon listens on, once observed. + pub fn port(&self) -> u16 { + self.port + } + /// The labels it was created with, kept in its record. + pub fn labels(&self) -> &[(String, String)] { + &self.labels + } + /// The facts its record keeps, for reopening this exact Sandbox later. + pub fn local_record(&self) -> Option { + let process = self.process()?; + Some(LocalRecord { + name: self.directory().file_name()?.to_str()?.to_owned(), + pid: process.pid, + start_time: process.start, + boot: process.boot, + port: self.port(), + labels: self.labels().to_vec(), + }) + } + /// The daemon's endpoint while that exact process runs. A daemon that + /// exited (idle close, crash, kill) is the stopped observation + /// ([`WireFailure::container_stopped`]): its Workspaces went with it. + pub fn endpoint(&self) -> Result> { + let Some(process) = self.process else { + return Err(self.invalid("running endpoint admission")); + }; + if self.port == 0 || self.stop_attempted || self.delete_attempted { + return Err(self.invalid("running endpoint admission")); + } + self.reap(); + if !process.alive() { + let mut failure = self.invalid(crate::types::STOPPED); + failure.observed_container = Some(self.id); + return Err(failure); + } + Ok(SocketAddr::from((Ipv4Addr::LOCALHOST, self.port))) + } + /// Detaches its mounts while the daemon can still answer, then stops + /// the daemon once: TERM, up to `seconds` for it to end, then + /// KILL (a daemon this process started has ended once reaped); then + /// KILL to every process group of its commands and its daemon, as a + /// container's end kills what ran in it. A daemon already gone is the + /// same stopped state. Crash teardown, not a graceful filesystem drain. + pub fn stop(&mut self, seconds: u16) -> Result<(), Box> { + if self.stop_attempted || self.delete_attempted { + return Err(self.invalid("original local Stop admission")); + } + self.stop_attempted = true; + // A live NFS server can finish client writeback during detach. A + // failed detach does not prevent crash teardown; Delete retains its + // existing checked detach before removing anything through a mount. + if let Err(error) = platform::mounts::detach(&self.directory) { + eprintln!("local Stop detach {}: {error}", self.directory.display()); + } + if let Some(process) = self.process { + self.end( + process, + Signal::SIGTERM, + Duration::from_secs(u64::from(seconds)), + )?; + if !self.gone(process) { + self.end(process, Signal::SIGKILL, KILL_WAIT)?; + } + if process.alive() { + return Err(self.invalid("local daemon still runs after KILL")); + } + } + self.stopped = true; + groups::end(&self.directory, self.process, KILL_WAIT).map_err(|error| self.io(error))?; + self.ended = true; + Ok(()) + } + /// Deletes its directory once, after a known stop: any process left in + /// its groups is killed first (when its stop did not run here, as for a + /// daemon found stopped), and any mount its daemon left is detached, so + /// nothing writes into it or is removed through a mount. The shared + /// Store is outside it and never touched. + pub fn delete(&mut self) -> Result<(), Box> { + if self.delete_attempted || (self.process.is_some() && !self.stopped) { + return Err(self.invalid("original local Delete admission; known stop required")); + } + self.delete_attempted = true; + self.reap(); + if self.process.is_some_and(LocalProcess::alive) { + return Err(self.invalid("local daemon runs again; nothing was deleted")); + } + if !self.ended { + groups::end(&self.directory, self.process, KILL_WAIT) + .map_err(|error| self.io(error))?; + self.ended = true; + } + platform::mounts::detach(&self.directory).map_err(|error| self.io(error))?; + match fs::remove_dir_all(&self.directory) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(self.io(error)), + } + self.deleted = true; + Ok(()) + } + /// The daemon output's last `lines` lines, keeping only the final `cap` + /// bytes. Read-only and never retried. + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + if self.delete_attempted { + return Err(self.invalid("log read before Delete")); + } + let read = || -> io::Result> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(nix::fcntl::OFlag::O_NOFOLLOW.bits()) + .open(self.directory.join(LOG))?; + let length = file.metadata()?.len(); + // a line is rarely longer than this; the window bounds the read + let window = (cap as u64).saturating_mul(4).max(64 * 1024); + file.seek(SeekFrom::Start(length.saturating_sub(window)))?; + let mut bytes = Vec::new(); + file.take(window).read_to_end(&mut bytes)?; + Ok(bytes) + }; + let bytes = read().map_err(|error| self.io(error))?; + let body = bytes.strip_suffix(b"\n").unwrap_or(&bytes); + let start = (body.iter().enumerate().rev()) + .filter(|(_, b)| **b == b'\n') + .nth(usize::from(lines).saturating_sub(1)) + .map_or(0, |(i, _)| i + 1); + let tail = &bytes[start..]; + Ok(tail[tail.len().saturating_sub(cap)..].to_vec()) + } + /// Names the process group `pid` leads or is about to lead: a process + /// started at `start` by a command of this Sandbox in a session of its + /// own (the SDK's `setsid` background process), so it ends with the + /// Sandbox too. One already gone needs nothing. + pub fn track(&self, pid: u32, start: u64) -> Result<(), Box> { + match LocalProcess::observe(pid) { + Ok(process) if process.start == start && process.alive() => { + groups::name(&self.directory, process).map_err(|error| self.io(error)) + } + _ => Ok(()), + } + } + /// Reaps the daemon this process started, if it exited. True when no + /// daemon this process started is left unreaped. + pub(crate) fn reap(&self) -> bool { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + let Some(started) = child.as_mut() else { + return true; + }; + let reaped = matches!(started.try_wait(), Ok(Some(_))); + if reaped { + *child = None; + } + reaped + } + /// The daemon has ended and, if this process started it, is reaped. Its + /// main thread can end (a zombie id, so not alive) while its other + /// threads still run and it cannot be reaped yet: not gone. + fn gone(&self, process: LocalProcess) -> bool { + self.reap() && !process.alive() + } + /// Sends `signal` once and waits up to `wait` for the process to be gone. + /// A KILL also goes to the daemon this process started and has not + /// reaped: its id cannot be reused until reaped, so it is still the one + /// started, though its main thread may have ended already. + fn end( + &self, + process: LocalProcess, + signal: Signal, + wait: Duration, + ) -> Result<(), Box> { + process.signal(signal).map_err(|error| self.io(error))?; + if signal == Signal::SIGKILL { + let mut child = self.child.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(started) = child.as_mut() { + let _ = started.kill(); + } + } + let deadline = Instant::now() + wait; + loop { + if self.gone(process) || Instant::now() >= deadline { + return Ok(()); + } + std::thread::sleep(PAUSE); + } + } + pub(super) fn invalid(&self, message: &'static str) -> Box { + self.failure(RuntimeError::Protocol(message)) + } + pub(super) fn io(&self, error: io::Error) -> Box { + self.failure(RuntimeError::Io(error)) + } + pub(super) fn failure(&self, cause: RuntimeError) -> Box { + Box::new(WireFailure { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: Some(self.id), + requested_exec: None, + error_body: None, + }) + } +} + +impl Drop for Sandbox { + /// A daemon this process started and that has exited is reaped, so a + /// Sandbox dropped on any path leaves no zombie of it. A running daemon + /// is left alone: stopping it is [`Sandbox::stop`]'s, once. + fn drop(&mut self) { + self.reap(); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/native/search.rs b/engine/crates/layerfs-sandbox/src/backend/native/search.rs new file mode 100644 index 0000000000..3db862f089 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/native/search.rs @@ -0,0 +1,26 @@ +//! The search path a local daemon and its commands start with: a container's +//! on Linux, this process's own on macOS (the native route). +use std::ffi::OsString; + +/// A container's search path: what a local daemon and its commands start with +/// on Linux, where the local runtime stands in for a container. +const CONTAINER_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on Linux [`CONTAINER_PATH`], whatever this +/// process has. +#[cfg(not(target_os = "macos"))] +pub(crate) fn search_path() -> OsString { + OsString::from(CONTAINER_PATH) +} + +/// The search path a local daemon and its commands start with, before a +/// command's own environment: on macOS this process's own `PATH`; without +/// one, or an empty one, [`CONTAINER_PATH`] and Homebrew's `/opt/homebrew/bin`. +#[cfg(target_os = "macos")] +pub(crate) fn search_path() -> OsString { + match std::env::var_os("PATH") { + Some(path) if !path.is_empty() => path, + _ => OsString::from(format!("{CONTAINER_PATH}:/opt/homebrew/bin")), + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/runtime.rs b/engine/crates/layerfs-sandbox/src/backend/runtime.rs new file mode 100644 index 0000000000..43ed8208a0 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/runtime.rs @@ -0,0 +1,307 @@ +//! The concrete runtimes a Sandbox can run on: one enum names the deployment's +//! choice, and [`Sandbox`] is the launched owner over it. There is no provider +//! trait; each backend is a concrete type and dispatch is a `match` here. +use super::docker::{self, DockerProvider}; +#[cfg(target_os = "linux")] +use super::namespaces::{self, NamespaceProvider}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use super::native::{self, NativeLinuxProvider, NativeMacosProvider}; +use crate::provider::{ + ExecExit, LaunchFailure, LaunchRequest, LocalRecord, ProviderCapabilities, SandboxInspection, + SandboxRecord, +}; +use crate::{ContainerId, ExecRequest, RuntimeError, WireFailure}; +use std::{io::Write, net::SocketAddr, path::PathBuf}; + +/// What Windows would need and does not have yet: no host runtime exists. +const WINDOWS: &str = "windows native Sandbox runtime"; + +/// One deployment's runtime choice, kept for every Sandbox it starts. +/// +/// A Sandbox's record names the variant that made it; a platform without a +/// runtime is refused before any effect, never replaced by another backend. +#[derive(Clone, Debug)] +pub enum Runtime { + /// A local Docker Engine container; the default. + Docker(DockerProvider), + /// A local daemon process serving loopback NFS as this user (macOS). + #[cfg(any(target_os = "linux", target_os = "macos"))] + MacosNative(NativeMacosProvider), + /// A root local daemon process serving FUSE, commands as a nonroot + /// identity (Linux). + #[cfg(any(target_os = "linux", target_os = "macos"))] + LinuxNative(NativeLinuxProvider), + /// A root daemon in its own mount, PID and IPC namespaces and cgroup, + /// commands entering them as a nonroot identity (Linux). + #[cfg(target_os = "linux")] + Namespaces(NamespaceProvider), + /// Placeholder: selecting Windows refuses before any effect. + Windows, +} + +impl Runtime { + pub fn capabilities(&self) -> Result { + match self { + Self::Docker(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => Ok(provider.capabilities()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => Ok(provider.capabilities()), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => Ok(provider.capabilities()), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// Local resource ownership, checked by API before encoding daemon topology. + /// Nonlocal providers return None. No resource is created by this observation. + pub fn local_directory(&self, name: &str) -> Result, RuntimeError> { + match self { + Self::Docker(_) => Ok(None), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name).map(|paths| Some(paths.directory)), + Self::Windows => Err(RuntimeError::Unsupported(WINDOWS)), + } + } + + /// The named native Sandbox's paths, for its daemon setup. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn native_paths(&self, name: &str) -> Result { + match self { + Self::MacosNative(provider) => provider.paths(name), + Self::LinuxNative(provider) => provider.paths(name), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider.paths(name), + Self::Docker(_) | Self::Windows => Err(RuntimeError::Unsupported( + "native Sandbox paths need a native runtime", + )), + } + } + + /// Starts one Sandbox once; a failed launch keeps the acknowledged owner. + pub fn launch(&self, request: LaunchRequest) -> Result> { + match self { + Self::Docker(provider) => provider.launch(request).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.launch(request).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .launch(request) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(Box::new(LaunchFailure { + phase: crate::provider::LaunchPhase::Validate, + instance: None, + cause: RuntimeError::Unsupported(WINDOWS), + })), + } + } + + /// Reopen starts nothing and checks the recorded deployment or process identity. + pub fn reopen( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } + + /// A known stopped record admits only the existing runtime's single Delete. + pub fn reopen_stopped( + &self, + id: ContainerId, + record: &SandboxRecord, + ) -> Result> { + match self { + Self::Docker(provider) => provider.reopen_stopped(id, record).map(Sandbox::Docker), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::MacosNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::LinuxNative(provider) => provider.reopen_stopped(id, record).map(Sandbox::Native), + #[cfg(target_os = "linux")] + Self::Namespaces(provider) => provider + .reopen_stopped(id, record) + .map(|sandbox| Sandbox::Namespaces(Box::new(sandbox))), + Self::Windows => Err(WireFailure::local(RuntimeError::Unsupported(WINDOWS))), + } + } +} + +/// The launched owner of one Sandbox, on the runtime that started it. +#[derive(Debug)] +pub enum Sandbox { + Docker(docker::Sandbox), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Native(native::Sandbox), + #[cfg(target_os = "linux")] + Namespaces(Box), +} + +impl Sandbox { + pub fn identity(&self) -> ContainerId { + match self { + Self::Docker(sandbox) => sandbox.identity(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.identity(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.identity(), + } + } + + pub fn endpoint(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.endpoint(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.endpoint(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.endpoint(), + } + } + + /// One command to its end, its streams copied to the sinks as they arrive. + pub fn run( + &self, + request: &ExecRequest, + stdout: &mut (dyn Write + Send), + stderr: &mut (dyn Write + Send), + ) -> Result> { + match self { + Self::Docker(sandbox) => docker::run_command(sandbox, request, stdout, stderr), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + let exit = sandbox.run(request, stdout, stderr)?; + Ok(ExecExit { + code: exit.code, + pid: Some(exit.pid), + stdout_bytes: exit.stdout, + stderr_bytes: exit.stderr, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.run(request, stdout, stderr), + } + } + + /// Names a background process a command started in a session of its own, + /// so it ends with the Sandbox as a container's descendants do. + pub fn track(&self, pid: u32, start_time: u64) -> Result<(), Box> { + match self { + // Container teardown owns all descendants; no host process is adopted. + Self::Docker(_) => Ok(()), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.track(pid, start_time), + // its cgroup owns every process the Sandbox started, whatever + // session it left, so nothing needs naming + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.track(pid, start_time), + } + } + + pub fn log_tail(&self, lines: u16, cap: usize) -> Result, Box> { + match self { + Self::Docker(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.log_tail(lines, cap), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.log_tail(lines, cap), + } + } + + pub fn inspect(&self) -> Result> { + match self { + Self::Docker(sandbox) => sandbox.inspect(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => { + sandbox.reap(); + Ok(SandboxInspection { + identity: sandbox.identity(), + running: sandbox.process().is_some_and(|p| p.alive()), + exit_code: None, + process_id: sandbox.process().map(|p| p.pid), + scratch_bytes: None, + }) + } + } + } + + pub fn stop(&mut self, timeout_seconds: u16) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.stop(timeout_seconds), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.stop(timeout_seconds), + } + } + + pub fn delete(&mut self) -> Result<(), Box> { + match self { + Self::Docker(sandbox) => sandbox.delete(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.delete(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.delete(), + } + } + + /// Where its commands keep scratch files: `/tmp` in a container, the + /// Sandbox's own directory for a native one, deleted with it. + pub fn scratch_directory(&self) -> String { + match self { + Self::Docker(_) => "/tmp".into(), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.tmp().display().to_string(), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.tmp().display().to_string(), + } + } + + /// Physical facts for the binding's existing record format, when this is native. + pub fn local_record(&self) -> Option { + match self { + Self::Docker(_) => None, + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(sandbox) => sandbox.local_record(), + #[cfg(target_os = "linux")] + Self::Namespaces(_) => None, + } + } + + /// Persisted deployment facts for reopening this runtime instance. + pub fn record(&self) -> Option { + match self { + Self::Docker(sandbox) => Some(SandboxRecord::Docker(sandbox.request().clone().into())), + #[cfg(any(target_os = "linux", target_os = "macos"))] + Self::Native(_) => self.local_record().map(SandboxRecord::Local), + #[cfg(target_os = "linux")] + Self::Namespaces(sandbox) => sandbox.record(), + } + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/streams.rs b/engine/crates/layerfs-sandbox/src/backend/streams.rs new file mode 100644 index 0000000000..6c6e011a17 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/streams.rs @@ -0,0 +1,23 @@ +//! An ordinary command's streams, copied to the caller's sinks as they +//! arrive. Both local backends read the same two pipes and count only what +//! each sink accepted. +use std::io::{self, Read, Write}; + +/// Copies one stream to its sink until end of stream; the bytes delivered. +pub(crate) fn copy(source: Option, sink: &mut (dyn Write + Send)) -> io::Result { + let Some(mut source) = source else { + return Ok(0); + }; + let mut total = 0u64; + let mut window = [0u8; 8192]; + loop { + let n = match source.read(&mut window) { + Ok(0) => return Ok(total), + Ok(n) => n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + }; + sink.write_all(&window[..n])?; + total = total.saturating_add(n as u64); + } +} diff --git a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs b/engine/crates/layerfs-sandbox/src/backend/strings.rs similarity index 86% rename from engine/crates/layerfs-sandbox/src/backend/docker/strings.rs rename to engine/crates/layerfs-sandbox/src/backend/strings.rs index 1d2afc32e8..fb22cf7c79 100644 --- a/engine/crates/layerfs-sandbox/src/backend/docker/strings.rs +++ b/engine/crates/layerfs-sandbox/src/backend/strings.rs @@ -41,6 +41,27 @@ impl Json { self.string_each(|c| text.push(c, capture))?; Ok(text) } + /// Reads a string value into `out` as UTF-8, refusing one longer than + /// `max`: the Engine's selected fields are short, and the longest string + /// this crate reads is an image archive's path. Its only caller unpacks + /// images, which this crate does on Linux only. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn text_into(&mut self, out: &mut Vec, max: usize) -> Result<(), RuntimeError> { + let mut too_long = false; + self.string_each(|c| { + let mut encoded = [0; 4]; + let bytes = c.encode_utf8(&mut encoded).as_bytes(); + if out.len() + bytes.len() > max { + too_long = true; + } else { + out.extend_from_slice(bytes); + } + })?; + if too_long { + return Err(RuntimeError::Protocol("JSON string length")); + } + Ok(()) + } pub fn string_equals(&mut self, expected: &str) -> Result { let mut expected = expected.chars(); let mut equal = true; diff --git a/engine/crates/layerfs-sandbox/src/backend/windows/README.md b/engine/crates/layerfs-sandbox/src/backend/windows/README.md new file mode 100644 index 0000000000..b667620c81 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/backend/windows/README.md @@ -0,0 +1,49 @@ +# Windows：占位（尚未实现） + +状态：占位。选到 `Runtime::Windows` 时返回既有的“不支持此平台”错误 +（`RuntimeError::Unsupported`），不 panic、不新建公开错误码、不产生任何副作用。 + +## 现在有什么 + +- `layerfs_sandbox::Runtime` 的四个取值：`Docker`、`MacosNative`、 + `LinuxNative`、`Windows`。前三个已有后端，Windows 只有这个变体。 +- `Runtime::{capabilities, local_directory, launch, reopen, reopen_stopped}` + 在 Windows 上全部返回 `Unsupported`；`Sandbox` 句柄没有 Windows 分支， + 因为 Windows 上不可能启动出实例。 +- `layerfs-sandbox` 的 `backend` 只在 unix 下编译，`layerfs-node` 也只用 + unix 系统调用，所以今天在 Windows 主机上选不到这个变体：它存在的意义是 + 把“没有 Windows 宿主运行时”写进类型，而不是让调用方自己判平台。 + +## 为什么不做“Windows 本地进程”后端 + +macOS 原生的前提是：daemon 以本用户身份跑 NFS 回环服务，命令以同一身份 +运行，一切在同一个内核里。Windows 上没有等价物：LayerFS 的 FUSE 服务端是 +Linux 内核特性，Store 的 WAL 与目录锁只在一个内核内协调（见 #352 直接 +Store 访问的安全条件），命令身份、挂载可见性和进程组语义都要另建一套。 +因此 Windows 的路线不是“再写一个本地后端”，而是把 Linux 侧的能力搬到一个 +Linux 宿主里。 + +## 计划方向 + +两条路线，都由 #515 的 placement 设置表达，而不是在 Windows 本机上跑 daemon： + +1. **WSL2 分发（近期）**：Windows 上的集成适配器把 SDK 调用转发给 + 每个 LayerFS home 一个的、仅回环且带密钥的 worker；worker 在一个选定的 + WSL2 Linux 分发里直接用真实 SDK 打开共享 Store（ext4/xfs/btrfs）， + 并用该分发的 Docker Engine 起 Sandbox daemon。Windows 只拥有 UI、绑定和 + trace，Linux 拥有本地句柄和项目发现。这是仓库根 `AGENTS.md` 已记录的 + Windows 例外，规则不变：只连回环、每次调用只尝试一次、丢失的回复不重放、 + 结果不确定的 Init 保留其 Linux home 和卷。 +2. **Hyper-V / VM 方向（后续）**：每个 Store 一个 Linux VM，Sandbox 是 VM 内 + 的命名空间；宿主侧只通过一条私有流连到 VM 内的 worker。见 #515 §6 + （`vm:` placement 与各 provider 的 VM/磁盘/私有流对照表）和 + #512 的实验结论。 + +## 实现时要先定的事 + +- placement 与后端选择的设置名（#515 V9 提议 `LAYERFS_PLACEMENT` 与 + `LAYERFS_SANDBOX_BACKEND`，只读 `LAYERFS_*`；集成自己的前缀由集成映射）。 +- Windows 上 `deploy.sandbox`、`deploy.localDir` 等现有设置的语义：#515 + 要求 product SDK 与 daemon 不为 Windows 改动，Windows 差异留在适配器里。 +- 到那时 `Runtime` 是否新增一个持有 worker 端点的变体（那时才需要数据）， + 以及记录里如何标注该 Sandbox 由哪个运行时创建（无字段即 Docker）。 diff --git a/engine/crates/layerfs-sandbox/src/lib.rs b/engine/crates/layerfs-sandbox/src/lib.rs index d1b8a35e44..497fa2024b 100644 --- a/engine/crates/layerfs-sandbox/src/lib.rs +++ b/engine/crates/layerfs-sandbox/src/lib.rs @@ -1,9 +1,14 @@ //! Ordinary runtime operations, separate from filesystem admission and data ownership. -#![forbid(unsafe_code)] +// unsafe only in backend/macos/darwin.rs: macOS process facts come from +// proc_pidinfo, a system call (`ps` is setuid, which Seatbelt refuses) +#![deny(unsafe_code)] #[cfg(unix)] pub mod backend; +pub mod provider; mod types; +#[cfg(unix)] +pub use backend::{Runtime, Sandbox}; pub use types::{ - CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, PendingRequest, - RuntimeError, WireFailure, + CommandIdentity, ContainerId, ErrorDiagnostic, ExecId, ExecInspection, ExecRequest, Image, + PendingRequest, RuntimeError, WireFailure, }; diff --git a/engine/crates/layerfs-sandbox/src/provider/capabilities.rs b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs new file mode 100644 index 0000000000..a11548c3eb --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/capabilities.rs @@ -0,0 +1,30 @@ +//! Explicit runtime capabilities; these describe support, not qualification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProviderCapabilities { + pub name: &'static str, + pub isolation: IsolationModel, + pub mount_driver: MountDriver, + pub requires_image: bool, + pub search_path_policy: SearchPathPolicy, + pub supports_resource_limits: bool, + pub direct_store_access: bool, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum IsolationModel { + Container, + HostProcess, + MicroVm, + RemoteCloud, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MountDriver { + Fuse, + LoopbackNfs, + SharedFs, + None, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchPathPolicy { + FixedContainer, + InheritHost, +} diff --git a/engine/crates/layerfs-sandbox/src/provider/launch.rs b/engine/crates/layerfs-sandbox/src/provider/launch.rs new file mode 100644 index 0000000000..0c654ad481 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/launch.rs @@ -0,0 +1,26 @@ +//! One-attempt launch failure: acknowledged partial ownership and the last phase reached. +use crate::backend::Sandbox; +use crate::RuntimeError; +use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LaunchPhase { + Validate, + Create, + Start, + Listener, +} +/// The caller retains the exact resource even when launch fails after creating it. +pub struct LaunchFailure { + pub phase: LaunchPhase, + pub instance: Option>, + pub cause: RuntimeError, +} +impl fmt::Debug for LaunchFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchFailure") + .field("phase", &self.phase) + .field("identity", &self.instance.as_ref().map(|i| i.identity())) + .field("cause", &self.cause) + .finish() + } +} diff --git a/engine/crates/layerfs-sandbox/src/provider/mod.rs b/engine/crates/layerfs-sandbox/src/provider/mod.rs new file mode 100644 index 0000000000..b9eee1e304 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/mod.rs @@ -0,0 +1,12 @@ +//! Runtime inputs and observations; concrete backends own their own lifecycle. +mod capabilities; +mod launch; +mod options; +pub use capabilities::{IsolationModel, MountDriver, ProviderCapabilities, SearchPathPolicy}; +pub use launch::{LaunchFailure, LaunchPhase}; +pub use options::{ + DockerRecord, ExecExit, LaunchRequest, LocalRecord, NamespaceRecord, ProviderLaunchOptions, + SandboxInspection, SandboxRecord, +}; +/// Compatible name for the existing exact 64-hex runtime identity. +pub type SandboxId = crate::ContainerId; diff --git a/engine/crates/layerfs-sandbox/src/provider/options.rs b/engine/crates/layerfs-sandbox/src/provider/options.rs new file mode 100644 index 0000000000..0ea23bd979 --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/provider/options.rs @@ -0,0 +1,193 @@ +//! Launch inputs and observations independent of the daemon control protocol. +use crate::{CommandIdentity, ContainerId}; +use std::{fmt, path::PathBuf, time::Duration}; +/// Configuration bytes may contain secrets; Debug only reports their length. +#[derive(Clone)] +pub struct LaunchRequest { + pub name: String, + pub daemon_executable: PathBuf, + pub daemon_config: Vec, + pub labels: Vec<(String, String)>, + pub startup_timeout: Duration, + pub options: ProviderLaunchOptions, +} +impl fmt::Debug for LaunchRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LaunchRequest") + .field("name", &self.name) + .field("daemon_executable", &self.daemon_executable) + .field("config_bytes", &self.daemon_config.len()) + .field("startup_timeout", &self.startup_timeout) + .finish_non_exhaustive() + } +} +#[derive(Clone, Debug)] +pub enum ProviderLaunchOptions { + Local { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + }, + /// The Linux namespace backend: a namespaced daemon plus the Store file it + /// opens. `store` is the exact file; its directory is what the Sandbox's + /// private root binds, so SQLite's WAL and the writers' directory lock see + /// the same inode as every other opener on this kernel. `image` is the + /// digest its root is an overlay of, and `archive` the docker-archive in + /// the bundle that digest names, which is unpacked once per digest. + Namespaces { + command_identity: CommandIdentity, + environment: Vec<(String, String)>, + store: PathBuf, + image: crate::Image, + archive: PathBuf, + }, + Docker { + image: String, + store_volume: String, + port: u16, + connect_host: Option, + }, +} +/// Persisted physical facts supplied by the record owner; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalRecord { + pub name: String, + pub pid: u32, + pub start_time: u64, + pub boot: Option, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// A namespaced Linux Sandbox's record: the local facts (its directory name, +/// init process identity and listener port) and the image digest its root is +/// an overlay of, so a reopen finds the same unpacked image. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamespaceRecord { + pub local: LocalRecord, + pub image: crate::Image, +} +impl NamespaceRecord { + /// The physical local facts every record of this backend keeps. + pub fn local(&self) -> &LocalRecord { + &self.local + } + /// The image digest the Sandbox's root is an overlay of. + pub fn image(&self) -> &crate::Image { + &self.image + } +} +/// Persisted deployment facts for a Docker container; no filesystem discovery here. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DockerRecord { + pub image: String, + pub store_volume: String, + pub port: u16, + pub labels: Vec<(String, String)>, +} +/// Recorded runtime identity and deployment facts for reopen. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SandboxRecord { + Local(LocalRecord), + /// A namespaced Linux Sandbox; the physical facts kept are the same as a + /// local one (its directory name, init process identity and listener port), + /// plus the image digest its root is an overlay of. The variant itself + /// names the backend, so a record is never reopened by another one. + Namespaces(NamespaceRecord), + Docker(DockerRecord), +} +impl SandboxRecord { + pub fn local(&self) -> Option<&LocalRecord> { + match self { + Self::Local(record) => Some(record), + _ => None, + } + } + pub fn namespaces(&self) -> Option<&NamespaceRecord> { + match self { + Self::Namespaces(record) => Some(record), + _ => None, + } + } + pub fn docker(&self) -> Option<&DockerRecord> { + match self { + Self::Docker(record) => Some(record), + _ => None, + } + } +} +impl From for SandboxRecord { + fn from(record: LocalRecord) -> Self { + Self::Local(record) + } +} +impl From<&LocalRecord> for SandboxRecord { + fn from(record: &LocalRecord) -> Self { + Self::Local(record.clone()) + } +} +impl From for SandboxRecord { + fn from(record: DockerRecord) -> Self { + Self::Docker(record) + } +} +impl From<&DockerRecord> for SandboxRecord { + fn from(record: &DockerRecord) -> Self { + Self::Docker(record.clone()) + } +} +impl From for DockerRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image, + store_volume: request.store_volume, + port: request.port, + labels: request.labels, + } + } +} +impl From<&crate::backend::docker::SandboxRequest> for DockerRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self { + image: request.image.clone(), + store_volume: request.store_volume.clone(), + port: request.port, + labels: request.labels.clone(), + } + } +} +impl From for crate::backend::docker::SandboxRequest { + fn from(record: DockerRecord) -> Self { + Self { + image: record.image, + store_volume: record.store_volume, + port: record.port, + labels: record.labels, + } + } +} +impl From for SandboxRecord { + fn from(request: crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +impl From<&crate::backend::docker::SandboxRequest> for SandboxRecord { + fn from(request: &crate::backend::docker::SandboxRequest) -> Self { + Self::Docker(request.into()) + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExecExit { + pub code: i32, + /// Docker may not expose a process id after exit. + pub pid: Option, + pub stdout_bytes: u64, + pub stderr_bytes: u64, +} +/// Unknown resource/timestamp fields stay None rather than invented zeroes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SandboxInspection { + pub identity: ContainerId, + pub running: bool, + pub exit_code: Option, + pub process_id: Option, + pub scratch_bytes: Option, +} diff --git a/engine/crates/layerfs-sandbox/src/types/failure.rs b/engine/crates/layerfs-sandbox/src/types/failure.rs index de4ddfe27d..26c3b3d915 100644 --- a/engine/crates/layerfs-sandbox/src/types/failure.rs +++ b/engine/crates/layerfs-sandbox/src/types/failure.rs @@ -3,6 +3,9 @@ use std::{fmt, io}; /// Original ordinary-runtime refusal or I/O cause. #[derive(Debug)] pub enum RuntimeError { + /// Original Docker provider stage failure, including retained transfer/stream owners. + #[cfg(unix)] + Docker(Box), Io(io::Error), Protocol(&'static str), /// Original HTTP status, with a separately retained bounded error-body diagnostic. @@ -18,6 +21,8 @@ impl From for RuntimeError { impl fmt::Display for RuntimeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + #[cfg(unix)] + Self::Docker(failure) => write!(f, "Docker provider: {failure:?}"), Self::Io(e) => write!(f, "runtime I/O: {e}"), Self::Protocol(e) => write!(f, "runtime protocol: {e}"), Self::Http(s) => write!(f, "runtime HTTP status {s}"), @@ -53,6 +58,24 @@ pub struct WireFailure { /// The cause [`WireFailure::container_stopped`] reports. pub(crate) const STOPPED: &str = "exact stopped container observation"; impl WireFailure { + /// A non-HTTP cause before an attempt, with no invented transport facts. + pub fn local(cause: RuntimeError) -> Box { + Box::new(Self { + attempted: false, + sent_bytes: 0, + request_header_bytes: 0, + sent_body_bytes: 0, + pending_request: Default::default(), + status: None, + cause, + fence_error: None, + observed_exec: None, + observed_container: None, + requested_container: None, + requested_exec: None, + error_body: None, + }) + } /// The Engine still has the exact requested container, intact, but it is /// not running: its daemon exited, so nothing it held survives. pub fn container_stopped(&self) -> bool { diff --git a/engine/crates/layerfs-sandbox/src/types/image.rs b/engine/crates/layerfs-sandbox/src/types/image.rs new file mode 100644 index 0000000000..107ce35bae --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/image.rs @@ -0,0 +1,43 @@ +//! A Sandbox image's identity: the digest a deployment names, which is also +//! the name its unpacked root is cached under. It is a value type on every +//! platform, so a record written where the image backend runs can still be +//! read where it cannot. +use super::RuntimeError; + +/// The prefix every image digest carries. +const ALGORITHM: &str = "sha256:"; + +/// A Sandbox image's identity: `sha256:<64 lowercase hex>`, the same text the +/// Docker backend's `deploy.image` holds and a docker-archive names its config +/// blob by. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Image(String); + +impl Image { + /// Parses `sha256:<64 lowercase hex>`, the only form an archive's config + /// digest is compared with. + pub fn parse(text: &str) -> Result { + let hex = text.strip_prefix(ALGORITHM).unwrap_or(""); + if hex.len() != 64 + || !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(RuntimeError::Protocol( + "Sandbox image must be an exact digest, sha256:<64 lowercase hex>", + )); + } + Ok(Self(text.to_owned())) + } + + /// The whole digest text, `sha256:<64 hex>`. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The 64 hex digits without the algorithm: the name this image's + /// unpacked root is cached under. + pub fn hex(&self) -> &str { + &self.0[ALGORITHM.len()..] + } +} diff --git a/engine/crates/layerfs-sandbox/src/types/mod.rs b/engine/crates/layerfs-sandbox/src/types/mod.rs index 69827343cb..2d8c1768ae 100644 --- a/engine/crates/layerfs-sandbox/src/types/mod.rs +++ b/engine/crates/layerfs-sandbox/src/types/mod.rs @@ -2,9 +2,13 @@ mod execution; mod failure; mod identity; +mod image; +mod request; mod transfer; pub use execution::{CommandIdentity, ExecInspection}; pub(crate) use failure::STOPPED; pub use failure::{ErrorDiagnostic, RuntimeError, WireFailure}; pub use identity::{ContainerId, ExecId}; +pub use image::Image; +pub use request::ExecRequest; pub use transfer::PendingRequest; diff --git a/engine/crates/layerfs-sandbox/src/types/request.rs b/engine/crates/layerfs-sandbox/src/types/request.rs new file mode 100644 index 0000000000..e14e795a9b --- /dev/null +++ b/engine/crates/layerfs-sandbox/src/types/request.rs @@ -0,0 +1,48 @@ +//! Provider-independent ordinary command metadata and validation. +use crate::{CommandIdentity, RuntimeError}; +use std::fmt; +/// Caller-owned ordinary command metadata. No hidden shell/classifier/preparation. +#[derive(Clone)] +pub struct ExecRequest { + pub identity: CommandIdentity, + pub arguments: Vec, + pub environment: Vec, + pub directory: String, + pub stdin: bool, +} +impl fmt::Debug for ExecRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecRequest") + .field("identity", &self.identity) + .field("arguments_count", &self.arguments.len()) + .field("environment_count", &self.environment.len()) + .field("directory", &self.directory) + .field("stdin", &self.stdin) + .finish() + } +} +impl ExecRequest { + pub(crate) fn check(&self) -> Result<(), RuntimeError> { + self.identity.check()?; + if self.arguments.is_empty() + || self.arguments[0].is_empty() + || !std::path::Path::new(&self.directory).is_absolute() + || self + .arguments + .iter() + .chain(&self.environment) + .chain(std::iter::once(&self.directory)) + .any(|v| v.as_bytes().contains(&0)) + { + return Err(RuntimeError::Protocol("ordinary command fields")); + } + if self + .environment + .iter() + .any(|v| !v.contains('=') || v.starts_with('=')) + { + return Err(RuntimeError::Protocol("ordinary environment field")); + } + Ok(()) + } +} diff --git a/engine/crates/layerfs-sandbox/tests/docker_provider.rs b/engine/crates/layerfs-sandbox/tests/docker_provider.rs new file mode 100644 index 0000000000..8022f295d6 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/docker_provider.rs @@ -0,0 +1,394 @@ +//! Docker provider contracts exercised through bounded Unix HTTP peers, without an Engine. +#![cfg(unix)] +#[allow(dead_code)] +#[path = "support/engine.rs"] +mod peer; +use layerfs_sandbox::{ + backend::docker::{DockerProvider, ProviderFailure, Sandbox, SandboxRequest}, + provider::{ + DockerRecord, LaunchPhase, LaunchRequest, LocalRecord, ProviderLaunchOptions, SandboxRecord, + }, + CommandIdentity, ContainerId, ExecRequest, Runtime, RuntimeError, +}; +use peer::{fixed, frame, upgrade, Peer}; +use std::{ + fs, io, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; +const CID: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const EID: &str = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; +const IMAGE: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +static NEXT: AtomicUsize = AtomicUsize::new(0); +fn deployment() -> SandboxRequest { + SandboxRequest { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + } +} +struct Binary(PathBuf); +impl Binary { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "h22-provider-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::write(&path, b"sealed executable").unwrap(); + Self(path) + } + fn launch(&self) -> LaunchRequest { + LaunchRequest { + name: "h22-sandbox-1".into(), + daemon_executable: self.0.clone(), + daemon_config: b"private config".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(2), + options: ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: None, + }, + } + } +} +impl Drop for Binary { + fn drop(&mut self) { + fs::remove_file(&self.0).unwrap(); + } +} +fn logs(payload: &[u8]) -> Vec { + let mut bytes = format!("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.multiplexed-stream\r\nContent-Length: {}\r\n\r\n", payload.len()).into_bytes(); + bytes.extend(payload); + bytes +} +fn launch_responses() -> Vec> { + vec![ + fixed( + 200, + br#"{"Name":"provider-store","Driver":"local","Scope":"local","Options":null}"#, + ), + fixed(201, format!(r#"{{"Id":"{CID}"}}"#).as_bytes()), + fixed(200, b""), + fixed(204, b""), + logs(&frame(1, b"LAYERFS_DAEMON_LISTEN 0.0.0.0:30421\n")), + ] +} +fn exec_request() -> ExecRequest { + ExecRequest { + identity: CommandIdentity { uid: 501, gid: 20 }, + arguments: vec!["bash".into(), "-c".into(), "printf ordinary".into()], + environment: vec![], + directory: "/workspaces/1".into(), + stdin: false, + } +} +fn exec_inspection(running: bool, exit: &str) -> Vec { + fixed(200, format!(r#"{{"ID":"{EID}","ContainerID":"{CID}","Running":{running},"ExitCode":{exit},"Pid":42}}"#).as_bytes()) +} +#[test] +fn launch_stream_log_stop_delete_and_negotiated_versions_use_the_original_runtime() { + let binary = Binary::new(); + for minor in [48, 54] { + let mut responses = vec![fixed( + 200, + format!(r#"{{"ApiVersion":"1.{minor}"}}"#).as_bytes(), + )]; + responses.extend(launch_responses()); + let mut payload = frame(1, &vec![b'x'; 1024 * 1024 + 3]); + payload.extend(frame(2, b"err")); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&payload), + exec_inspection(false, "7"), + logs(&frame(1, b"daemon log\n")), + fixed(204, b""), + fixed(204, b""), + ]); + let (docker, peer) = Peer::scripted(responses, 8192); + let provider = Runtime::Docker(DockerProvider::new(docker)); + assert_eq!(provider.capabilities().unwrap().name, "docker"); + let mut instance = provider.launch(binary.launch()).unwrap(); + assert_eq!(instance.identity().to_string(), CID); + assert_eq!(instance.scratch_directory(), "/tmp"); + assert!(instance.local_record().is_none()); + assert_eq!( + instance.record(), + Some(SandboxRecord::Docker(DockerRecord { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + labels: vec![], + })) + ); + instance.track(42, 1).unwrap(); + let (mut stdout, mut stderr) = (Vec::new(), Vec::new()); + let exit = instance + .run(&exec_request(), &mut stdout, &mut stderr) + .unwrap(); + assert_eq!((exit.code, exit.pid), (7, Some(42))); + assert_eq!(exit.stdout_bytes, 1024 * 1024 + 3); + assert_eq!(stdout.len() as u64, exit.stdout_bytes); + assert_eq!(stderr, b"err"); + assert_eq!(instance.log_tail(20, 2048).unwrap(), b"daemon log\n"); + instance.stop(1).unwrap(); + instance.delete().unwrap(); + assert!(instance.stop(1).is_err()); + assert!(instance.delete().is_err()); + let calls = peer.finish(); + assert_eq!(calls.len(), 12); + assert!(calls[0].starts_with(b"GET /version ")); + for call in &calls[1..] { + assert!(String::from_utf8_lossy(call) + .split(' ') + .nth(1) + .unwrap() + .starts_with(&format!("/v1.{minor}/"))); + } + assert!(String::from_utf8_lossy(&calls[2]).contains("create?name=h22-sandbox-1")); + assert!(String::from_utf8_lossy(calls.last().unwrap()).contains("force=false&v=false")); + } +} +#[test] +fn uncertain_create_retains_partial_id_and_never_adopts_or_cleans_it() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.truncate(2); + responses[1] = fixed(201, format!(r#"{{"Id":"{CID}","Id":"{CID}"}}"#).as_bytes()); + let (docker, peer) = Peer::new(responses, 1); + let failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, LaunchPhase::Create); + assert!(failure.instance.is_none()); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed original cause") + }; + let ProviderFailure::Create(cause) = *cause else { + panic!("create custody") + }; + assert_eq!(cause.transfer.observed_container.unwrap().to_string(), CID); + assert!(cause.transfer.attempted); + assert_eq!(peer.finish().len(), 2); +} +#[test] +fn upload_start_and_listener_failures_keep_acknowledged_instance_and_original_receipts() { + let binary = Binary::new(); + for (stage, expected) in [ + (2, LaunchPhase::Create), + (3, LaunchPhase::Start), + (4, LaunchPhase::Listener), + ] { + let mut responses = launch_responses(); + responses.truncate(stage + 1); + responses[stage] = fixed(409, b"refused"); + let (docker, peer) = Peer::new(responses, 8192); + let mut failure = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .err() + .unwrap(); + assert_eq!(failure.phase, expected); + assert_eq!(failure.instance.take().unwrap().identity().to_string(), CID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed receipt") + }; + match (*cause, stage) { + (ProviderFailure::Upload(f), 2) => { + assert!(f.work.tar_bytes > 0); + assert_eq!(f.transfer.status, Some(409)); + } + (ProviderFailure::Start(f), 3) => assert_eq!(f.status, Some(409)), + (ProviderFailure::Listener(f), 4) => assert_eq!(f.transfer.status, Some(409)), + _ => panic!("original stage lost"), + } + assert_eq!(peer.finish().len(), stage + 1); + } +} +#[test] +fn validation_and_stdin_refuse_without_an_engine_attempt() { + let binary = Binary::new(); + let (docker, peer) = Peer::new(vec![], 1); + let provider = Runtime::Docker(DockerProvider::new(docker.clone())); + let mut request = binary.launch(); + request.name = "../escape".into(); + assert_eq!( + provider.launch(request).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut bad_host = binary.launch(); + bad_host.options = ProviderLaunchOptions::Docker { + image: IMAGE.into(), + store_volume: "provider-store".into(), + port: 30421, + connect_host: Some("bad host\0".into()), + }; + assert_eq!( + provider.launch(bad_host).err().unwrap().phase, + LaunchPhase::Validate + ); + let mut wrong_options = binary.launch(); + wrong_options.options = ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }; + assert_eq!( + provider.launch(wrong_options).err().unwrap().phase, + LaunchPhase::Validate + ); + let foreign = SandboxRecord::Local(LocalRecord { + name: "test".into(), + pid: 1, + start_time: 1, + boot: None, + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(CID).unwrap(); + assert!(!provider.reopen(id, &foreign).err().unwrap().attempted); + assert!( + !provider + .reopen_stopped(id, &foreign) + .err() + .unwrap() + .attempted + ); + let stopped = Sandbox::stopped(docker, ContainerId::parse(CID).unwrap(), deployment()); + let mut request = exec_request(); + request.stdin = true; + assert!( + !layerfs_sandbox::Sandbox::Docker(stopped) + .run(&request, &mut Vec::new(), &mut Vec::new()) + .unwrap_err() + .attempted + ); + assert!(peer.finish().is_empty()); +} +#[test] +fn eof_with_unknown_exit_preserves_the_handle_without_polling_or_replay() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"ok")), + exec_inspection(true, "null"), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecExitUnknown { + handle, + inspection, + progress, + } = *cause + else { + panic!("EOF must not invent exit") + }; + assert_eq!(progress.stdout, 2); + assert_eq!(progress.stderr, 0); + assert!(inspection.running); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(peer.finish().len(), 8); +} +#[test] +fn exec_stage_failures_keep_deciding_wire_facts_and_delivered_progress() { + let binary = Binary::new(); + for stage in 0..3 { + let mut responses = launch_responses(); + let mut exec = vec![ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"delivered")), + exec_inspection(false, "0"), + ]; + exec.truncate(stage + 1); + exec[stage] = fixed(409, b"original refusal"); + responses.extend(exec); + let (docker, peer) = Peer::new(responses, 8192); + let owner = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = owner + .run(&exec_request(), &mut Vec::new(), &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.status, Some(409)); + assert!(failure.sent_bytes > 0); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("original custody") + }; + match (*cause, stage) { + (ProviderFailure::ExecCreate(f), 0) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes) + } + (ProviderFailure::ExecStart(f), 1) => { + assert_eq!(failure.sent_bytes, f.transfer.sent_bytes); + assert_eq!(f.created.identity().1.to_string(), EID); + } + ( + ProviderFailure::ExecInspect { + failure: f, + handle, + progress, + }, + 2, + ) => { + assert_eq!(failure.sent_bytes, f.sent_bytes); + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(progress.stdout, 9); + assert_eq!(progress.stderr, 0); + } + _ => panic!("wrong stage"), + } + assert_eq!(peer.finish().len(), 6 + stage); + } +} +struct BrokenSink; +impl io::Write for BrokenSink { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn failed_sink_retains_pending_output_and_exec_identity_without_inspection_or_cleanup() { + let binary = Binary::new(); + let mut responses = launch_responses(); + responses.extend([ + fixed(201, format!(r#"{{"Id":"{EID}"}}"#).as_bytes()), + upgrade(&frame(1, b"undelivered")), + ]); + let (docker, peer) = Peer::new(responses, 8192); + let instance = Runtime::Docker(DockerProvider::new(docker)) + .launch(binary.launch()) + .unwrap(); + let failure = instance + .run(&exec_request(), &mut BrokenSink, &mut Vec::new()) + .unwrap_err(); + assert!(failure.attempted); + assert_eq!(failure.requested_exec.unwrap().to_string(), EID); + let RuntimeError::Docker(cause) = failure.cause else { + panic!("typed custody") + }; + let ProviderFailure::ExecOutput { handle, failure } = *cause else { + panic!("stream owner retained") + }; + assert_eq!(handle.identity().1.to_string(), EID); + assert_eq!(failure.output.progress().stdout, 0); + assert_eq!(failure.output.progress().wire_payload, 11); + assert_eq!(peer.finish().len(), 7); +} diff --git a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs index 9c59dc4c25..e3d18209d7 100644 --- a/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs +++ b/engine/crates/layerfs-sandbox/tests/engine_lifecycle.rs @@ -405,6 +405,9 @@ fn reopened_from(body: &str) -> Box { fn only_the_exact_intact_container_not_running_is_a_stopped_container() { let stopped = reopened_from(&exited()); assert!(stopped.container_stopped()); + assert!(stopped.attempted); + assert_eq!(stopped.status, Some(200)); + assert!(stopped.sent_bytes >= stopped.request_header_bytes && stopped.request_header_bytes > 0); assert!(!stopped.container_gone()); let other = "c".repeat(64); assert!(!reopened_from(&exited().replace(CID, &other)).container_stopped()); @@ -414,6 +417,47 @@ fn only_the_exact_intact_container_not_running_is_a_stopped_container() { assert!(!reopened_from(&image).container_stopped()); } +#[test] +fn provider_inspects_and_reopens_exact_running_and_stopped_deployments() { + use layerfs_sandbox::backend::docker::DockerProvider; + use layerfs_sandbox::provider::SandboxRecord; + let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); + let running = inspect().replace( + r#""Running":true"#, + r#""Running":true,"Pid":42,"ExitCode":0"#, + ); + let stopped = exited().replace( + r#""Running":false"#, + r#""Running":false,"Pid":0,"ExitCode":7"#, + ); + let (docker, peer) = Peer::new( + vec![ + fixed(200, running.as_bytes()), + fixed(200, running.as_bytes()), + fixed(200, stopped.as_bytes()), + fixed(204, b""), + ], + 64, + ); + let provider = DockerProvider::new(docker); + let record = SandboxRecord::from(request()); + let mut owner = provider.reopen(id, &record).unwrap(); + let facts = owner.inspect().unwrap(); + assert!(facts.running); + assert_eq!(facts.process_id, Some(42)); + assert_eq!(facts.exit_code, None); + assert_eq!(facts.scratch_bytes, None); + let facts = owner.inspect().unwrap(); + assert!(!facts.running); + assert_eq!(facts.process_id, None); + assert_eq!(facts.exit_code, Some(7)); + // A read-only observation does not authorize Delete on the running owner. + assert!(!owner.delete().unwrap_err().attempted); + let mut stopped = provider.reopen_stopped(id, &record).unwrap(); + stopped.delete().unwrap(); + assert_eq!(peer.finish().len(), 4); +} + #[test] fn a_stopped_container_is_deleted_once_and_never_forced() { let id = layerfs_sandbox::ContainerId::parse(CID).unwrap(); diff --git a/engine/crates/layerfs-sandbox/tests/native_platform.rs b/engine/crates/layerfs-sandbox/tests/native_platform.rs new file mode 100644 index 0000000000..46ffae1258 --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/native_platform.rs @@ -0,0 +1,71 @@ +//! Platform refusal precedes filesystem access; a runtime never selects another backend. +#![cfg(any(target_os = "linux", target_os = "macos"))] +use layerfs_sandbox::{ + provider::{DockerRecord, LaunchPhase, LaunchRequest, ProviderLaunchOptions, SandboxRecord}, + CommandIdentity, ContainerId, Runtime, RuntimeError, +}; +use std::time::Duration; + +#[test] +fn wrong_platform_is_refused_before_root_access() { + #[cfg(target_os = "macos")] + let result = layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider", + ); + #[cfg(target_os = "linux")] + let result = layerfs_sandbox::backend::native::NativeMacosProvider::new( + "/does-not-exist/native-provider", + ); + assert!(matches!(result, Err(RuntimeError::Unsupported(_)))); +} + +#[test] +fn native_linux_requires_root_before_root_access() { + if cfg!(target_os = "linux") && !nix::unistd::Uid::effective().is_root() { + assert!(matches!( + layerfs_sandbox::backend::native::NativeLinuxProvider::new( + "/does-not-exist/native-provider" + ), + Err(RuntimeError::Unsupported(_)) + )); + } +} + +/// Windows has no host runtime: the placeholder variant refuses every +/// operation before any effect, and never falls back to Docker or native. +#[test] +fn windows_is_refused_before_any_effect() { + let runtime = Runtime::Windows; + assert!(matches!( + runtime.capabilities(), + Err(RuntimeError::Unsupported(_)) + )); + assert!(matches!( + runtime.local_directory("one"), + Err(RuntimeError::Unsupported(_)) + )); + let request = LaunchRequest { + name: "one".into(), + daemon_executable: "/does-not-exist/daemon".into(), + daemon_config: b"setup".to_vec(), + labels: vec![], + startup_timeout: Duration::from_secs(1), + options: ProviderLaunchOptions::Local { + command_identity: CommandIdentity { uid: 501, gid: 20 }, + environment: vec![], + }, + }; + let failure = runtime.launch(request).err().unwrap(); + assert_eq!(failure.phase, LaunchPhase::Validate); + assert!(failure.instance.is_none()); + assert!(matches!(failure.cause, RuntimeError::Unsupported(_))); + let record = SandboxRecord::Docker(DockerRecord { + image: format!("sha256:{}", "a".repeat(64)), + store_volume: "lfs-store".into(), + port: 30421, + labels: vec![], + }); + let id = ContainerId::parse(&"b".repeat(64)).unwrap(); + assert!(runtime.reopen(id, &record).is_err()); + assert!(runtime.reopen_stopped(id, &record).is_err()); +} diff --git a/engine/crates/layerfs-sandbox/tests/runtime_enum.rs b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs new file mode 100644 index 0000000000..1f7c1ba78f --- /dev/null +++ b/engine/crates/layerfs-sandbox/tests/runtime_enum.rs @@ -0,0 +1,15 @@ +//! The Runtime enum's Docker selection: no local directory, container capabilities. +use layerfs_sandbox::backend::docker::{Docker, DockerProvider}; +use layerfs_sandbox::Runtime; +use std::time::Duration; + +/// Selecting Docker names an Engine endpoint only; nothing is dialed here. +#[test] +fn docker_reports_container_capabilities_and_owns_no_local_directory() { + let docker = Docker::new("/does-not-exist/docker.sock", Duration::from_secs(1)).unwrap(); + let runtime = Runtime::Docker(DockerProvider::new(docker)); + let capabilities = runtime.capabilities().unwrap(); + assert_eq!(capabilities.name, "docker"); + assert!(capabilities.requires_image); + assert!(runtime.local_directory("anything").unwrap().is_none()); +} diff --git a/tools/check_product_boundary.py b/tools/check_product_boundary.py index c4159c3c12..97201c5102 100644 --- a/tools/check_product_boundary.py +++ b/tools/check_product_boundary.py @@ -17,12 +17,14 @@ # Crates whose `unsafe` surface is bounded. layerfs-storage keeps one audited # FFI module (encoding/codec.rs); Persistence has the owner-authorized macOS -# seal file-control module. Their siblings stay unsafe-free. The +# seal file-control module; Sandbox reads macOS process facts through +# proc_pidinfo in backend/macos/darwin.rs. Their siblings stay unsafe-free. The # comment in a lint name (`unsafe_code`) is not the bare word, so attr lines # never trip the scan once comments are stripped. UNSAFE_AUDITED_MODULE = { "layerfs-storage": "src/encoding/codec.rs", "layerfs-persistence": "src/backend/sqlite/file_control.rs", + "layerfs-sandbox": "src/backend/macos/darwin.rs", } UNSAFE_FREE_CRATES = ( "layerfs-content", @@ -32,10 +34,10 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-api", "layerfs-bridge", - "layerfs-sandbox", ) UNSAFE_ROOT_ATTR = { "layerfs-storage": "#![deny(unsafe_code)]", @@ -47,10 +49,11 @@ "layerfs-workspace": "#![forbid(unsafe_code)]", "layerfs-store-service": "#![forbid(unsafe_code)]", "layerfs-fuse": "#![forbid(unsafe_code)]", + "layerfs-nfs": "#![forbid(unsafe_code)]", "layerfs-daemon": "#![forbid(unsafe_code)]", "layerfs-api": "#![forbid(unsafe_code)]", "layerfs-bridge": "#![forbid(unsafe_code)]", - "layerfs-sandbox": "#![forbid(unsafe_code)]", + "layerfs-sandbox": "#![deny(unsafe_code)]", } @@ -84,6 +87,8 @@ "layerfs-overlay": set(), "layerfs-workspace": {"layerfs-content", "layerfs-overlay", "layerfs-telemetry"}, "layerfs-fuse": {"layerfs-content", "layerfs-overlay", "layerfs-workspace"}, + # The NFS export reuses the kernel ports and operations; never daemon. + "layerfs-nfs": {"layerfs-content", "layerfs-overlay", "layerfs-workspace", "layerfs-fuse"}, # The Store engine (#352 F-020): the SDK's Store calls in-process and the # Store/overlay ports daemon Workspace Commits use; it opens the Store. "layerfs-store-service": { @@ -103,6 +108,7 @@ "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-bridge", "layerfs-content", "layerfs-storage", @@ -135,6 +141,7 @@ "layerfs-bridge": ENGINE, "layerfs-daemon": ENGINE, "layerfs-fuse": ENGINE, + "layerfs-nfs": ENGINE, "layerfs-overlay": ENGINE, "layerfs-persistence": ENGINE, "layerfs-project": ENGINE, @@ -196,6 +203,7 @@ def test_path_violations(path, source, packages): "layerfs-workspace", "layerfs-store-service", "layerfs-fuse", + "layerfs-nfs", "layerfs-daemon", "layerfs-bridge", "layerfs-sandbox", diff --git a/tools/test_check_product_boundary.py b/tools/test_check_product_boundary.py index ec4196fbff..55f600b5f4 100644 --- a/tools/test_check_product_boundary.py +++ b/tools/test_check_product_boundary.py @@ -126,7 +126,15 @@ def test_sdk_source_edges_and_unsafe_boundary(self): def test_sandbox_active_edges_and_unsafe_boundary(self): path = Path("engine/crates/layerfs-sandbox/src/backend/docker/http.rs") self.assertTrue(unsafe_violations(path, "unsafe fn signal_pid() {}")) - self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + self.assertFalse(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![deny(unsafe_code)]")) + self.assertTrue(unsafe_violations(Path("engine/crates/layerfs-sandbox/src/lib.rs"), "#![forbid(unsafe_code)]")) + darwin = Path("engine/crates/layerfs-sandbox/src/backend/macos/darwin.rs") + self.assertFalse(unsafe_violations(darwin, "fn f() { unsafe { libc::proc_listallpids(); } }")) + self.assertTrue( + unsafe_violations( + Path("engine/crates/layerfs-sandbox/src/backend/macos/mounts.rs"), "fn f() { unsafe { call(); } }" + ) + ) prefix = '[package]\nname="layerfs-sandbox"\n[dependencies]\n' for dependency in ( "layerfs-api-core", From fccce8fb889d3e6665f33216dda91fc5543bab23 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:00:00 +0800 Subject: [PATCH 64/64] test(sandbox): leave Linux local_runtime tests to the Linux native branch (Area B3) engine/crates/layerfs-sandbox/tests/local_runtime.rs arrived in B3 with the #427 merge (fa02a1b2f). It is gated with #![cfg(target_os = "linux")] and tests the Linux native process table, which B3 does not carry: B3's backend/linux/ is byte-identical to B1's placeholder, so all nine cases fail on Linux CI with Io(Unsupported, "Linux native process table"). The real implementation and the file itself belong to B2 (b2-linux-native-v2), and merging B2 into B3 keeps B2's copy unchanged, so removing it here neither loses coverage nor changes the merged tree. local_search_path.rs and local_stop_mount.rs are macOS-only and stay. References: none in Cargo.toml, tools/ci_tests.py, workflows or docs. The only remaining "local_runtime" text is layerfs-node's a_record_of_the_former_local_runtime_is_never_used_nor_deleted test, which covers former `local` sandbox records and is unrelated to this target. Deleted tests: local_runtime (9 cases on Linux); the behaviour they check is covered by the same file on B2. Validation (macOS 27.2 ARM64, Rust 1.85.1, this worktree's own target): cargo fmt --all --check exit 0; cargo clippy --locked --workspace --exclude layerfs-node --all-targets -- -D warnings exit 0. tools/ci_tests.py targets fast no longer emits --test local_runtime (259 -> 258 targets) and filter fast is byte-identical. merge-tree --write-tree of b2-linux-native-v2 and this branch is conflict-free and keeps local_runtime.rs identical to B2; the resulting tree diffs empty against the H43e reference 5c6a238b. Production LOC: 110634 -> 110634 (delta 0) Scope: first-party production Rust, runtime SQL and published JavaScript; tests, docs, tooling, fixtures and generated files excluded. Method: tools/production_loc.py --root --json, identical counter and scope for both snapshots. Parent comparison: 1dba37b06 to this commit. The change deletes one test file only. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-sandbox/tests/local_runtime.rs | 672 ------------------ 1 file changed, 672 deletions(-) delete mode 100644 engine/crates/layerfs-sandbox/tests/local_runtime.rs diff --git a/engine/crates/layerfs-sandbox/tests/local_runtime.rs b/engine/crates/layerfs-sandbox/tests/local_runtime.rs deleted file mode 100644 index 1db9a03b7c..0000000000 --- a/engine/crates/layerfs-sandbox/tests/local_runtime.rs +++ /dev/null @@ -1,672 +0,0 @@ -//! The native process runtime (`backend::native`): one daemon process per -//! Sandbox directory, observed by process id and start time, its commands run -//! as their own identity, and its stop and delete each attempted once. A -//! small shell script stands in for the daemon; nothing here needs root. -//! The runtime is Linux only, and so are these tests. -#![cfg(target_os = "linux")] -use layerfs_sandbox::{ - backend::{ - docker::ExecRequest, - native::{Local, LocalPhase, LocalProcess, Sandbox}, - }, - CommandIdentity, RuntimeError, -}; -use std::{ - fs, - os::unix::fs::{MetadataExt, PermissionsExt}, - path::{Path, PathBuf}, - process::Command, - sync::atomic::{AtomicUsize, Ordering}, - time::{Duration, Instant}, -}; - -static NEXT: AtomicUsize = AtomicUsize::new(0); -const WAIT: Duration = Duration::from_secs(10); - -/// A fresh 0755 runtime root with a stand-in daemon `daemon.sh` beside it. -fn fixture(script: &str) -> (PathBuf, PathBuf) { - let base = std::env::temp_dir().join(format!( - "layerfs-local-{}-{}", - std::process::id(), - NEXT.fetch_add(1, Ordering::Relaxed) - )); - let _ = fs::remove_dir_all(&base); - fs::create_dir_all(base.join("root")).unwrap(); - fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); - fs::set_permissions(base.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); - let daemon = base.join("daemon.sh"); - fs::write(&daemon, format!("#!/bin/sh\n{script}\n")).unwrap(); - fs::set_permissions(&daemon, fs::Permissions::from_mode(0o755)).unwrap(); - (base.join("root"), daemon) -} - -/// Prints the listener line after checking its arguments, then waits. -const LISTENS: &str = r#"[ "$1" = --config ] && [ -f "$2" ] || exit 3 -env >&2 -echo "starting" -echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" -exec sleep 60"#; - -/// The command identity: this user, or nobody when the tests run as root. -fn me() -> CommandIdentity { - let meta = fs::metadata("/proc/self").unwrap(); - if meta.uid() == 0 { - CommandIdentity { - uid: 65534, - gid: 65534, - } - } else { - CommandIdentity { - uid: meta.uid(), - gid: meta.gid(), - } - } -} - -fn mode(path: &Path) -> u32 { - fs::symlink_metadata(path).unwrap().mode() & 0o7777 -} - -fn started(root: &Path, daemon: &Path, name: &str) -> Sandbox { - Local::new(root) - .unwrap() - .naming(name) - .unwrap() - .create( - daemon, - b"setup bytes", - WAIT, - vec![("layerfs.role".into(), "sandbox".into())], - ) - .unwrap() -} - -fn run(sandbox: &Sandbox, arguments: &[&str], environment: &[&str]) -> (i32, String, String) { - let request = ExecRequest { - identity: me(), - arguments: arguments.iter().map(|a| a.to_string()).collect(), - environment: environment.iter().map(|a| a.to_string()).collect(), - directory: sandbox.directory().display().to_string(), - stdin: false, - }; - let (mut out, mut err) = (Vec::new(), Vec::new()); - let exit = sandbox.run(&request, &mut out, &mut err).unwrap(); - assert_eq!(exit.stdout, out.len() as u64); - assert_eq!(exit.stderr, err.len() as u64); - ( - exit.code, - String::from_utf8(out).unwrap(), - String::from_utf8(err).unwrap(), - ) -} - -#[test] -fn a_root_must_be_absolute_owned_private_to_write_and_traversable() { - let (root, _) = fixture("exit 0"); - assert!(matches!( - Local::new("relative/root"), - Err(RuntimeError::Protocol(_)) - )); - assert!(matches!( - Local::new(root.join("..").join("root")), - Err(RuntimeError::Protocol(_)) - )); - assert!(matches!( - Local::new(root.join("missing")), - Err(RuntimeError::Io(_)) - )); - fs::set_permissions(&root, fs::Permissions::from_mode(0o775)).unwrap(); - assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); - fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); - assert!(matches!(Local::new(&root), Err(RuntimeError::Protocol(_)))); - fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); - let local = Local::new(&root).unwrap(); - for bad in ["", ".hidden", "-x", "a/b", "..", "a b", &"x".repeat(129)] { - assert!(local.clone().naming(bad).is_err(), "{bad:?} admitted"); - } - assert!(local.paths().is_err(), "paths before a name"); - let paths = local.naming("layerfs-shop-1").unwrap().paths().unwrap(); - assert_eq!(paths.directory, root.join("layerfs-shop-1")); - assert_eq!(paths.config, paths.directory.join("config/daemon.setup")); - assert_eq!( - paths.overlay, - paths.directory.join("overlay/overlay.sqlite") - ); - assert_eq!(paths.mounts, paths.directory.join("workspaces")); - assert_eq!(paths.log, paths.directory.join("daemon.log")); - assert_eq!(paths.tmp, paths.directory.join("tmp")); -} - -#[test] -fn a_start_writes_a_private_setup_spawns_once_and_reads_the_listener() { - let (root, daemon) = fixture(LISTENS); - let mut sandbox = started(&root, &daemon, "layerfs-shop-1"); - let directory = root.join("layerfs-shop-1"); - assert_eq!(sandbox.directory(), directory); - assert_eq!(sandbox.port(), 40404); - assert_eq!( - sandbox.endpoint().unwrap(), - "127.0.0.1:40404".parse().unwrap() - ); - assert_eq!( - sandbox.labels(), - [("layerfs.role".to_owned(), "sandbox".to_owned())] - ); - let process = sandbox.process().unwrap(); - assert!(process.alive()); - assert_eq!(LocalProcess::observe(process.pid).unwrap(), process); - assert!(!LocalProcess { - start: process.start + 1, - ..process - } - .alive()); - // private setup, private overlay home, traversable Sandbox directory - assert_eq!(mode(&directory), 0o755); - assert_eq!(mode(&directory.join("config")), 0o700); - assert_eq!(mode(&directory.join("config/daemon.setup")), 0o600); - assert_eq!( - fs::read(directory.join("config/daemon.setup")).unwrap(), - b"setup bytes" - ); - assert_eq!(mode(&directory.join("overlay")), 0o700); - assert_eq!(mode(&directory.join("daemon.log")), 0o600); - assert_eq!(mode(&directory.join("tmp")), 0o1777); - assert_eq!(sandbox.tmp(), directory.join("tmp")); - // the daemon inherits nothing of this process's environment but PATH - let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); - assert!(log.contains("PATH=/usr/local/sbin:"), "{log}"); - assert!(!log.contains("CARGO"), "{log}"); - assert_eq!( - String::from_utf8(sandbox.log_tail(1, 4096).unwrap()).unwrap(), - "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404\n" - ); - assert_eq!(sandbox.log_tail(1, 5).unwrap(), b"0404\n"); - // the same name again is refused before anything is made - let again = Local::new(&root) - .unwrap() - .naming("layerfs-shop-1") - .unwrap() - .create(&daemon, b"x", WAIT, Vec::new()) - .unwrap_err(); - assert_eq!(again.phase, LocalPhase::Validate); - assert!(again.sandbox.is_none()); - // delete needs a known stop; stop ends the process; each is attempted once - assert!(sandbox.delete().is_err()); - let mut sandbox = Sandbox::reopen( - directory.clone(), - sandbox.identity(), - process, - 40404, - Vec::new(), - ) - .unwrap(); - sandbox.stop(1).unwrap(); - assert!(!process.alive()); - assert!(sandbox.stop(1).is_err()); - assert!(sandbox.endpoint().is_err()); - sandbox.delete().unwrap(); - assert!(!directory.exists()); - assert!(sandbox.delete().is_err()); -} - -#[test] -fn a_command_runs_as_its_identity_in_its_directory_with_only_its_environment() { - let (root, daemon) = fixture(LISTENS); - let mut sandbox = started(&root, &daemon, "layerfs-shop-2"); - let identity = me(); - let (code, out, err) = run( - &sandbox, - &[ - "bash", - "-c", - "id -u; id -g; pwd; echo \"$HOME|${CARGO:-none}\"; id -G >&2; echo oops >&2; exit 7", - ], - &["HOME=/tmp"], - ); - assert_eq!(code, 7); - assert_eq!( - out, - format!( - "{}\n{}\n{}\n/tmp|none\n", - identity.uid, - identity.gid, - sandbox.directory().display() - ) - ); - let (groups, err) = err.split_once('\n').unwrap(); - assert_eq!(err, "oops\n"); - // as root, the command keeps no supplementary group of this process - if fs::metadata("/proc/self").unwrap().uid() == 0 { - assert_eq!(groups, identity.gid.to_string()); - } - // a command a signal ended reports 128 + the signal, as the Engine does - let (code, _, _) = run(&sandbox, &["bash", "-c", "kill -9 $$"], &[]); - assert_eq!(code, 137); - // more output than one pipe holds, on both streams at once - let (code, out, err) = run( - &sandbox, - &[ - "bash", - "-c", - "head -c 300000 /dev/zero | tr '\\0' a; head -c 200000 /dev/zero | tr '\\0' b >&2", - ], - &[], - ); - assert_eq!((code, out.len(), err.len()), (0, 300000, 200000)); - // refused before any effect: root, stdin, a relative directory, a bad variable - let mut request = ExecRequest { - identity, - arguments: vec!["true".into()], - environment: Vec::new(), - directory: "/".into(), - stdin: false, - }; - for change in [ - (|r: &mut ExecRequest| r.identity = CommandIdentity { uid: 0, gid: 0 }) - as fn(&mut ExecRequest), - |r| r.stdin = true, - |r| r.directory = "relative".into(), - |r| r.environment = vec!["NOEQUALS".into()], - |r| r.arguments = Vec::new(), - ] { - let mut changed = request.clone(); - change(&mut changed); - let failure = sandbox - .run(&changed, &mut Vec::new(), &mut Vec::new()) - .unwrap_err(); - assert!(!failure.attempted, "{failure:?}"); - } - sandbox.stop(1).unwrap(); - // no command once the daemon is stopped - request.directory = "/".into(); - assert!(sandbox - .run(&request, &mut Vec::new(), &mut Vec::new()) - .is_err()); - sandbox.delete().unwrap(); -} - -#[test] -fn the_daemon_and_its_commands_start_with_a_containers_search_path() { - // on Linux, whatever this process's PATH is (macOS: tests/local_search_path.rs) - const CONTAINER: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; - let (root, daemon) = fixture(LISTENS); - let mut sandbox = started(&root, &daemon, "layerfs-shop-path"); - let log = String::from_utf8(sandbox.log_tail(100, 4096).unwrap()).unwrap(); - assert!( - log.lines().any(|line| line == format!("PATH={CONTAINER}")), - "{log}" - ); - let (code, out, _) = run(&sandbox, &["/bin/sh", "-c", "printf %s \"$PATH\""], &[]); - assert_eq!((code, out.as_str()), (0, CONTAINER)); - // a command's own PATH still wins - let (code, out, _) = run( - &sandbox, - &["/bin/sh", "-c", "printf %s \"$PATH\""], - &["PATH=/opt/tools/bin:/usr/bin:/bin"], - ); - assert_eq!((code, out.as_str()), (0, "/opt/tools/bin:/usr/bin:/bin")); - sandbox.stop(1).unwrap(); - sandbox.delete().unwrap(); -} - -#[test] -fn a_daemon_that_ends_is_the_stopped_observation_and_only_its_delete_remains() { - let (root, daemon) = fixture(LISTENS); - let sandbox = started(&root, &daemon, "layerfs-shop-3"); - let process = sandbox.process().unwrap(); - let directory = sandbox.directory().to_owned(); - // killed from outside, as a crash or the OOM killer would - assert!(Command::new("kill") - .args(["-9", &process.pid.to_string()]) - .status() - .unwrap() - .success()); - let deadline = Instant::now() + WAIT; - let failure = loop { - match sandbox.endpoint() { - Err(failure) => break failure, - Ok(_) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(20)), - Ok(_) => panic!("killed daemon still observed running"), - } - }; - assert!(failure.container_stopped(), "{failure:?}"); - assert!(!failure.container_gone()); - // Stopped can be observed before the child is reapable. Keep observing - // through its owner so try_wait reaps it once all its threads have ended. - let deadline = Instant::now() + WAIT; - let stat = loop { - let failure = sandbox.endpoint().unwrap_err(); - assert!(failure.container_stopped(), "{failure:?}"); - assert!(!failure.container_gone()); - let stat = fs::read_to_string(format!("/proc/{}/stat", process.pid)); - if stat.as_ref().map_or(true, |stat| !stat.contains(") Z ")) || Instant::now() >= deadline { - break stat; - } - std::thread::sleep(Duration::from_millis(20)); - }; - // this process reaped its child: no zombie remains - assert!(stat.is_err() || !stat.unwrap().contains(") Z ")); - // a reopen from the record observes the same - let reopened = Sandbox::reopen( - directory.clone(), - sandbox.identity(), - process, - 40404, - Vec::new(), - ) - .unwrap_err(); - assert!(reopened.container_stopped()); - let mut stopped = Sandbox::stopped( - directory.clone(), - sandbox.identity(), - process, - 40404, - Vec::new(), - ); - assert!(stopped.endpoint().is_err()); - stopped.delete().unwrap(); - assert!(!directory.exists()); -} - -#[test] -fn a_failed_start_returns_what_it_made() { - // exits before its listener line - let (root, daemon) = fixture("echo broken >&2; exit 9"); - let failure = Local::new(&root) - .unwrap() - .naming("layerfs-shop-4") - .unwrap() - .create(&daemon, b"x", WAIT, Vec::new()) - .unwrap_err(); - assert_eq!(failure.phase, LocalPhase::Listener); - let mut sandbox = failure.sandbox.unwrap(); - assert_eq!(sandbox.log_tail(5, 100).unwrap(), b"broken\n"); - sandbox.stop(1).unwrap(); - sandbox.delete().unwrap(); - assert!(!root.join("layerfs-shop-4").exists()); - // never prints it: the wait ends - let (root, daemon) = fixture("exec sleep 60"); - let failure = Local::new(&root) - .unwrap() - .naming("layerfs-shop-5") - .unwrap() - .create(&daemon, b"x", Duration::from_millis(300), Vec::new()) - .unwrap_err(); - assert_eq!(failure.phase, LocalPhase::Listener); - assert!( - matches!(&failure.cause, RuntimeError::Io(e) if e.kind() == std::io::ErrorKind::TimedOut) - ); - let mut sandbox = failure.sandbox.unwrap(); - let process = sandbox.process().unwrap(); - assert!(process.alive()); - sandbox.stop(1).unwrap(); - assert!(!process.alive()); - sandbox.delete().unwrap(); - // a listener that is not loopback is refused - let (root, daemon) = fixture("echo 'LAYERFS_DAEMON_LISTEN 0.0.0.0:5'; exec sleep 60"); - let failure = Local::new(&root) - .unwrap() - .naming("layerfs-shop-6") - .unwrap() - .create(&daemon, b"x", WAIT, Vec::new()) - .unwrap_err(); - assert_eq!(failure.phase, LocalPhase::Listener); - let mut sandbox = failure.sandbox.unwrap(); - sandbox.stop(1).unwrap(); - sandbox.delete().unwrap(); - // a missing executable or an empty setup makes nothing - let failure = Local::new(&root) - .unwrap() - .naming("layerfs-shop-7") - .unwrap() - .create(&root.join("missing"), b"x", WAIT, Vec::new()) - .unwrap_err(); - assert_eq!( - (failure.phase, failure.sandbox.is_none()), - (LocalPhase::Validate, true) - ); - let failure = Local::new(&root) - .unwrap() - .naming("layerfs-shop-7") - .unwrap() - .create(&daemon, b"", WAIT, Vec::new()) - .unwrap_err(); - assert_eq!( - (failure.phase, failure.sandbox.is_none()), - (LocalPhase::Validate, true) - ); - assert!(!root.join("layerfs-shop-7").exists()); -} - -#[test] -fn a_zombie_is_not_alive() { - let mut child = Command::new("true").spawn().unwrap(); - let process = LocalProcess::observe(child.id()).unwrap(); - let deadline = Instant::now() + WAIT; - while process.alive() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process.alive(), "an exited, unreaped child counted alive"); - child.wait().unwrap(); -} - -#[test] -fn every_directory_above_a_root_must_be_traversable_by_others() { - let (root, _) = fixture("exit 0"); - let base = root.parent().unwrap().to_owned(); - // a root inside a directory others cannot pass, as `/root` (0700) is - let closed = base.join("closed"); - fs::create_dir(&closed).unwrap(); - fs::create_dir(closed.join("root")).unwrap(); - fs::set_permissions(closed.join("root"), fs::Permissions::from_mode(0o755)).unwrap(); - fs::set_permissions(&closed, fs::Permissions::from_mode(0o700)).unwrap(); - let refused = |path: &Path| match Local::new(path) { - Err(RuntimeError::Io(error)) => { - assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); - error.to_string() - } - other => panic!("{} admitted: {other:?}", path.display()), - }; - let message = refused(&closed.join("root")); - assert!( - message.contains(&format!("{} is not traversable", closed.display())), - "{message}" - ); - // the same through a symlinked parent: the directory it resolves to counts - std::os::unix::fs::symlink(&closed, base.join("link")).unwrap(); - let message = refused(&base.join("link").join("root")); - assert!(message.contains("is not traversable"), "{message}"); - // traversable without being listable is enough - fs::set_permissions(&closed, fs::Permissions::from_mode(0o711)).unwrap(); - Local::new(closed.join("root")).unwrap(); - Local::new(base.join("link").join("root")).unwrap(); -} - -/// The process ids a command printed, one per line. -fn pids(out: &str) -> Vec { - out.lines() - .map(|line| line.trim().parse().unwrap()) - .collect() -} - -/// True once none of `pids` runs any more (a zombie has ended). -fn gone(pids: &[u32]) -> bool { - pids.iter().all( - |pid| match fs::read_to_string(format!("/proc/{pid}/stat")) { - Ok(stat) => matches!( - stat.rsplit_once(") ").map(|(_, rest)| &rest[..1]), - Some("Z" | "X") - ), - Err(_) => true, - }, - ) -} - -/// Leaves behind what an exec can: a `nohup` server writing into the -/// Sandbox's scratch directory, and a `setsid` one named with -/// [`Sandbox::track`] as the SDK names its background processes. -fn leave_running(sandbox: &Sandbox) -> Vec { - let home = format!("HOME={}", sandbox.tmp().display()); - let (code, out, err) = run( - sandbox, - &[ - "bash", - "-c", - "cd \"$HOME\"; nohup sh -c 'while :; do echo x >> \"$HOME/server.out\"; sleep 0.01; done' >/dev/null 2>&1 & echo $!", - ], - &[&home], - ); - assert_eq!(code, 0, "{err}"); - let mut left = pids(&out); - let (code, out, err) = run( - sandbox, - &[ - "bash", - "-c", - "setsid sleep 300 >/dev/null 2>&1 = 2, "{named}"); - let (code, _, _) = run(sandbox, &["true"], &[]); - assert_eq!(code, 0); - assert_eq!(groups(), named); - let deadline = Instant::now() + WAIT; - while !sandbox.tmp().join("server.out").exists() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!gone(&left)); - left -} - -#[test] -fn what_commands_leave_running_ends_with_the_sandbox() { - // stopped and deleted by its owner - let (root, daemon) = fixture(LISTENS); - let mut sandbox = started(&root, &daemon, "layerfs-shop-8"); - let left = leave_running(&sandbox); - sandbox.stop(1).unwrap(); - assert!(gone(&left), "{left:?} outlived the stop"); - sandbox.delete().unwrap(); - assert!(!root.join("layerfs-shop-8").exists()); - // its daemon killed from outside, then found stopped from its record - // and deleted, as a reclaim does - let sandbox = started(&root, &daemon, "layerfs-shop-9"); - let left = leave_running(&sandbox); - let process = sandbox.process().unwrap(); - assert!(Command::new("kill") - .args(["-9", &process.pid.to_string()]) - .status() - .unwrap() - .success()); - let deadline = Instant::now() + WAIT; - while sandbox.endpoint().is_ok() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(20)); - } - assert!(!gone(&left), "a command ended with its daemon"); - let directory = sandbox.directory().to_owned(); - let mut stopped = Sandbox::stopped( - directory.clone(), - sandbox.identity(), - process, - 40404, - Vec::new(), - ); - stopped.delete().unwrap(); - assert!(gone(&left), "{left:?} outlived the delete"); - assert!(!directory.exists()); -} - -#[test] -fn a_process_of_another_boot_is_not_alive_and_one_without_a_boot_is_never_signalled() { - let (root, daemon) = fixture(LISTENS); - let sandbox = started(&root, &daemon, "layerfs-shop-10"); - let process = sandbox.process().unwrap(); - let boot = fs::read_to_string("/proc/sys/kernel/random/boot_id").unwrap(); - let boot = u128::from_str_radix(&boot.trim().replace('-', ""), 16).unwrap(); - assert_eq!(process.boot, Some(boot)); - // the same id and start time in another boot (a reboot or a restored - // snapshot) is another process - let other = LocalProcess { - boot: Some(boot ^ 1), - ..process - }; - assert!(!other.alive()); - assert!(Sandbox::reopen( - root.join("layerfs-shop-10"), - sandbox.identity(), - other, - 40404, - Vec::new() - ) - .unwrap_err() - .container_stopped()); - // a record from before boot ids is matched by id and start as then, and - // a stop through it signals nothing - let legacy = LocalProcess { - boot: None, - ..process - }; - assert!(legacy.alive()); - let mut reopened = Sandbox::reopen( - root.join("layerfs-shop-10"), - sandbox.identity(), - legacy, - 40404, - Vec::new(), - ) - .unwrap(); - assert!(reopened.stop(1).is_err()); - assert!(process.alive()); - let mut sandbox = sandbox; - sandbox.stop(1).unwrap(); - assert!(!process.alive()); - sandbox.delete().unwrap(); -} - -/// A daemon whose main thread ends while another thread still runs: its -/// id is then a zombie that cannot be reaped yet, so it is not alive, but -/// its exit has not happened. A stop ends and reaps it before returning, -/// so it never stays a zombie of this process (the stand-in needs python3). -#[test] -fn a_stop_reaps_a_daemon_whose_main_thread_ended_first() { - let python = Command::new("python3").args(["-c", ""]).status(); - if !python.is_ok_and(|status| status.success()) { - eprintln!("skipped: the stand-in daemon needs python3"); - return; - } - let (root, daemon) = fixture( - r#"echo "LAYERFS_DAEMON_LISTEN 127.0.0.1:40404" -exec python3 -c 'import ctypes, threading, time; threading.Thread(target=time.sleep, args=(60,)).start(); ctypes.CDLL(None).pthread_exit(None)'"#, - ); - let mut sandbox = started(&root, &daemon, "layerfs-shop-11"); - let process = sandbox.process().unwrap(); - let deadline = Instant::now() + WAIT; - while process.alive() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process.alive(), "its main thread did not end"); - let tasks = fs::read_dir(format!("/proc/{}/task", process.pid)) - .unwrap() - .count(); - assert!(tasks >= 2, "its other thread ended already ({tasks})"); - sandbox.stop(1).unwrap(); - assert!( - fs::metadata(format!("/proc/{}", process.pid)).is_err(), - "the stopped daemon is still a process of this kernel (unreaped)" - ); - sandbox.delete().unwrap(); - assert!(!root.join("layerfs-shop-11").exists()); -}