diff --git a/api/src/lib.rs b/api/src/lib.rs index 54124a0..4f989e4 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -388,7 +388,7 @@ impl JobWait { match self { Self::None => true, Self::Start => !matches!(status, Queued { .. }), - Self::Stop => matches!(status, Cancelled { .. } | Error { .. } | Stopped { .. }), + Self::Stop => status.is_terminal(), } } } diff --git a/client/src/cli.rs b/client/src/cli.rs index 6f86bd4..86eadb4 100644 --- a/client/src/cli.rs +++ b/client/src/cli.rs @@ -91,7 +91,7 @@ impl Cli { let path = match BaseDirectories::with_prefix(PREFIX).place_state_file(SESSION_FILE_NAME) { Ok(path) => path, Err(error) => { - eprintln!("⚠️ The session will not persist: {error}"); + eprintln!("❗ The session will not persist: {error}"); return; } }; @@ -102,17 +102,17 @@ impl Cli { session, }) => *self.session.lock().unwrap() = Some(session), Ok(SavedSession { version, .. }) => { - eprintln!("⚠️ Ignoring a version {version} saved session") + eprintln!("❗ Ignoring a version {version} saved session") } - Err(error) => eprintln!("⚠️ Ignoring the saved session: {error}"), + Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), }, Err(error) if error.kind() == ErrorKind::NotFound => (), - Err(error) => eprintln!("⚠️ Ignoring the saved session: {error}"), + Err(error) => eprintln!("❗ Ignoring the saved session: {error}"), } self.session_file = Some(path); match BaseDirectories::with_prefix(PREFIX).place_state_file(TOKEN_FILE_NAME) { Ok(path) => self.token_file = Some(path), - Err(error) => eprintln!("⚠️ Signing tokens will not persist: {error}"), + Err(error) => eprintln!("❗ Signing tokens will not persist: {error}"), } } @@ -149,7 +149,7 @@ impl Cli { }, }; if let Err(error) = result { - eprintln!("⚠️ The session was not saved: {error}"); + eprintln!("❗ The session was not saved: {error}"); } } } @@ -203,6 +203,11 @@ fn short_status_row(status: &JobStatus) -> String { JobStatus::Error { time_error, error, .. } => format!("Error at {time_error}: {error}"), + JobStatus::Skipped { + time_skipped, + reason, + .. + } => format!("Skipped at {time_skipped}: {reason}"), } } @@ -304,7 +309,7 @@ impl CommandContext for Cli { .map_err(io::Error::other) .and_then(|json| write_private(path, &json)); if let Err(error) = result { - eprintln!("⚠️ The token was not saved: {error}"); + eprintln!("❗ The token was not saved: {error}"); } } @@ -900,6 +905,19 @@ impl CommandContext for Cli { Error:\t{error}" ) } + JobStatus::Skipped { + job_id, + time_skipped, + reason, + } => { + println!( + "⏩ Job ID:\t{job_id}\n \ + Target:\t{baseboard_id}\n \ + Job status:\tSkipped\n \ + Skipped at:\t{time_skipped}\n \ + Reason:\t{reason}" + ) + } } } } diff --git a/client/src/commands.rs b/client/src/commands.rs index b7df19b..7f719fb 100644 --- a/client/src/commands.rs +++ b/client/src/commands.rs @@ -51,7 +51,7 @@ use sush_common::interactive::{InteractiveJobError, InteractiveJobMessage}; use sush_common::jobs::JobOutputStream::{self, Stderr, Stdout}; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputHash, JobOutputState, JobStatus, JobStatusMap, - Session, SessionId, SessionSignerNonce, SignedJob, job_status_try_from_json_map, + Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, job_status_try_from_json_map, }; #[cfg(feature = "permslip")] use sush_common::jobs::{JobStartRequest, SessionSushNonce}; @@ -2073,6 +2073,9 @@ async fn job_output_from( Some(JobStatus::Started { job_id, .. }) => { return Err(CommandError::JobStillRunning(job_id.to_owned())); } + Some(JobStatus::Skipped { job_id, reason, .. }) => { + return Err(CommandError::JobSkipped(job_id.to_owned(), *reason)); + } Some(JobStatus::Stopped { output, .. }) => output, }; let len = match stream { @@ -2526,6 +2529,8 @@ pub enum CommandError { JobDidNotRun(JobId), #[error("❌ Job `{0}` is not yet running")] JobNotYetRunning(JobId), + #[error("⏩ Job `{0}` was skipped on this sled: {1}")] + JobSkipped(JobId, SkipReason), #[error("❌ Job `{0}` is still running")] JobStillRunning(JobId), #[error("❌ JSON error: {0}")] diff --git a/client/src/tunnel.rs b/client/src/tunnel.rs index 73c937a..d387152 100644 --- a/client/src/tunnel.rs +++ b/client/src/tunnel.rs @@ -125,7 +125,7 @@ impl Tunnel { let target = Arc::clone(&target); connections.spawn(async move { if let Err(error) = forward(&target, stream).await { - eprintln!("⚠️ Tunnel connection failed: {error}"); + eprintln!("❗ Tunnel connection failed: {error}"); } }); } diff --git a/common/src/authn.rs b/common/src/authn.rs index 38111cc..42966d2 100644 --- a/common/src/authn.rs +++ b/common/src/authn.rs @@ -153,7 +153,7 @@ impl RequestKey { codephrase_newtype! { /// The server half of an ephemeral request-signing key. - #[derive(Clone, Deserialize, Eq, PartialEq, Serialize)] + #[derive(Clone, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct RequestVerifier = Full; } @@ -347,7 +347,7 @@ impl SeqWindow { /// Response to an authentication challenge, containing the server-chosen /// nonce and a fresh client-chosen nonce. This is the structure that is /// signed and verified as authentication credentials. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub struct ChallengeResponse { nonce: Nonce, cnonce: Nonce, @@ -613,6 +613,24 @@ mod test { use super::*; + /// A login is signed, gossiped, and re-verified by every sled, + /// which rebuilds it from the wire to check the signature. Two + /// shapes in one rack disagree about which logins verify. This + /// pin freezes the shape: any field change fails here. Do not + /// re-pin; add a new wire version. + #[test] + fn pin_challenge_response_schema() { + let schema = + serde_json::to_string_pretty(&schemars::schema_for!(ChallengeResponse)).unwrap(); + let path = "tests/output/challenge-response-schema.json"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &schema).unwrap(); + } else { + let expected = std::fs::read_to_string(path).expect("missing snapshot"); + assert_eq!(schema, expected, "the signed login's shape changed"); + } + } + /// Values to be signed must match even across versions. #[test] fn pin_to_be_signed() { diff --git a/common/src/jobs.rs b/common/src/jobs.rs index 186def1..26eeb0e 100644 --- a/common/src/jobs.rs +++ b/common/src/jobs.rs @@ -109,6 +109,7 @@ impl SessionId { LastJob::None => hash(&[b"None", self.0.to_be_bytes().as_slice()].concat()), LastJob::Some(job) => hash(&[b"Some", job.to_be_signed().as_slice()].concat()), LastJob::Burned(job_id) => hash(&[b"Burned", job_id.to_be_bytes().as_slice()].concat()), + LastJob::Resumed(next) => return *next, }) } } @@ -153,6 +154,7 @@ pub enum LastJob { None, Some(SignedJob), Burned(JobId), + Resumed(JobId), } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -201,11 +203,13 @@ impl Session { self.last_job = LastJob::Some(job) } - /// Burn `job_id`, either as the session's next job or by - /// rewinding it from the chain head. The rewind unwinds a - /// signed-but-unrun job on a signer. On a server it converges a - /// skip that raced the start it names, keeping the execution in - /// history. Returns whether the chain moved. + /// Burn `job_id` when it is the session's next job or the job at + /// the chain head. Burning the next job skips it before it runs. + /// Burning the head rewrites the chain to continue from the burn: + /// a signer unwinds a job it signed that never ran, and a server + /// converges with that signer when a skip request arrives after + /// the start of the job it names. An execution already in history + /// stays there. Returns whether the chain moved. pub fn skip_job(&mut self, job_id: JobId) -> bool { if job_id == self.next_job_id() || matches!(&self.last_job, LastJob::Some(job) if *job.job_id() == job_id) @@ -220,6 +224,14 @@ impl Session { pub fn next_job_id(&self) -> JobId { self.session_id.next_job_id(&self.last_job) } + + /// Resume the chain at `successor`, the position a boundary + /// record stored at its last commitment. Every position before + /// `successor` was already handled by the sled that stored the + /// record. + pub fn resume_at(&mut self, successor: JobId) { + self.last_job = LastJob::Resumed(successor); + } } /// How a job runs. The streaming modes allow **unrecorded** I/O. @@ -375,6 +387,37 @@ pub enum JobStatus { result: Result, output: JobOutputState, }, + /// The reporting sled decided it will never run this job. A skip + /// is a decision, not a failure: the job may have run on other + /// sleds, and the operator decides whether to resubmit. + Skipped { + job_id: JobId, + time_skipped: DateTime, + reason: SkipReason, + }, +} + +/// Why a sled will never run a job. +#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum SkipReason { + /// The job's session sits at or below the sled's execution floor, + /// where the sled cannot tell replay from re-run. + BelowFloor, + /// The job's chain position precedes the sled's recorded + /// commitment: a previous life already handled it. + AlreadyHandled, + SessionEnded, +} + +impl fmt::Display for SkipReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::BelowFloor => "the session sits below this sled's execution floor", + Self::AlreadyHandled => "a previous life of this sled already handled it", + Self::SessionEnded => "the session ended before the job could start", + }) + } } pub type JobStatusMap = BTreeMap; @@ -476,7 +519,10 @@ impl JobStatus { pub fn is_terminal(&self) -> bool { matches!( self, - Self::Cancelled { .. } | Self::Error { .. } | Self::Stopped { .. } + Self::Cancelled { .. } + | Self::Error { .. } + | Self::Stopped { .. } + | Self::Skipped { .. } ) } @@ -488,12 +534,13 @@ impl JobStatus { Self::Error { time_error, .. } => *time_error, Self::Started { time_started, .. } => *time_started, Self::Stopped { time_stopped, .. } => *time_stopped, + Self::Skipped { time_skipped, .. } => *time_skipped, } } pub fn time_elapsed(&self) -> TimeDelta { match self { - Self::Cancelled { .. } | Self::Error { .. } => TimeDelta::zero(), + Self::Cancelled { .. } | Self::Error { .. } | Self::Skipped { .. } => TimeDelta::zero(), Self::Queued { time_queued, .. } => Utc::now() - time_queued, Self::Started { time_started, .. } => Utc::now() - time_started, Self::Stopped { @@ -510,7 +557,8 @@ impl JobStatus { | Self::Queued { job_id, .. } | Self::Error { job_id, .. } | Self::Started { job_id, .. } - | Self::Stopped { job_id, .. } => job_id, + | Self::Stopped { job_id, .. } + | Self::Skipped { job_id, .. } => job_id, } } @@ -685,6 +733,23 @@ mod test { use crate::keys::{EccR, EccS, EncodedSignature}; use crate::targets::SledId; + /// A job request is signed, and every sled rebuilds it from the + /// wire to check the signature, so two sleds with different + /// request shapes disagree about what verifies. This pin freezes + /// the shape: any field change fails here. Do not re-pin; add a + /// new wire version. + #[test] + fn pin_job_start_request_schema() { + let schema = serde_json::to_string_pretty(&schemars::schema_for!(JobStartRequest)).unwrap(); + let path = "tests/output/job-start-request-schema.json"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &schema).unwrap(); + } else { + let expected = std::fs::read_to_string(path).expect("missing snapshot"); + assert_eq!(schema, expected, "the signed request's shape changed"); + } + } + /// A request's defaulted fields stay out of the signed material, /// so a signature made before a field existed still verifies /// after it is added. The literal hash pins the scheme for diff --git a/common/src/keys.rs b/common/src/keys.rs index 6d62799..bb89ac6 100644 --- a/common/src/keys.rs +++ b/common/src/keys.rs @@ -113,6 +113,14 @@ impl<'de> Deserialize<'de> for SshPublicKey { } impl SshPublicKey { + pub fn to_openssh(&self) -> Result { + Ok(self.0.to_openssh()?) + } + + pub fn from_openssh(openssh: &str) -> Result { + Ok(Self(ssh_key::PublicKey::from_openssh(openssh)?)) + } + pub fn key_id(&self) -> Result { KeyId::try_from(&self.0) } diff --git a/common/tests/output/challenge-response-schema.json b/common/tests/output/challenge-response-schema.json new file mode 100644 index 0000000..f57ce26 --- /dev/null +++ b/common/tests/output/challenge-response-schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ChallengeResponse", + "description": "Response to an authentication challenge, containing the server-chosen nonce and a fresh client-chosen nonce. This is the structure that is signed and verified as authentication credentials.", + "type": "object", + "required": [ + "cnonce", + "epk", + "nonce" + ], + "properties": { + "cnonce": { + "$ref": "#/definitions/Nonce" + }, + "epk": { + "$ref": "#/definitions/RequestVerifier" + }, + "nonce": { + "$ref": "#/definitions/Nonce" + } + }, + "definitions": { + "Nonce": { + "description": "A unique random string. Authentication credentials have two of these: one generated by the server, and one by the client. This structure is agnostic to the syntax of the string.", + "type": "string" + }, + "RequestVerifier": { + "description": "The server half of an ephemeral request-signing key.", + "type": "string" + } + } +} \ No newline at end of file diff --git a/common/tests/output/job-start-request-schema.json b/common/tests/output/job-start-request-schema.json new file mode 100644 index 0000000..c5cd765 --- /dev/null +++ b/common/tests/output/job-start-request-schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "JobStartRequest", + "description": "A request to run the given `command` as `job_id`.", + "type": "object", + "required": [ + "command", + "job_id", + "session_id" + ], + "properties": { + "command": { + "type": "string" + }, + "job_id": { + "$ref": "#/definitions/JobId" + }, + "mode": { + "$ref": "#/definitions/JobMode" + }, + "session_id": { + "$ref": "#/definitions/SessionId" + }, + "target": { + "description": "The sleds this job runs on.", + "type": "string" + } + }, + "definitions": { + "JobId": { + "description": "A globally unique identifier for a job within a session.", + "type": "string" + }, + "JobMode": { + "description": "How a job runs. The streaming modes allow **unrecorded** I/O.", + "type": "string", + "enum": [ + "batch", + "interactive", + "stream-input", + "stream-output" + ] + }, + "SessionId": { + "description": "A globally unique identifier for a session.", + "type": "string" + } + } +} \ No newline at end of file diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 59277fe..ef662bf 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.97.1" +channel = "1.98.1" components = ["clippy", "rustfmt"] diff --git a/server/Cargo.toml b/server/Cargo.toml index ce03ab9..a4b1bd4 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,6 +18,7 @@ test-support = [] [dependencies] atomicwrites.workspace = true bytes.workspace = true +ciborium.workspace = true camino.workspace = true bytesize.workspace = true chrono.workspace = true diff --git a/server/src/bloom.rs b/server/src/bloom.rs new file mode 100644 index 0000000..5ede9cd --- /dev/null +++ b/server/src/bloom.rs @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A fixed-size Bloom filter for durable, grow-only sets. +//! +//! Designed for the boundary record's burned networks set. Entries may +//! never be dropped (once burned, a network stays burned), the size must +//! stay bounded under adversarial growth (we must not fill the M.2s), +//! and the filter may err only by claiming a key it never held. The +//! caller must treat that claim as refusal. +//! +//! The layout and hash algorithm determine the on-disk format, and must +//! not be changed without updating the boundary record's magic. Each key +//! probes the table at seven positions, cut as 16-bit words from its +//! SHA3-256 digest. The table is currently sized at 8192 bits, for +//! which seven probes is the optimal count out to about 800 entries +//! (bits * ln 2 / probes). The false positive rate there is about 0.7%, +//! degrading gradually past it. Real occupancy should stay in the tens, +//! where false positives are negligible. + +use std::array::from_fn; +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use sush_common::hash::{OUT_LEN, hash}; +use sush_common::wire::ExactBytes; + +const BYTES: usize = 1024; +const BITS: u64 = (BYTES * 8) as u64; +const PROBES: usize = 7; + +// The probes must fit in the digest, and the table size must divide +// 2^16, or reducing a 16-bit word to a table slot would favor some +// slots over others. +const _: () = { + assert!(2 * PROBES <= OUT_LEN); + assert!((1u64 << 16).is_multiple_of(BITS)); +}; + +/// A grow-only set of byte-string keys. +#[derive(Clone, Eq, PartialEq)] +pub struct Bloom { + bits: Box<[u8; BYTES]>, +} + +/// The probe positions for `key` are seven 16-bit words cut from +/// the leading bytes of the key's SHA3-256 digest, each reduced to +/// a table slot. +fn probes(key: &[u8]) -> [u64; PROBES] { + let digest = *hash(key).as_bytes(); + from_fn(|i| { + let j = 2 * i; + let word = u16::from_le_bytes(digest[j..j + 2].try_into().expect("two bytes")); + u64::from(word) % BITS + }) +} + +/// The byte index and mask selecting `probe`'s bit. +fn bit(probe: u64) -> (usize, u8) { + ((probe / 8) as usize, 1 << (probe % 8)) +} + +impl Bloom { + pub fn new() -> Self { + Self { + bits: Box::new([0; BYTES]), + } + } + + /// Permanently add `key` to the set. + pub fn insert(&mut self, key: &[u8]) { + for (byte, mask) in probes(key).map(bit) { + self.bits[byte] |= mask; + } + } + + /// Whether `key` may have been inserted. Never returns a false negative, + /// but false positives occur at the documented rate. + pub fn contains(&self, key: &[u8]) -> bool { + probes(key) + .map(bit) + .into_iter() + .all(|(byte, mask)| self.bits[byte] & mask != 0) + } +} + +impl Default for Bloom { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for Bloom { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let set: u32 = self.bits.iter().map(|byte| byte.count_ones()).sum(); + write!(f, "Bloom({set}/{BITS} bits set)") + } +} + +impl Serialize for Bloom { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_bytes(&self.bits[..]) + } +} + +impl<'de> Deserialize<'de> for Bloom { + fn deserialize>(deserializer: D) -> Result { + let bits = deserializer.deserialize_bytes(ExactBytes::)?; + Ok(Self { + bits: Box::new(bits), + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn round_trip() { + let mut bloom = Bloom::new(); + assert!(!bloom.contains(b"lost-network")); + bloom.insert(b"lost-network"); + assert!(bloom.contains(b"lost-network")); + assert!(!bloom.contains(b"other-network")); + } + + #[test] + fn serde_preserves_bits() { + let mut bloom = Bloom::new(); + bloom.insert(b"burned"); + let mut bytes = Vec::new(); + ciborium::ser::into_writer(&bloom, &mut bytes).unwrap(); + let back: Bloom = ciborium::de::from_reader(bytes.as_slice()).unwrap(); + assert_eq!(bloom, back); + assert!(back.contains(b"burned")); + } + + #[test] + fn false_positives_are_rare() { + let mut bloom = Bloom::new(); + for i in 0..128 { + bloom.insert(format!("burned-{i}").as_bytes()); + } + let hits = (0..10_000) + .filter(|i| bloom.contains(format!("probe-{i}").as_bytes())) + .count(); + assert_eq!( + hits, 0, + "false positives at plausible occupancy: {hits}/10000" + ); + } + + /// If this test fails, STOP! The on-disk format may have changed! + #[test] + fn pin_probes() { + assert_eq!(probes(b"sush"), [3392, 1007, 8018, 5416, 1335, 7804, 1908]); + } +} diff --git a/server/src/bookmark.rs b/server/src/bookmark.rs index 7ccd8a9..cdb22c1 100644 --- a/server/src/bookmark.rs +++ b/server/src/bookmark.rs @@ -4,143 +4,95 @@ //! Durable gossip peer identity across restarts. //! -//! A rumors [`Bookmark`] records who a peer is and how far it has -//! advanced, so a restarted sled reclaims its old identity instead of -//! stranding it. Rumors owns the record format and decides when to load -//! and store. We supply raw byte storage obeying two constraints: stores -//! are atomic, and a load never returns a record older than the newest -//! store we reported `Ok` (stale records corrupt causality, whereas -//! lost records merely strand identities). +//! A rumors [`Bookmark`] records a peer's identity and how far it has +//! advanced, so that a restarted sled may reclaim its previous identity +//! instead of stranding it. The invariant is (as usual) that we must +//! never adopt stale data, because in this case it could lead to causality +//! violations (which are bad). //! -//! Storage is one small file per configured (M.2) slot. Loads read every -//! slot, and take the record with the highest sequence number; that slot -//! becomes the *home*. Stores go only to the home, since writing both -//! would either make the server dependent on the health of both or, done -//! merely best-effort, let a stale record load after a fresher disk dies, -//! violating the constraint above. The slots must never both be written -//! by live peers, and a record must never be restored from a backup. -//! A [`BookmarkSource`] hands out one handle per peer, with generation -//! numbers ensuring that a straggler from an abandoned universe can't -//! clobber its successor's record. - -use std::fs::Permissions; -use std::io::{self, Cursor, Write as _}; -use std::os::unix::fs::PermissionsExt as _; +//! The record format and when to load & store are dictated by rumors. +//! We use a [`Tenant`] of a [`Locker`] to store it on disk(s); +//! if a load fails or the slots disagree, we assume a new identity +//! rather than risk resuming with a stale one. The record keeps every +//! universe's identities, so a lost write costs at most a stranded +//! identity, never a stale one. + +use std::io::{self, Cursor}; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use atomicwrites::{AtomicFile, OverwriteBehavior}; -use camino::{Utf8Path, Utf8PathBuf}; use rumors::{Bookmark, BookmarkError, Serialized}; -use slog::{Logger, o, warn}; +use slog::{Discard, Logger, o, warn}; use thiserror::Error; -use tokio::fs::read; use tokio::io::AsyncWrite; -use tokio::sync::Mutex; -use tokio::task::spawn_blocking; - -/// The envelope magic. The payload includes its own magic. -/// A change to our envelope means a new magic. -const MAGIC: &[u8; 12] = b"SUSHBOOKMARK"; - -/// Envelope layout: magic, big-endian sequence number, digest of the -/// sequence number and record together, record. The digest keeps a -/// damaged sequence number from silently reordering the slots. -const SEQ_LEN: usize = 8; -const DIGEST_LEN: usize = 32; - -fn digest(seq: u64, record: &[u8]) -> [u8; DIGEST_LEN] { - let mut hasher = sush_common::hash::Hasher::new(); - hasher.update(&seq.to_be_bytes()); - hasher.update(record); - *hasher.finalize().as_bytes() + +use crate::format::{self, NoFormat, Record, Versioned}; +use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; + +pub const BOOKMARK: TenantSpec = TenantSpec { + file: "bookmark", + magic: b"SUSHBOOKMARK", +}; + +/// Wrap the opaque bytes rumors writes. +#[derive(serde::Deserialize, serde::Serialize)] +struct BookmarkRecord(#[serde(with = "format::cbor_bytes")] Vec); + +impl Versioned for BookmarkRecord { + const VERSION: u16 = 0; +} + +impl Record for BookmarkRecord { + type Previous = NoFormat; +} + +impl TryFrom for BookmarkRecord { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } } /// What a bookmark load or store failed at. #[derive(Debug, Error)] pub enum BookmarkIoError { - #[error("bookmark I/O failed on `{path}`: {error}")] - Io { - path: Utf8PathBuf, - #[source] - error: io::Error, - }, - #[error("every bookmark slot is corrupt (last: `{path}`)")] - Corrupt { path: Utf8PathBuf }, #[error("serializing the bookmark record failed: {0}")] Serialize(#[source] io::Error), - #[error("no bookmark slot is writable")] - NoSlot, - #[error("the bookmark was handed to a newer peer")] - Fenced, + #[error("storing the bookmark failed: {0}")] + Store(#[source] StoreError), } -/// This server's bookmark storage. Hands out one handle per peer, -/// each superseding the last. +/// This server's bookmark storage. Every handle shares the one record. #[derive(Clone, Debug)] pub struct BookmarkSource { - shared: Arc, -} - -#[derive(Debug)] -struct SharedStore { log: Logger, - /// Candidate record files, one per boot M.2. Empty means this - /// server persists no identity (the standalone server, tests). - slots: Vec, - /// The newest generation. A handle from an older one may load - /// but not store. - generation: AtomicU64, - /// Serializes loads and stores across handles, so the newest - /// record on disk is always the newest store anyone `Ok`'d. - state: Mutex, - /// The newest sequence number committed to disk by this process, - /// under the lock every rename takes. Renames commit in sequence - /// order or not at all; see [`commit`]. - committed: std::sync::Mutex, -} - -#[derive(Debug, Default)] -struct StoreState { - /// The slot holding the newest record, once known. - home: Option, - /// The sequence number of the newest record. - seq: u64, + tenant: Arc, } impl BookmarkSource { - /// A source persisting to `slots`, each on its own device. - /// - /// Construct exactly one source per slot set per process, and feed - /// that same source to both [`seed_gossip`](crate::seed_gossip) - /// and [`spawn_gossip`](crate::gossip::spawn_gossip): everything - /// serializing the store lives inside it. The caller creates the - /// parent directories, one per boot M.2, writable by this server's - /// user. The record files are created and owned here. - pub fn new(log: &Logger, slots: Vec) -> Self { + /// A source persisting to `locker`. + /// [`Seed::grow`](crate::gossip::Seed::grow) makes the one source + /// a locker gets per process. + pub fn new(log: &Logger, locker: &Locker) -> Self { Self { - shared: Arc::new(SharedStore { - log: log.new(o!("component" => "bookmark")), - slots, - generation: AtomicU64::new(0), - state: Mutex::new(StoreState::default()), - committed: std::sync::Mutex::new(0), - }), + log: log.new(o!("component" => "bookmark")), + tenant: Arc::new(locker.tenant(BOOKMARK)), } } /// A source that loads and persists nothing. pub fn null() -> Self { - Self::new(&Logger::root(slog::Discard, o!()), Vec::new()) + Self::new(&Logger::root(Discard, o!()), &Locker::null()) } - /// A ratcheting handle for the next peer. - /// All earlier handles are superseded. - pub fn next_handle(&self) -> SushBookmark { - let generation = self.shared.generation.fetch_add(1, Ordering::SeqCst) + 1; + /// A persisting handle for a peer. Rumors persists a bookmark only + /// when a gossip session starts, and the gossip manager stops + /// every session before it hands a new peer its handle, so no two + /// peers persist concurrently; see the migration notes in + /// [`gossip`](crate::gossip). + pub fn handle(&self) -> SushBookmark { SushBookmark { - shared: self.shared.clone(), - generation, + log: self.log.clone(), + tenant: self.tenant.clone(), shed: false, } } @@ -149,109 +101,21 @@ impl BookmarkSource { /// gossiping after its real bookmark failed. pub fn shed_handle(&self) -> SushBookmark { SushBookmark { - shared: self.shared.clone(), - generation: 0, + log: self.log.clone(), + tenant: self.tenant.clone(), shed: true, } } - - /// A probing handle: reads like the current peer's, but without - /// superseding anything. - fn probe_handle(&self) -> SushBookmark { - SushBookmark { - shared: self.shared.clone(), - generation: self.shared.generation.load(Ordering::SeqCst), - shed: false, - } - } - - /// Does the storage work? Reads every slot and proves at least one - /// writable by writing. - pub async fn probe(&self) -> Result<(), BookmarkIoError> { - if self.shared.slots.is_empty() { - return Ok(()); - } - let usable = match self.probe_handle().load().await { - Ok(_) => { - let mut writable = false; - for path in &self.shared.slots { - let probe = path.with_extension("probe"); - if tokio::fs::write(&probe, b"").await.is_ok() { - let _ = tokio::fs::remove_file(&probe).await; - writable = true; - break; - } - } - if writable { - Ok(()) - } else { - Err(BookmarkIoError::NoSlot) - } - } - Err(error) => Err(error), - }; - if let Err(error) = &usable { - warn!(self.shared.log, "no usable bookmark storage"; "error" => %error); - } - usable - } } /// One peer's handle on the [`BookmarkSource`]. #[derive(Debug)] pub struct SushBookmark { - shared: Arc, - generation: u64, + log: Logger, + tenant: Arc, shed: bool, } -impl SushBookmark { - /// Has this bookmark been overtaken by events? - fn obe(&self) -> bool { - self.generation < self.shared.generation.load(Ordering::SeqCst) - } - - /// Split a checksummed envelope into its sequence number and record. - fn parse(bytes: &[u8]) -> Option<(u64, Vec)> { - let payload = bytes.strip_prefix(MAGIC)?; - let (seq, rest) = payload.split_first_chunk::()?; - let (sum, record) = rest.split_first_chunk::()?; - let seq = u64::from_be_bytes(*seq); - (digest(seq, record) == *sum).then(|| (seq, record.to_vec())) - } - - /// Build the checksummed envelope around `record`. - fn envelope(seq: u64, record: &[u8]) -> Vec { - let mut bytes = Vec::with_capacity(MAGIC.len() + SEQ_LEN + DIGEST_LEN + record.len()); - bytes.extend_from_slice(MAGIC); - bytes.extend_from_slice(&seq.to_be_bytes()); - bytes.extend_from_slice(&digest(seq, record)); - bytes.extend_from_slice(record); - bytes - } -} - -/// Rename `envelope` into place iff `seq` is newer than everything -/// committed by this process. A store future dropped at its await -/// detaches the blocking write, whose rename would otherwise land on -/// top of the newer record that beat it. Runs on the blocking pool. -fn commit(shared: &SharedStore, path: &Utf8Path, seq: u64, envelope: &[u8]) -> io::Result<()> { - let mut committed = shared.committed.lock().unwrap(); - if seq <= *committed { - return Err(io::Error::other("superseded by a newer record")); - } - AtomicFile::new(path, OverwriteBehavior::AllowOverwrite) - .write(|file| { - file.set_permissions(Permissions::from_mode(0o600))?; - file.write_all(envelope) - }) - .map_err(|error| match error { - atomicwrites::Error::Internal(error) | atomicwrites::Error::User(error) => error, - })?; - *committed = seq; - Ok(()) -} - impl BookmarkError for SushBookmark { type Error = BookmarkIoError; } @@ -260,58 +124,27 @@ impl Bookmark for SushBookmark { type Reader = Cursor>; async fn load(&self) -> Result, Self::Error> { - if self.shed || self.shared.slots.is_empty() { + if self.shed { return Ok(None); } - if self.obe() { - return Err(BookmarkIoError::Fenced); - } - let mut state = self.shared.state.lock().await; - let mut newest: Option<(u64, usize, Vec)> = None; - let mut corrupt: Option<&Utf8Path> = None; - for (index, path) in self.shared.slots.iter().enumerate() { - let bytes = match read(path).await { - Ok(bytes) => bytes, - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) => { - return Err(BookmarkIoError::Io { - path: path.clone(), - error, - }); - } - }; - match Self::parse(&bytes) { - Some((seq, record)) => { - if newest.as_ref().is_none_or(|(newest, ..)| seq > *newest) { - newest = Some((seq, index, record)); + let mut guard = self.tenant.lock().await; + match guard.load().await { + Verdict::Adopt(record) | Verdict::Restore(record) => { + match format::decode::(&record) { + Ok(BookmarkRecord(bytes)) => Ok(Some(Cursor::new(bytes))), + // Stranding the old identity is harmless; + // resuming from a misread record is not. + Err(error) => { + warn!(self.log, "assuming a fresh identity"; "reason" => %error); + Ok(None) } } - None => { - warn!( - self.shared.log, "skipping corrupt bookmark slot"; - "path" => %path, - ); - corrupt = Some(path); - } } - } - match newest { - Some((seq, home, record)) => { - // A reserved sequence number outranks a re-read of the - // disk. Regressing would let a cancelled write's - // straggler collide with a fresh reservation. - if seq >= state.seq { - state.home = Some(home); - state.seq = seq; - } - Ok(Some(Cursor::new(record))) + Verdict::Empty => Ok(None), + Verdict::Discard(reason) => { + warn!(self.log, "assuming a fresh identity"; "reason" => %reason); + Ok(None) } - // A present-but-unreadable record is an error: - // rumors must not mistake it for a fresh start. - None => match corrupt { - Some(path) => Err(BookmarkIoError::Corrupt { path: path.into() }), - None => Ok(None), - }, } } @@ -319,52 +152,18 @@ impl Bookmark for SushBookmark { where F: for<'a> FnOnce(&'a mut (dyn AsyncWrite + Unpin + Send)) -> Serialized<'a> + Send, { - if self.shed || self.shared.slots.is_empty() { + if self.shed { return Ok(()); } - let mut buf = Cursor::new(Vec::new()); write(&mut buf).await.map_err(BookmarkIoError::Serialize)?; - let record = buf.into_inner(); + let record = BookmarkRecord(buf.into_inner()); - let mut state = self.shared.state.lock().await; - if self.obe() { - return Err(BookmarkIoError::Fenced); - } - let home = match state.home { - Some(home) => home, - None => self - .shared - .slots - .iter() - .position(|path| { - path.parent() - .is_some_and(|parent| parent.as_std_path().is_dir()) - }) - .ok_or(BookmarkIoError::NoSlot)?, - }; - let path = self.shared.slots[home].clone(); - - // Reserve the sequence number first, since a cancelled write - // may still land and must be outnumbered. - state.seq += 1; - let seq = state.seq; - let envelope = Self::envelope(seq, &record); - - let shared = self.shared.clone(); - let target = path.clone(); - let written = spawn_blocking(move || commit(&shared, &target, seq, &envelope)) + let mut guard = self.tenant.lock().await; + guard + .store(&format::encode(&record)) .await - .map_err(|join| io::Error::other(join.to_string())) - .and_then(|result| result); - - match written { - Ok(()) => { - state.home = Some(home); - Ok(()) - } - Err(error) => Err(BookmarkIoError::Io { path, error }), - } + .map_err(BookmarkIoError::Store) } } @@ -372,7 +171,7 @@ impl Bookmark for SushBookmark { mod test { use super::*; - use std::fs::{create_dir, metadata, read, write}; + use std::fs::create_dir; use camino::Utf8PathBuf; use tempfile::TempDir; @@ -385,20 +184,24 @@ mod test { move |w| Box::pin(async move { w.write_all(bytes).await }) } - /// Two slot paths in separate directories, like two M.2s. + /// Two slot directories, like two M.2s. fn slots(dir: &TempDir) -> Vec { ["m2a", "m2b"] .iter() .map(|m2| { - let parent = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); - create_dir(&parent).unwrap(); - parent.join("bookmark") + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot }) .collect() } - fn envelope(seq: u64, record: &[u8]) -> Vec { - SushBookmark::envelope(seq, record) + fn test_log() -> Logger { + Logger::root(Discard, o!()) + } + + fn source(slots: Vec) -> BookmarkSource { + BookmarkSource::new(&test_log(), &Locker::new(&test_log(), slots)) } async fn read_back(handle: &SushBookmark) -> Option> { @@ -408,157 +211,55 @@ mod test { Some(bytes) } - fn test_log() -> Logger { - Logger::root(slog::Discard, o!()) - } - - /// A stored record loads back verbatim, sequenced and private. + /// A stored record loads back verbatim. #[tokio::test] async fn round_trip() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - let source = BookmarkSource::new(&test_log(), slots.clone()); + let source = source(slots(&dir)); - let handle = source.next_handle(); + let handle = source.handle(); assert!(read_back(&handle).await.is_none()); handle.store(record(b"who we are")).await.unwrap(); assert_eq!(read_back(&handle).await.unwrap(), b"who we are"); - - let bytes = read(&slots[0]).unwrap(); - assert_eq!(bytes, envelope(1, b"who we are")); - let mode = metadata(&slots[0]).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o600); - } - - /// The newest record wins the load regardless of slot, and its - /// slot becomes the home every store then writes. - #[tokio::test] - async fn newest_slot_is_home() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - write(&slots[0], envelope(5, b"stale")).unwrap(); - write(&slots[1], envelope(9, b"fresh")).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - let handle = source.next_handle(); - assert_eq!(read_back(&handle).await.unwrap(), b"fresh"); - - handle.store(record(b"fresher")).await.unwrap(); - assert_eq!(read(&slots[0]).unwrap(), envelope(5, b"stale")); - assert_eq!(read(&slots[1]).unwrap(), envelope(10, b"fresher")); - } - - /// Minting a new handle fences the old one's stores. - #[tokio::test] - async fn stale_generations_cannot_store() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); - - let old = source.next_handle(); - old.store(record(b"before")).await.unwrap(); - let new = source.next_handle(); - assert!(matches!( - old.store(record(b"after")).await, - Err(BookmarkIoError::Fenced) - )); - assert_eq!(read_back(&new).await.unwrap(), b"before"); - } - - /// A corrupt slot is skipped when another is valid, and is an - /// error rather than absence when nothing valid remains. - #[tokio::test] - async fn corruption_is_never_absence() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - write(&slots[0], b"scribble").unwrap(); - write(&slots[1], envelope(3, b"good")).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - assert_eq!(read_back(&source.next_handle()).await.unwrap(), b"good"); - - write(&slots[1], b"more scribble").unwrap(); - assert!(matches!( - source.next_handle().load().await, - Err(BookmarkIoError::Corrupt { .. }) - )); - } - - /// A damaged sequence number fails the digest rather than silently - /// reordering the slots. - #[tokio::test] - async fn a_flipped_sequence_number_is_corruption() { - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - let mut bytes = envelope(3, b"good"); - bytes[MAGIC.len()] ^= 0x80; - write(&slots[0], bytes).unwrap(); - - let source = BookmarkSource::new(&test_log(), slots.clone()); - assert!(matches!( - source.next_handle().load().await, - Err(BookmarkIoError::Corrupt { .. }) - )); } - /// A straggling write from a dropped store future cannot land on - /// top of a newer committed record. + /// A discarded verdict is a fresh start, not an error. #[tokio::test] - async fn stragglers_cannot_clobber_newer_commits() { + async fn discard_assumes_fresh_identity() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); let slots = slots(&dir); - let source = BookmarkSource::new(&test_log(), slots.clone()); - - let newer = envelope(7, b"newer"); - let straggler = envelope(6, b"stale"); - commit(&source.shared, &slots[0], 7, &newer).unwrap(); - assert!(commit(&source.shared, &slots[0], 6, &straggler).is_err()); - assert_eq!(read(&slots[0]).unwrap(), newer); + for (slot, bytes) in slots.iter().zip([b"one", b"two"]) { + let lone = source(vec![slot.clone()]); + lone.handle().store(record(bytes)).await.unwrap(); + } + assert!(read_back(&source(slots).handle()).await.is_none()); } - /// A superseded handle can no longer load: its view of home and - /// sequence state belongs to a dead peer. + /// Handles share the record: one stores, another reads it back. #[tokio::test] - async fn stale_generations_cannot_load() { + async fn handles_share_record() { let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); - let old = source.next_handle(); - old.store(record(b"before")).await.unwrap(); - let _new = source.next_handle(); - assert!(matches!(old.load().await, Err(BookmarkIoError::Fenced))); - } + let source = source(slots(&dir)); - /// Probing proves writability by writing, not by guessing from - /// directory metadata. - #[tokio::test] - async fn probe_rejects_unwritable_storage() { - let source = BookmarkSource::new( - &test_log(), - vec![Utf8PathBuf::from("/nonexistent/sush/bookmark")], - ); - assert!(matches!(source.probe().await, Err(BookmarkIoError::NoSlot))); - - let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let source = BookmarkSource::new(&test_log(), slots(&dir)); - source.probe().await.unwrap(); + source.handle().store(record(b"shared")).await.unwrap(); + assert_eq!(read_back(&source.handle()).await.unwrap(), b"shared"); } - /// A slotless source and a shed handle persist nothing and never - /// fail, and a shed handle ignores even an existing record. + /// A null source and a shed handle persist nothing and never fail, + /// and a shed handle ignores even an existing record. #[tokio::test] - async fn none_and_shed_touch_nothing() { - let source = BookmarkSource::null(); - let handle = source.next_handle(); - assert!(read_back(&handle).await.is_none()); + async fn null_and_shed_touch_nothing() { + let null = BookmarkSource::null(); + let handle = null.handle(); handle.store(record(b"lost")).await.unwrap(); assert!(read_back(&handle).await.is_none()); let dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slots = slots(&dir); - write(&slots[0], envelope(7, b"kept")).unwrap(); - let source = BookmarkSource::new(&test_log(), slots.clone()); + let source = source(slots(&dir)); + source.handle().store(record(b"kept")).await.unwrap(); let shed = source.shed_handle(); assert!(read_back(&shed).await.is_none()); shed.store(record(b"dropped")).await.unwrap(); - assert_eq!(read(&slots[0]).unwrap(), envelope(7, b"kept")); + assert_eq!(read_back(&source.handle()).await.unwrap(), b"kept"); } } diff --git a/server/src/boundary.rs b/server/src/boundary.rs new file mode 100644 index 0000000..8514e3c --- /dev/null +++ b/server/src/boundary.rs @@ -0,0 +1,565 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The boundary between jobs we executed and jobs we only heard about. +//! +//! One record, rewritten before each spawn, carries what this sled +//! last committed to: the job, its session, the chain position after +//! the job, the universe they belong to, and the job's ending once +//! it has one. A universe is one shared gossip history, identified +//! by its network; sleds join universes, leave them, and sometimes +//! return. Alongside the commitment, the record keeps the join +//! of every session start this sled has executed under in that +//! universe, and the set of universes it burned by leaving. The +//! record is this sled's execution watermark: nothing at or below it +//! may run again. +//! +//! After a restart, replayed gossip rebuilds the sessions. When the +//! committed session activates, the sled resumes its chain at the +//! stored successor. The previous life had already moved the chain +//! past every earlier position, so the session's queue never +//! releases them here, and the chain continues from the successor +//! whether its request arrives by replay or by resubmission. A +//! session that starts strictly above the executed join has never +//! run here and is served. A session the record cannot order makes +//! the sled hop: the sled reports the hop to the gossip set as an +//! error and sets a floor, in memory, at the frontier that includes +//! the report. A session that does not start above the floor has +//! its jobs skipped; a session started after the hop is served. The +//! join folds in every session this sled has executed under in this +//! universe, so a session it once ran under can never screen as +//! new: the sled hops, and the session's jobs are skipped rather +//! than re-run. If replay gives the recorded job no status, the sled +//! adjudicates it: it announces the recorded ending when the record +//! holds one, and an interrupted ending when it does not. +//! +//! Universes have no order. The record instead keeps a burned set, +//! holding the network of every universe whose watermark it +//! overwrote by moving on. A sled that re-enters a burned universe +//! has flip-flopped, and raises its floor: it reports the flip-flop +//! to the gossip set and sets the floor, in memory, at the frontier +//! that includes the report. No older message can contain a version +//! born at that instant, so a session started before the re-entry +//! never lies above the floor, and its jobs are skipped. A sled that +//! enters a universe with history while holding no record for it +//! raises a floor the same way. The floor is never persisted: the +//! burn, the missing record, or the unordered session is still there +//! after a restart, and raises it again. A floor written to disk +//! could carry a version that died with the life that created it; +//! no later session could ever dominate such a floor. +//! +//! A boundary that cannot be written means the job must not run. A +//! boundary that cannot be trusted means no job may run at all, since +//! we cannot tell what the previous life committed to. Recovery is an +//! M.2 swap or a clean slate. + +use std::sync::Mutex as SyncMutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use ciborium::ser::into_writer as into_cbor; +use rumors::{Network, Version}; +use serde::{Deserialize, Serialize}; +use slog::{Logger, o, warn}; +use thiserror::Error; + +use sush_common::jobs::{JobId, JobStatus, ProcessError, SessionId}; + +use crate::bloom::Bloom; +use crate::format::{self, NoFormat, Record, Versioned}; +use crate::locker::{Locker, StoreError, Tenant, TenantSpec, Verdict}; + +pub const BOUNDARY: TenantSpec = TenantSpec { + file: "boundary", + magic: b"SUSHBOUNDARY", +}; + +/// The execution boundary: what this sled last committed to, in which +/// universe, and which universes it has left behind ("burned"). +/// +/// Versions do not compare across universes, so `network` scopes every +/// version in the record. `burned` holds the network of every +/// universe whose watermark this record overwrote by moving on: a +/// sled re-entering one raises its floor in memory, and the record on +/// disk stays the displaced universe's true watermark until a commit +/// overwrites it. +/// +/// `executed` is the join of the start versions of every session +/// this sled has executed under in this universe. A single stored +/// start would forget the sessions before it, and a third session +/// could then replay the first session's jobs; the join never +/// forgets. Session starts are witnessed messages, and only those +/// keep their meaning across a crash. A start that only this sled +/// ever saw belongs to a session whose history is lost, and refusal +/// is the right answer there anyway. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Boundary { + pub network: Network, + pub burned: Bloom, + #[serde(with = "version_bytes")] + pub executed: Version, + pub job: Option, +} + +/// The last job this sled committed to running, and how far it got. +/// We also store the chain position *after* `job`, computed from the +/// request's signed bytes at commit time, to allow session resumption. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Committed { + pub session: SessionId, + pub job: JobId, + pub successor: JobId, + pub outcome: JobOutcome, +} + +impl Boundary { + /// Whether this record burned `network`: left its universe behind + /// and overwrote its watermark. + pub fn is_burned(&self, network: Network) -> bool { + self.burned.contains(&network_key(network)) + } + + /// The burned set for the replacement record, committed in + /// `network`. A replacement in a different universe burns this + /// record's own network. + pub fn burned_for(&self, network: Network) -> Bloom { + let mut burned = self.burned.clone(); + if self.network != network { + burned.insert(&network_key(self.network)); + } + burned + } + + /// The executed-session join for the replacement record, + /// committed in `network` and folding in `started`. Joins never + /// cross universes, so a replacement elsewhere starts its join + /// fresh. + pub fn executed_for(&self, network: Network, started: &Version) -> Version { + if self.network == network { + self.executed.clone() | started.clone() + } else { + started.clone() + } + } +} + +/// A network's Bloom key is its CBOR bytes. +fn network_key(network: Network) -> Vec { + let mut bytes = Vec::new(); + into_cbor(&network, &mut bytes).expect("writing to a Vec cannot fail"); + bytes +} + +mod version_bytes { + use rumors::Version; + use serde::de::Error as _; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(version: &Version, serializer: S) -> Result { + serializer.serialize_bytes(&version.encode()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let bytes = >::deserialize(deserializer)?; + Version::decode(bytes.as_slice()).map_err(D::Error::custom) + } +} + +/// How far the boundary job got. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum JobOutcome { + /// Committed to run, with no ending recorded. + Committed, + /// The job's terminal status, error endings included. + Ended(JobStatus), +} + +/// Version 0 is the shipped baseline. The pinned record snapshot in +/// this module's tests freezes its bytes; see [`crate::format`] for +/// the steps a format change requires. +impl Versioned for Boundary { + const VERSION: u16 = 0; +} + +impl Record for Boundary { + type Previous = NoFormat; +} + +impl TryFrom for Boundary { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } +} + +#[derive(Debug, Error)] +pub enum BoundaryError { + #[error("the boundary store is untrusted")] + Untrusted, + #[error(transparent)] + Store(#[from] StoreError), +} + +/// Durable storage for the [`Boundary`]. +#[derive(Debug)] +pub struct BoundaryStore { + log: Logger, + tenant: Tenant, + /// The record, readable synchronously by the state machine. + boundary: SyncMutex>, + /// Untrusted until loaded, and forever if the load discards: a + /// write would overwrite the disagreeing slots, and the next load + /// would see agreement that never happened. + untrusted: AtomicBool, + loaded: AtomicBool, +} + +impl BoundaryStore { + pub fn new(log: &Logger, locker: &Locker) -> Self { + Self { + log: log.new(o!("component" => "boundary")), + tenant: locker.tenant(BOUNDARY), + boundary: SyncMutex::new(None), + untrusted: AtomicBool::new(true), + loaded: AtomicBool::new(false), + } + } + + /// Load the stored record once, at startup, before any job runs. + /// A later load could regress the in-memory record below a + /// spawned job, so a second call panics. + pub async fn load(&self) { + assert!( + !self.loaded.swap(true, Ordering::SeqCst), + "the boundary store loads once, at startup", + ); + let boundary = match self.tenant.load().await { + Verdict::Adopt(record) | Verdict::Restore(record) => { + match format::decode::(&record) { + Ok(boundary) => Some(boundary), + // An unreadable record is not an absent one: + // absent would mean a clean slate, forgetting the + // previous life's commitments. The store stays + // untrusted instead, and no job runs. + Err(error) => { + warn!(self.log, "unusable boundary record"; "error" => %error); + return; + } + } + } + Verdict::Empty => None, + Verdict::Discard(_) => return, + }; + *self.boundary.lock().unwrap() = boundary; + self.untrusted.store(false, Ordering::SeqCst); + } + + pub fn untrusted(&self) -> bool { + self.untrusted.load(Ordering::SeqCst) + } + + pub fn boundary(&self) -> Option { + self.boundary.lock().unwrap().clone() + } + + /// Record how the boundary job ended. A stop displaces an adjudicated + /// `Interrupted`, mirroring the status arms in the state machine. + /// Nothing else is overwritten, and a record that has moved on to + /// a newer job ignores the old job's ending. + pub async fn record_outcome(&self, job_id: &JobId, outcome: &JobStatus) { + debug_assert!(outcome.is_terminal()); + if self.untrusted() { + return; + } + let mut guard = self.tenant.lock().await; + let updated = { + let recorded = self.boundary.lock().unwrap(); + let Some(boundary) = recorded.as_ref() else { + return; + }; + let Some(committed) = boundary.job.as_ref().filter(|c| c.job == *job_id) else { + return; + }; + let displaces = matches!( + (&committed.outcome, outcome), + (JobOutcome::Committed, _) + | ( + JobOutcome::Ended(JobStatus::Error { + error: ProcessError::Interrupted, + .. + }), + JobStatus::Stopped { .. }, + ) + ); + if !displaces { + return; + } + Boundary { + job: Some(Committed { + outcome: JobOutcome::Ended(outcome.clone()), + ..committed.clone() + }), + network: boundary.network, + burned: boundary.burned.clone(), + executed: boundary.executed.clone(), + } + }; + if let Err(error) = guard.store(&format::encode(&updated)).await { + warn!( + self.log, "failed to record the boundary job's outcome"; + "job_id" => %job_id, "error" => %error, + ); + return; + } + *self.boundary.lock().unwrap() = Some(updated); + } + + /// Commit to executing the job in `boundary`. On failure the + /// caller must not run the job. + pub async fn advance(&self, boundary: &Boundary) -> Result<(), BoundaryError> { + if self.untrusted() { + // Defense in depth: the state machine already refuses + // execution when the store is untrusted, so no launch + // reaches this arm. + return Err(BoundaryError::Untrusted); + } + let mut guard = self.tenant.lock().await; + guard.store(&format::encode(boundary)).await?; + *self.boundary.lock().unwrap() = Some(boundary.clone()); + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::fs::create_dir; + + use camino::Utf8PathBuf; + use slog::Discard; + use tempfile::TempDir; + + /// Two M.2 slots. + fn slots(dir: &TempDir) -> Vec { + ["m2a", "m2b"] + .iter() + .map(|m2| { + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot + }) + .collect() + } + + fn test_log() -> Logger { + Logger::root(Discard, o!()) + } + + async fn store(slots: Vec) -> BoundaryStore { + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + store.load().await; + store + } + + fn network(seed: u8) -> Network { + serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() + } + + fn boundary() -> Boundary { + Boundary { + network: network(1), + burned: Bloom::new(), + executed: "(1, 1, (0, 0, 2))".parse().unwrap(), + job: Some(Committed { + session: SessionId::random(), + job: JobId::random(), + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), + } + } + + fn job_of(boundary: &Boundary) -> JobId { + boundary.job.as_ref().expect("a committed job").job + } + + /// The record's bytes are on-disk format, frozen at version 0. If + /// this fails, STOP: do not re-pin. Copy the old shape into a + /// frozen module and add a new version instead; see + /// [`crate::format`]. + #[test] + fn pin_record_format_v0() { + let record = Boundary { + network: network(1), + burned: { + let mut burned = Bloom::new(); + burned.insert(&network_key(network(2))); + burned + }, + executed: "(1, 1, (0, 0, 2))".parse().unwrap(), + job: Some(Committed { + session: "abandon-ability".parse().unwrap(), + job: "zoo-zero".parse().unwrap(), + successor: "able-about".parse().unwrap(), + outcome: JobOutcome::Committed, + }), + }; + let bytes = format::encode(&record); + let path = "tests/output/boundary-record-v0.bin"; + if std::env::var("EXPECTORATE").as_deref() == Ok("overwrite") { + std::fs::write(path, &bytes).unwrap(); + } else { + let expected = std::fs::read(path).expect("missing snapshot"); + assert_eq!(bytes, expected, "record format changed: {bytes:02x?}"); + } + let decoded: Boundary = format::decode(&bytes).unwrap(); + assert_eq!(decoded.network, record.network); + assert!(decoded.is_burned(network(2))); + assert_eq!(decoded.executed, record.executed); + assert_eq!( + decoded.job.unwrap().successor, + record.job.unwrap().successor + ); + } + + /// A record from a newer software version loads as untrusted, and + /// the sled reports it instead of guessing at the format. + #[tokio::test] + async fn future_record_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + #[derive(Serialize)] + struct Envelope(u16, #[serde(with = "crate::format::cbor_bytes")] Vec); + let mut bytes = Vec::new(); + into_cbor(&Envelope(1, b"from the future".to_vec()), &mut bytes).unwrap(); + let scratch = Locker::new(&test_log(), slots.clone()); + scratch.tenant(BOUNDARY).store(&bytes).await.unwrap(); + + let store = store(slots).await; + assert!(store.untrusted()); + } + + /// The burned set's keys are on-disk format: a change to the + /// network's serde shape would silently forget every burn, and a + /// forgotten burn admits a flip-flop instead of refusing it. If + /// this fails, STOP, and see the warning on [`crate::bloom`]. + #[test] + fn pin_network_keys() { + assert_eq!( + network_key(network(1)), + [0x50, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + + #[tokio::test] + async fn commitments_survive_restarts() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let first = store(slots.clone()).await; + assert!(first.boundary().is_none()); + + let (a, mut b) = (boundary(), boundary()); + b.network = network(2); + b.burned = a.burned_for(b.network); + first.advance(&a).await.unwrap(); + first.advance(&b).await.unwrap(); + + let next = store(slots).await; + assert!(!next.untrusted()); + let recorded = next.boundary().unwrap(); + assert_eq!(recorded.network, b.network); + assert_eq!(job_of(&recorded), job_of(&b)); + assert!(recorded.is_burned(network(1))); + assert!(!recorded.is_burned(network(3))); + assert_eq!(recorded.executed, b.executed); + let (recorded, expected) = (recorded.job.unwrap(), b.job.unwrap()); + assert_eq!(recorded.session, expected.session); + assert_eq!(recorded.successor, expected.successor); + assert!(matches!(recorded.outcome, JobOutcome::Committed)); + } + + /// The boundary job's recorded ending survives into the next life. + /// A stop displaces an adjudicated interrupted; nothing else does, and + /// an ending for a superseded job is ignored. + #[tokio::test] + async fn outcomes_survive_and_heal() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let first = store(slots.clone()).await; + let b = boundary(); + let job = job_of(&b); + first.advance(&b).await.unwrap(); + + let interrupted = JobStatus::Error { + job_id: job, + time_error: chrono::Utc::now(), + error: ProcessError::Interrupted, + }; + let killed = JobStatus::Error { + job_id: job, + time_error: chrono::Utc::now(), + error: ProcessError::Killed(9), + }; + first.record_outcome(&JobId::random(), &killed).await; + assert!(matches!( + first.boundary().unwrap().job.unwrap().outcome, + JobOutcome::Committed + )); + + first.record_outcome(&job, &interrupted).await; + first.record_outcome(&job, &killed).await; + let next = store(slots).await; + assert!(matches!( + next.boundary().unwrap().job.unwrap().outcome, + JobOutcome::Ended(JobStatus::Error { + error: ProcessError::Interrupted, + .. + }) + )); + } + + #[tokio::test] + async fn disagreement_is_untrusted_and_pins() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + for slot in &slots { + let lone = store(vec![slot.clone()]).await; + lone.advance(&boundary()).await.unwrap(); + } + + let untrusted = store(slots.clone()).await; + assert!(untrusted.untrusted()); + assert!(untrusted.boundary().is_none()); + assert!(matches!( + untrusted.advance(&boundary()).await, + Err(BoundaryError::Untrusted) + )); + + let reload = store(slots).await; + assert!(reload.untrusted()); + } + + #[tokio::test] + async fn undecodable_record_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slots = slots(&dir); + let scratch = Locker::new(&test_log(), slots.clone()); + scratch.tenant(BOUNDARY).store(b"scribble").await.unwrap(); + + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots)); + store.load().await; + assert!(store.untrusted()); + } + + #[tokio::test] + async fn unloaded_is_untrusted() { + let dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let store = BoundaryStore::new(&test_log(), &Locker::new(&test_log(), slots(&dir))); + assert!(store.untrusted()); + assert!(matches!( + store.advance(&boundary()).await, + Err(BoundaryError::Untrusted) + )); + } +} diff --git a/server/src/executor.rs b/server/src/executor.rs index bae1c1a..1bf2950 100644 --- a/server/src/executor.rs +++ b/server/src/executor.rs @@ -7,7 +7,7 @@ //! Start, stop, and watch job processes. Driven by the session state //! machine, but session agnostic. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::io; use std::mem::MaybeUninit; use std::os::fd::AsRawFd as _; @@ -32,9 +32,11 @@ use tokio_util::sync::CancellationToken; use sush_api::JobStartParams; use sush_common::interactive::WindowSize; use sush_common::jobs::{ - JobId, JobMode, JobOutputStream, JobStartRequest, ProcessError, SignedJob, VerifiedJob, + JobId, JobMode, JobOutputStream, JobStartRequest, ProcessError, SignedJob, SkipReason, + VerifiedJob, }; +use crate::boundary::{Boundary, BoundaryStore}; use crate::io::JobIo; use crate::job::{Job, SocketSender}; use crate::messages::v0::{Event, JobEvent}; @@ -50,21 +52,42 @@ pub const DEFAULT_TERM: &str = "vt100"; /// experience backpressure; if we do, something is wrong. const EVENTS_CHANNEL_CAPACITY: usize = 16; +/// Each queued launch waits on one small fsync, so the queue drains in +/// milliseconds. A burst of concurrent jobs can still fill it, and a +/// full queue refuses the job with an error event rather than block +/// the state machine. +const LAUNCH_CHANNEL_CAPACITY: usize = 16; + pub struct Executor { log: Logger, events: Arc>>>, path_isolation: PathIsolation, output_dir: JobOutputDir, + launch: mpsc::Sender, shutdown: CancellationToken, stop: BTreeMap, } +/// One validated job on its way to the launcher. +struct Launch { + log: Logger, + boundary: Boundary, + events: mpsc::Sender, + output_dir: JobOutputDir, + request: VerifiedJob, + params: JobStartParams, + path_isolation: PathIsolation, + tx_attachment: watch::Sender>, + stop: CancellationToken, +} + /// Executor methods should be infallible; errors are reported via events. impl Executor { pub fn new( log: Logger, path_isolation: PathIsolation, output_dir: JobOutputDir, + boundary: Arc, shutdown: CancellationToken, ) -> (Self, impl Stream + Send + 'static) { let (tx_events, rx_events) = mpsc::channel(EVENTS_CHANNEL_CAPACITY); @@ -80,6 +103,31 @@ impl Executor { } }); + // The queue keeps boundary writes in release order. If each job + // advanced the boundary from its own task, two writes could land + // out of order, and the record would name an older job than one + // that already spawned. + let (launch, mut queued) = mpsc::channel::(LAUNCH_CHANNEL_CAPACITY); + spawn(async move { + while let Some(launch) = queued.recv().await { + if let Err(error) = boundary.advance(&launch.boundary).await { + let error = ProcessError::Io { + what: "recording the execution boundary".to_string(), + error: error.to_string(), + }; + send_error( + &launch.log, + launch.request.payload().job_id(), + &launch.events, + error, + ) + .await; + continue; + } + spawn(job_spawn(launch)); + } + }); + // Return the executor and event stream. ( Self { @@ -87,6 +135,7 @@ impl Executor { events, path_isolation, output_dir, + launch, shutdown, stop: BTreeMap::new(), }, @@ -94,16 +143,19 @@ impl Executor { ) } + /// Returns whether the job was queued for launch, so the caller + /// keeps attachment points only for jobs that can arrive. pub fn job_start( &mut self, certs: &mut Certificates, request: SignedJob, params: JobStartParams, tx_attachment: watch::Sender>, - ) { + boundary: Boundary, + ) -> bool { let Some(events) = self.events.read().unwrap().as_ref().cloned() else { // No more events ⇒ shutting down ⇒ no new jobs allowed. - return; + return false; }; // Validate the job request. @@ -116,7 +168,7 @@ impl Executor { spawn(async move { send_error(&log, &job_id, &events, $err).await; }); - return; + return false; }}; } if request.payload().command.starts_with('-') { @@ -134,16 +186,29 @@ impl Executor { let stop = self.shutdown.child_token(); self.stop.insert(job_id, stop.clone()); - spawn(job_spawn( - self.log.new(o!("job_id" => job_id)), + let refused = self.launch.try_send(Launch { + log: self.log.new(o!("job_id" => job_id)), + boundary, events, - self.output_dir.clone(), - verified_request, + output_dir: self.output_dir.clone(), + request: verified_request, params, - self.path_isolation, + path_isolation: self.path_isolation, tx_attachment, stop, - )); + }); + if let Err(error) = refused { + self.stop.remove(&job_id); + self.job_refused( + job_id, + ProcessError::Io { + what: "queueing the job for launch".to_string(), + error: error.to_string(), + }, + ); + return false; + } + true } pub fn job_stop(&mut self, job_id: &JobId) { @@ -156,6 +221,12 @@ impl Executor { let _ = self.stop.remove(job_id); } + /// Every job accepted for launch whose stop token is still held: + /// queued, spawning, or running. + pub fn in_flight(&self) -> BTreeSet { + self.stop.keys().copied().collect() + } + /// Announce a job that will never run here. pub fn job_refused(&self, job_id: JobId, error: ProcessError) { let Some(events) = self.events.read().unwrap().as_ref().cloned() else { @@ -167,26 +238,42 @@ impl Executor { }); } + /// Report that this sled will never run `job_id`; see + /// [`JobStatus::Skipped`](sush_common::jobs::JobStatus). + pub fn job_skipped(&self, job_id: JobId, reason: SkipReason) { + let Some(events) = self.events.read().unwrap().as_ref().cloned() else { + return; + }; + let log = self.log.clone(); + spawn(async move { + let event = Event::Job(JobEvent::Skipped(job_id, Utc::now(), reason)); + if let Err(error) = events.send(event).await { + warn!(log, "failed to send skip event"; "job_id" => %job_id, "error" => %error); + } + }); + } + pub fn output_dir(&self) -> &JobOutputDir { &self.output_dir } } /// Spawn a process for a job and return an attachment point if it is -/// interactive. Assumes the job request has already been validated, -/// e.g., as by [`crate::JobManager::job_start`]. -#[allow(clippy::too_many_arguments)] -async fn job_spawn( - log: Logger, - events: mpsc::Sender, - output_dir: JobOutputDir, - request: VerifiedJob, - params: JobStartParams, - path_isolation: PathIsolation, - tx_attachment: watch::Sender>, - stop: CancellationToken, -) { +/// interactive. Assumes [`Executor::job_start`] validated the request +/// and the launcher committed its boundary. +async fn job_spawn(launch: Launch) { use JobOutputStream::*; + let Launch { + log, + boundary: _, + events, + output_dir, + request, + params, + path_isolation, + tx_attachment, + stop, + } = launch; let JobStartRequest { job_id, session_id: _, diff --git a/server/src/format.rs b/server/src/format.rs new file mode 100644 index 0000000..a6b0e94 --- /dev/null +++ b/server/src/format.rs @@ -0,0 +1,393 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Versioned durable formats. +//! +//! Every version names its predecessor as its `Previous`. [`Record`] and +//! [`Wire`] each require a conversion from it; fallible for [`Record`], +//! infallible for [`Wire`]. A new version therefore does not compile +//! until the conversion from the old one exists, so by induction every +//! version ever shipped can be upgraded to the latest. +//! +//! A [`Locker`](crate::locker::Locker) tenant record is stored as a +//! two-element CBOR array, `[version, bytes]`, the byte string holding +//! the record's own CBOR encoding. The version lets a newer release +//! read every record an older release ever wrote: [`decode`] matches +//! the stored version against the chain of known formats, parses the +//! body as the format that matches, and converts the result up the +//! chain to the latest format. A gossip message instead carries its +//! version as the [`VersionedMessage`](crate::messages::VersionedMessage) +//! variant tag, and converts up when the state machine unwraps it. +//! +//! Tests must pin each version's serialized bytes. To change a format, +//! copy the live type into a frozen module under its version's name, +//! point the pin test at the copy, give the live type the next version, +//! name the frozen type as its `Previous`, and write the conversion the +//! compiler requires. + +use std::fmt::Display; + +use ciborium::{de::from_reader as from_cbor, ser::into_writer as into_cbor}; +use serde::{Serialize, de::DeserializeOwned}; +use thiserror::Error; + +/// A type that is one version of a durable format. Version numbers +/// must be unique within a chain and increase along it, so that any +/// version above the latest must belong to newer software. +pub trait Versioned: Sized { + const VERSION: u16; +} + +/// A [`Locker`](crate::locker::Locker) tenant's record format. +/// New versions will not compile without a conversion from their +/// predecessors. The conversion may fail, because the caller can +/// quarantine a record that will not convert; the boundary store, +/// for example, stays untrusted. +pub trait Record: Versioned + Serialize + DeserializeOwned { + /// The format this one supersedes. + type Previous: Record + TryInto; + + /// Parse `body` as the chain member whose version is `version`, + /// converting the result up to `Self`. This is not a public + /// interface; use [`decode`]. + fn walk(version: u16, body: &[u8]) -> Result { + const { + assert!( + Self::Previous::VERSION == NoFormat::VERSION + || Self::Previous::VERSION < Self::VERSION, + "chain versions must increase", + ) + }; + + if version == Self::VERSION { + return from_cbor(body).map_err(|error: ciborium::de::Error<_>| FormatError::Body { + version, + message: error.to_string(), + }); + } + + let previous = Self::Previous::walk(version, body)?; + previous.try_into().map_err(|error| FormatError::Convert { + from: Self::Previous::VERSION, + message: error.to_string(), + }) + } +} + +/// A wire format for messages, with the same upgrade guarantee +/// as [`Record`]. But here the conversion is infallible, because +/// message processing cannot in general skip a replayed message +/// or stop at it. +pub trait Wire: Versioned { + /// The format this one supersedes. + type Previous: Wire + Into; +} + +/// The end of every version chain. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub enum NoFormat {} + +impl Versioned for NoFormat { + // Reserved for the terminus. + const VERSION: u16 = u16::MAX; +} + +impl Record for NoFormat { + type Previous = NoFormat; + + fn walk(version: u16, _body: &[u8]) -> Result { + Err(FormatError::Unknown { version }) + } +} + +impl Wire for NoFormat { + type Previous = NoFormat; +} + +/// Why a record could not be read. +#[derive(Debug, Error)] +pub enum FormatError { + #[error("the record's version envelope did not parse")] + Envelope, + #[error("the record's format version {version} is newer than this software")] + Future { version: u16 }, + #[error("no format in this record's chain has version {version}")] + Unknown { version: u16 }, + #[error("the record did not parse as format version {version}: {message}")] + Body { version: u16, message: String }, + #[error("converting the record from format version {from} failed: {message}")] + Convert { from: u16, message: String }, +} + +/// The version envelope: `[version, body]`, with the body nested +/// as a CBOR byte string holding the encoded record. Nesting +/// keeps the body out of `ciborium::Value`, whose deserializer is +/// stricter than the byte-stream one and refuses serde adapters that +/// read CBOR byte strings; [`decode`] reads the version and hands +/// the untouched body bytes to the stream deserializer. +#[derive(serde::Serialize)] +struct Envelope(u16, #[serde(with = "cbor_bytes")] Vec); + +// Manual impl to refuse trailing elements. +impl<'de> serde::Deserialize<'de> for Envelope { + fn deserialize>(deserializer: D) -> Result { + use serde::de::{Error as _, IgnoredAny, SeqAccess, Visitor}; + + struct Body(Vec); + impl<'de> serde::Deserialize<'de> for Body { + fn deserialize>(deserializer: D) -> Result { + cbor_bytes::deserialize(deserializer).map(Body) + } + } + + struct EnvelopeVisitor; + impl<'de> Visitor<'de> for EnvelopeVisitor { + type Value = Envelope; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a two-element version envelope") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let version = seq + .next_element::()? + .ok_or_else(|| A::Error::custom("an envelope without a version"))?; + let Body(body) = seq + .next_element::()? + .ok_or_else(|| A::Error::custom("an envelope without a body"))?; + if seq.next_element::()?.is_some() { + return Err(A::Error::custom("trailing elements in the envelope")); + } + Ok(Envelope(version, body)) + } + } + + deserializer.deserialize_seq(EnvelopeVisitor) + } +} + +/// Serialize a byte buffer as a CBOR byte string rather than an +/// array of integers. +pub(crate) mod cbor_bytes { + use serde::de::Visitor; + use serde::{Deserializer, Serializer}; + use std::fmt; + + pub fn serialize(bytes: &[u8], serializer: S) -> Result { + serializer.serialize_bytes(bytes) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + struct Bytes; + impl<'de> Visitor<'de> for Bytes { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("bytes") + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(v.to_vec()) + } + } + deserializer.deserialize_bytes(Bytes) + } +} + +/// Encode `value` at the latest version, inside the version envelope. +pub fn encode(value: &T) -> Vec { + let mut body = Vec::new(); + into_cbor(value, &mut body).expect("writing to a Vec cannot fail"); + let mut bytes = Vec::new(); + into_cbor(&Envelope(T::VERSION, body), &mut bytes).expect("writing to a Vec cannot fail"); + bytes +} + +/// Decode a record at whatever version it was stored, converting up +/// the chain to `T`. +pub fn decode(bytes: &[u8]) -> Result { + let Envelope(version, body) = from_cbor(bytes).map_err(|_| FormatError::Envelope)?; + if version > T::VERSION { + return Err(FormatError::Future { version }); + } + T::walk(version, &body) +} + +#[cfg(test)] +mod test { + use super::*; + + use serde::Deserialize; + + #[derive(Debug, Deserialize, Serialize)] + struct TestV0 { + count: u8, + } + + #[derive(Debug, Deserialize, Serialize, PartialEq)] + struct TestV1 { + count: u32, + label: String, + } + + impl Versioned for TestV0 { + const VERSION: u16 = 0; + } + impl Record for TestV0 { + type Previous = NoFormat; + } + + // The gap at version 1 is deliberate: unknown_version_is_refused + // walks the whole chain without matching it. + impl Versioned for TestV1 { + const VERSION: u16 = 2; + } + impl Record for TestV1 { + type Previous = TestV0; + } + + impl TryFrom for TestV0 { + type Error = &'static str; + fn try_from(none: NoFormat) -> Result { + match none {} + } + } + + impl TryFrom for TestV1 { + type Error = &'static str; + fn try_from(old: TestV0) -> Result { + if old.count == u8::MAX { + return Err("saturated count"); + } + Ok(TestV1 { + count: old.count.into(), + label: String::new(), + }) + } + } + + #[test] + fn trailing_envelope_elements_are_refused() { + #[derive(Serialize)] + struct Extended(u16, #[serde(with = "cbor_bytes")] Vec, u16); + let mut body = Vec::new(); + into_cbor(&TestV0 { count: 7 }, &mut body).unwrap(); + let mut bytes = Vec::new(); + into_cbor(&Extended(0, body, 7), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Envelope) + )); + } + + #[test] + fn round_trip_at_latest() { + let value = TestV1 { + count: 7, + label: "seven".to_string(), + }; + let decoded: TestV1 = decode(&encode(&value)).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn old_version_converts_up() { + let old = encode(&TestV0 { count: 7 }); + let new: TestV1 = decode(&old).unwrap(); + assert_eq!( + new, + TestV1 { + count: 7, + label: String::new(), + } + ); + } + + #[test] + fn future_version_is_refused() { + let futuristic = encode(&TestV1 { + count: 1, + label: String::new(), + }); + assert!(matches!( + decode::(&futuristic), + Err(FormatError::Future { version: 2 }) + )); + } + + #[test] + fn unknown_version_is_refused() { + let mut bytes = Vec::new(); + into_cbor(&Envelope(1, Vec::new()), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Unknown { version: 1 }) + )); + } + + #[test] + fn failed_conversion_is_reported() { + let saturated = encode(&TestV0 { count: u8::MAX }); + assert!(matches!( + decode::(&saturated), + Err(FormatError::Convert { from: 0, .. }) + )); + } + + #[test] + fn garbage_is_refused() { + assert!(matches!( + decode::(b"scribble"), + Err(FormatError::Envelope) + )); + } + + #[derive(Debug, PartialEq)] + struct WireV0(u8); + #[derive(Debug, PartialEq)] + struct WireV1(u32); + + impl Versioned for WireV0 { + const VERSION: u16 = 0; + } + impl Wire for WireV0 { + type Previous = NoFormat; + } + + impl Versioned for WireV1 { + const VERSION: u16 = 1; + } + impl Wire for WireV1 { + type Previous = WireV0; + } + + impl From for WireV0 { + fn from(none: NoFormat) -> Self { + match none {} + } + } + + impl From for WireV1 { + fn from(old: WireV0) -> Self { + WireV1(old.0.into()) + } + } + + /// A wire chain converts infallibly; the bound will not accept a + /// fallible conversion. + #[test] + fn wire_chain_converts() { + assert_eq!(WireV1::from(WireV0(7)), WireV1(7)); + } + + #[test] + fn wrong_body_is_reported() { + let mut bytes = Vec::new(); + let mut body = Vec::new(); + into_cbor(&"not a struct", &mut body).unwrap(); + into_cbor(&Envelope(0, body), &mut bytes).unwrap(); + assert!(matches!( + decode::(&bytes), + Err(FormatError::Body { version: 0, .. }) + )); + } +} diff --git a/server/src/gossip.rs b/server/src/gossip.rs index bc64d4a..e08f02b 100644 --- a/server/src/gossip.rs +++ b/server/src/gossip.rs @@ -29,7 +29,7 @@ use std::net::{SocketAddr, SocketAddrV6}; use std::time::Duration; use futures::StreamExt as _; -use rumors::{Error, Joined, Network, Peer, Rumors, Ticks, Version}; +use rumors::{Error, Joined, Network, Peer, Rumors, Ticks}; use serde::Serialize; use serde::de::DeserializeOwned; use sled_hardware_types::BaseboardId; @@ -45,6 +45,7 @@ use rumors::link::routed::Endpoint; use crate::bookmark::{BookmarkSource, SushBookmark}; use crate::link::{AttestedBaseboards, CorpusSource, SprocketsDial, SprocketsLink, Transport}; +use crate::locker::Locker; /// The attested baseboards of our live gossip peers. pub type LinkedBaseboards = watch::Receiver>; @@ -71,22 +72,67 @@ impl Default for GossipConfig { } } -/// A gossip universe and where we entered it. +/// A gossip universe. #[derive(Clone, Debug)] pub struct Universe { /// The gossiped set. pub rumors: Rumors, - /// The causal frontier of the set received when we joined, - /// or `None` if we seeded the universe ourselves. - pub frontier: Option, } impl Universe { pub fn genesis(rumors: Rumors) -> Self { - Self { - rumors, - frontier: None, - } + Self { rumors } + } +} + +/// A seeded network paired with the source persisting its identity. +/// [`Seed::grow`] is the only constructor and [`spawn_gossip`] consumes +/// the pair whole, so a seed can never gossip against a source other +/// than its own. +#[derive(Debug)] +pub struct Seed { + rumors: Rumors, + bookmarks: BookmarkSource, +} + +impl Seed { + /// Seed a fresh universe with this server as its only peer, over + /// `locker`'s storage, making the locker's one [`BookmarkSource`]. + /// + /// The probe runs first because broken storage would otherwise + /// wedge gossip: rumors stores the bookmark at the start of every + /// session, a failed store aborts the session, and a peer that can + /// never hold a session can never join another universe. When the + /// probe fails we gossip with a shed handle instead, which + /// persists nothing; each restart then strands an identity, which + /// is harmless. + pub async fn grow(log: &Logger, locker: &Locker) -> Self + where + T: DeserializeOwned + Serialize + Send + Sync + 'static, + { + let bookmarks = BookmarkSource::new(log, locker); + let handle = match locker.probe().await { + Ok(()) => bookmarks.handle(), + Err(_) => bookmarks.shed_handle(), + }; + let rumors = match Peer::seed().bookmark(handle).await { + Ok(peer) => peer.into_rumors(), + Err(unbookmarked) => match unbookmarked.peer.bookmark(bookmarks.shed_handle()).await { + Ok(peer) => peer.into_rumors(), + Err(_) => unreachable!("a shed bookmark never touches storage"), + }, + }; + Self { rumors, bookmarks } + } + + pub fn rumors(&self) -> &Rumors { + &self.rumors + } + + /// The network alone, for a seed that will never gossip + /// (see [`isolated`]). + pub fn into_rumors(self) -> Rumors { + self.rumors } } @@ -132,8 +178,7 @@ pub async fn spawn_gossip( corpus: CorpusSource, listen_addr: SocketAddrV6, peers: watch::Receiver>, - seed: Rumors, - bookmarks: BookmarkSource, + seed: Seed, shutdown: CancellationToken, ) -> io::Result<(SocketAddrV6, watch::Receiver>, LinkedBaseboards)> where @@ -149,8 +194,7 @@ where ) .await?; let bound = transport.bound(); - let (universe, linked) = - spawn_gossip_manager(log, config, transport, peers, seed, bookmarks, shutdown); + let (universe, linked) = spawn_gossip_manager(log, config, transport, peers, seed, shutdown); Ok((bound, universe, linked)) } @@ -164,13 +208,16 @@ pub fn spawn_gossip_manager( config: GossipConfig, transport: Transport, peers: watch::Receiver>, - seed: Rumors, - bookmarks: BookmarkSource, + seed: Seed, shutdown: CancellationToken, ) -> (watch::Receiver>, LinkedBaseboards) where T: DeserializeOwned + Serialize + Send + Sync + 'static, { + let Seed { + rumors: seed, + bookmarks, + } = seed; let (publish, subscribe) = watch::channel(Universe::genesis(seed.clone())); let (linked, subscribe_linked) = watch::channel(BTreeSet::new()); let manager = Manager { @@ -401,11 +448,17 @@ where /// the next link retries; either way all links are rebuilt, since the /// old ones belong to the universe we are leaving. /// - /// The new peer gets a fresh bookmark handle, fencing off all the - /// abandoned universe's stores. If the received identity cannot be - /// persisted, we keep gossiping with a shed handle rather than take - /// the sled out of gossip; a stranded identity is harmless, unlike - /// a support shell that cannot reach a degraded rack. + /// The new peer gets its own handle on the same bookmark storage. + /// That is safe because rumors persists a bookmark only when a + /// session starts, and aborting the drivers above ends every + /// session before the handle exists: the abandoned peer can never + /// store again. A store it already had in flight either loses to + /// the locker's sequence guard, or records a session that was + /// aborted before it sent anything, so nothing on the wire + /// outruns the record. If the received identity cannot be + /// persisted, we keep gossiping with a shed handle rather than + /// take the sled out of gossip; a stranded identity is harmless, + /// unlike a support shell that cannot reach a degraded rack. async fn migrate(&mut self, peer: SocketAddr, mut link: SprocketsLink) { self.drivers.abort_all(); self.live.clear(); @@ -413,7 +466,7 @@ where self.log, "joining the universe that beat ours"; "peer" => %peer, "ours" => %self.rumors.network(), ); - let bootstrap = Peer::bootstrap().bookmark(self.bookmarks.next_handle()); + let bootstrap = Peer::bootstrap().bookmark(self.bookmarks.handle()); match timeout(self.config.join_timeout, bootstrap.join(&mut link)).await { Ok(Joined::Joined { peer }) => self.adopt(peer), Ok(Joined::Unbookmarked(unbookmarked)) => { @@ -442,10 +495,8 @@ where /// Follow the joined peer into its universe. fn adopt(&mut self, peer: Peer) { self.rumors = peer.into_rumors(); - let frontier = self.rumors.snapshot().latest().clone(); let _ = self.publish.send(Universe { rumors: self.rumors.clone(), - frontier: Some(frontier), }); self.joins.clear(); info!(self.log, "migrated"; "network" => %self.rumors.network()); diff --git a/server/src/lib.rs b/server/src/lib.rs index 2675827..8f59f09 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -10,14 +10,18 @@ extern crate function_name; #[cfg(all(feature = "embedded", feature = "test-support"))] compile_error!("`test-support` must not be enabled for an embedded server"); +pub mod bloom; pub mod bookmark; +pub mod boundary; pub mod error; pub mod executor; +pub mod format; pub mod gossip; pub mod history; pub mod io; pub mod job; pub mod link; +pub mod locker; pub mod manager; pub mod messages; pub mod mux; diff --git a/server/src/locker.rs b/server/src/locker.rs new file mode 100644 index 0000000..c63dab7 --- /dev/null +++ b/server/src/locker.rs @@ -0,0 +1,628 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Durable per-sled state that is never stale. +//! +//! A [`Locker`] spans the boot M.2s. Each M.2 contributes one *slot* +//! directory, and each [`Tenant`] owns one file in every slot. A store +//! writes every slot and returns `Ok` only when all of them hold the +//! new record. A load applies the pair rules: +//! +//! | slot A | slot B | verdict | +//! |---------------|-------------------------|--------------------------| +//! | record R | record R | adopt R | +//! | record R | missing | restore R (an M.2 swap) | +//! | missing | missing | fresh | +//! | R, nonce N, n | S ≠ R, nonce N, seq > n | adopt S (a torn write) | +//! | record R | record S ≠ R | discard | +//! | corrupt | anything | discard | +//! | I/O error | anything | discard | +//! +//! Every write carries a nonce drawn once per [`Locker`]. Slots that +//! disagree under one nonce were torn by a single process, so the +//! higher sequence number is that process's newest write, and a +//! newer-than-acknowledged record is safe to adopt: a store that +//! never returned had no effects. +//! +//! Discarding is a verdict, not an error, since each tenant decides +//! what starting over means. +//! +//! All of this is necessary to ensure the basic constraint that +//! **we must never adopt stale data**. + +use std::collections::BTreeSet; +use std::fs::Permissions; +use std::io::{self, Write as _}; +use std::os::unix::fs::PermissionsExt as _; +use std::sync::{Arc, Mutex as SyncMutex}; + +use atomicwrites::{AtomicFile, OverwriteBehavior}; +use camino::Utf8PathBuf; +use futures::TryFutureExt as _; +use slog::{Logger, o, warn}; +use thiserror::Error; +use tokio::fs::{read, remove_file, write}; +use tokio::sync::{Mutex, MutexGuard}; +use tokio::task::spawn_blocking; + +use sush_common::authn::Nonce; +use sush_common::hash::Hasher; + +const MAGIC_LEN: usize = 12; +const NONCE_LEN: usize = 32; +const SEQ_LEN: usize = 8; +const DIGEST_LEN: usize = 32; + +fn digest(nonce: &[u8; NONCE_LEN], seq: u64, record: &[u8]) -> [u8; DIGEST_LEN] { + let mut hasher = Hasher::new(); + hasher.update(nonce); + hasher.update(&seq.to_be_bytes()); + hasher.update(record); + *hasher.finalize().as_bytes() +} + +/// A file name and an envelope magic. +#[derive(Clone, Copy, Debug)] +pub struct TenantSpec { + pub file: &'static str, + pub magic: &'static [u8; MAGIC_LEN], +} + +/// What a load found across the slots. +#[derive(Debug)] +pub enum Verdict { + /// Every slot holds this record. + Adopt(Vec), + /// A slot is missing, but the survivors agree on this record. + Restore(Vec), + /// No slot holds a record. + Empty, + /// The slots cannot be trusted. + Discard(Discard), +} + +#[derive(Debug, Error)] +pub enum Discard { + #[error("the slots disagree")] + Disagree, + #[error("`{path}` is corrupt")] + Corrupt { path: Utf8PathBuf }, + #[error("reading `{path}` failed: {error}")] + Io { + path: Utf8PathBuf, + #[source] + error: io::Error, + }, +} + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("storing `{path}` failed: {error}")] + Io { + path: Utf8PathBuf, + #[source] + error: io::Error, + }, + #[error("superseded by a newer record")] + Superseded, + #[error("the store task died: {0}")] + Task(String), +} + +/// This sled's durable state storage. +#[derive(Clone, Debug)] +pub struct Locker { + log: Logger, + slots: Arc>, + /// Stamped on every envelope this process writes; see the torn + /// write rule in the module doc. + nonce: Nonce, + /// Files already claimed by a tenant. A second tenant over one + /// file would have its own sequence state, silently defeating + /// the straggler guard. + claimed: Arc>>, +} + +impl Locker { + /// A locker spans `slots`. Empty means nothing persists + /// (the standalone server, tests). + pub fn new(log: &Logger, slots: Vec) -> Self { + Self { + log: log.new(o!("component" => "locker")), + slots: Arc::new(slots), + nonce: Nonce::random(), + claimed: Arc::new(SyncMutex::new(BTreeSet::new())), + } + } + + /// A locker that loads and persists nothing. + pub fn null() -> Self { + Self::new(&Logger::root(slog::Discard, o!()), Vec::new()) + } + + /// A tenant of this locker, described by a (constant) specification. + /// Each file supports one tenant; a duplicate claim panics. + pub fn tenant(&self, spec: TenantSpec) -> Tenant { + assert!( + self.claimed.lock().unwrap().insert(spec.file), + "tenant `{}` is already claimed", + spec.file, + ); + Tenant { + log: self.log.new(o!("tenant" => spec.file)), + spec, + paths: self.slots.iter().map(|slot| slot.join(spec.file)).collect(), + nonce: self.nonce.clone(), + reserved: Mutex::new(0), + committed: Arc::new(SyncMutex::new(0)), + } + } + + /// Prove that every slot is writable by writing to every slot. + pub async fn probe(&self) -> Result<(), StoreError> { + for slot in self.slots.iter() { + let probe = slot.join("probe"); + if let Err(error) = write(&probe, b"").and_then(|()| remove_file(&probe)).await { + warn!(self.log, "unusable slot"; "path" => %probe); + return Err(StoreError::Io { path: probe, error }); + } + } + Ok(()) + } +} + +/// One tenant's files across the slots, with a two-stage +/// reserve/commit sequence number. +#[derive(Debug)] +pub struct Tenant { + log: Logger, + spec: TenantSpec, + paths: Vec, + nonce: Nonce, + /// The newest sequence number reserved or observed by this process. + reserved: Mutex, + /// The newest sequence number written to all slots. + committed: Arc>, +} + +impl Tenant { + /// Serialize loads and stores. Admission checks belong under the + /// guard. + pub async fn lock(&self) -> Guard<'_> { + Guard { + tenant: self, + reserved: self.reserved.lock().await, + } + } + + pub async fn load(&self) -> Verdict { + self.lock().await.load().await + } + + pub async fn store(&self, record: &[u8]) -> Result<(), StoreError> { + self.lock().await.store(record).await + } + + fn parse(&self, bytes: &[u8]) -> Option<(Nonce, u64, Vec)> { + let payload = bytes.strip_prefix(self.spec.magic.as_slice())?; + let (nonce, rest) = payload.split_first_chunk::()?; + let (seq, rest) = rest.split_first_chunk::()?; + let (sum, record) = rest.split_first_chunk::()?; + let seq = u64::from_be_bytes(*seq); + (digest(nonce, seq, record) == *sum) + .then(|| (Nonce::from_be_bytes(*nonce), seq, record.to_vec())) + } + + fn envelope(&self, seq: u64, record: &[u8]) -> Vec { + let mut bytes = + Vec::with_capacity(MAGIC_LEN + NONCE_LEN + SEQ_LEN + DIGEST_LEN + record.len()); + bytes.extend_from_slice(self.spec.magic); + let nonce = self.nonce.to_be_bytes(); + bytes.extend_from_slice(&nonce); + bytes.extend_from_slice(&seq.to_be_bytes()); + bytes.extend_from_slice(&digest(&nonce, seq, record)); + bytes.extend_from_slice(record); + bytes + } +} + +pub struct Guard<'a> { + tenant: &'a Tenant, + reserved: MutexGuard<'a, u64>, +} + +impl Guard<'_> { + pub async fn load(&mut self) -> Verdict { + let tenant = self.tenant; + if tenant.paths.is_empty() { + return Verdict::Empty; + } + + let mut found: Vec<(Nonce, u64, Vec)> = Vec::new(); + for path in &tenant.paths { + let bytes = match read(path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => { + return self.discard(Discard::Io { + path: path.clone(), + error, + }); + } + }; + match tenant.parse(&bytes) { + Some(parsed) => found.push(parsed), + None => return self.discard(Discard::Corrupt { path: path.clone() }), + } + } + if found.is_empty() { + return Verdict::Empty; + } + let restored = found.len() < tenant.paths.len(); + if !found.iter().all(|(_, _, record)| *record == found[0].2) { + // Slots torn by one process resolve to its newest write; + // see the module doc. Anything else is a genuine + // disagreement. + let (nonce, seq, record) = found + .iter() + .max_by_key(|(_, seq, _)| *seq) + .cloned() + .expect("found is non-empty"); + let torn = found.iter().all(|(n, ..)| *n == nonce) + && found.iter().filter(|(_, s, _)| *s == seq).count() == 1; + if !torn { + return self.discard(Discard::Disagree); + } + warn!(tenant.log, "adopted the newest write of a torn pair"; "seq" => seq); + if seq > *self.reserved { + *self.reserved = seq; + } + if let Err(error) = self.store(&record).await { + warn!(tenant.log, "failed to repair the torn pair"; "error" => %error); + } + return Verdict::Adopt(record); + } + + let (_, seq, record) = found.swap_remove(0); + let newest = found.iter().fold(seq, |max, (_, seq, _)| max.max(*seq)); + + // A reserved sequence number outranks a re-read of the disk. + // Regressing would let a cancelled write's straggler collide + // with a fresh reservation. + if newest > *self.reserved { + *self.reserved = newest; + } + if restored { + warn!(tenant.log, "restored from a lone slot"); + // Repair now: the survivor must not stay lone until the + // next natural store. + if let Err(error) = self.store(&record).await { + warn!(tenant.log, "failed to repair the lone slot"; "error" => %error); + } + Verdict::Restore(record) + } else { + Verdict::Adopt(record) + } + } + + pub async fn store(&mut self, record: &[u8]) -> Result<(), StoreError> { + let tenant = self.tenant; + if tenant.paths.is_empty() { + return Ok(()); + } + + *self.reserved += 1; + let seq = *self.reserved; + let envelope = tenant.envelope(seq, record); + let paths = tenant.paths.clone(); + let committed = tenant.committed.clone(); + spawn_blocking(move || commit(&committed, &paths, seq, &envelope)) + .await + .map_err(|join| StoreError::Task(join.to_string()))? + } + + fn discard(&self, reason: Discard) -> Verdict { + warn!(self.tenant.log, "discarding stored state"; "reason" => %reason); + Verdict::Discard(reason) + } +} + +/// Rename `envelope` into every slot iff `seq` is newer than the +/// latest committed version. A cancelled store drops only the async +/// side of the write; the blocking side is already detached, runs +/// to completion regardless, and must not overwrite a newer record. +fn commit( + committed: &SyncMutex, + paths: &[Utf8PathBuf], + seq: u64, + envelope: &[u8], +) -> Result<(), StoreError> { + let mut committed = committed.lock().unwrap(); + if seq <= *committed { + return Err(StoreError::Superseded); + } + for path in paths { + AtomicFile::new(path, OverwriteBehavior::AllowOverwrite) + .write(|file| { + file.set_permissions(Permissions::from_mode(0o600))?; + file.write_all(envelope) + }) + .map_err(|error| StoreError::Io { + path: path.clone(), + error: match error { + atomicwrites::Error::Internal(error) | atomicwrites::Error::User(error) => { + error + } + }, + })?; + } + *committed = seq; + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + use std::fs::{create_dir, metadata, read, remove_file as remove, write}; + + use tempfile::TempDir; + + const SPEC: TenantSpec = TenantSpec { + file: "record", + magic: b"SUSHLOCKTEST", + }; + + /// Two slot directories, like two M.2s. + fn slots(dir: &TempDir) -> Vec { + ["m2a", "m2b"] + .iter() + .map(|m2| { + let slot = Utf8PathBuf::from_path_buf(dir.path().join(m2)).unwrap(); + create_dir(&slot).unwrap(); + slot + }) + .collect() + } + + fn test_log() -> Logger { + Logger::root(slog::Discard, o!()) + } + + fn locker(slots: Vec) -> Locker { + Locker::new(&test_log(), slots) + } + + fn files(slots: &[Utf8PathBuf]) -> Vec { + slots.iter().map(|slot| slot.join(SPEC.file)).collect() + } + + /// A store lands the same envelope in every slot, sequenced and + /// private, and loads back adopted. + #[tokio::test] + async fn round_trip_writes_every_slot() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + + assert!(matches!(tenant.load().await, Verdict::Empty)); + tenant.store(b"who we are").await.unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"who we are" + )); + + let expected = tenant.envelope(1, b"who we are"); + for path in files(&slots) { + assert_eq!(read(&path).unwrap(), expected); + let mode = metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + } + + /// A missing slot restores from the survivor, and the next store + /// repairs it. + #[tokio::test] + async fn lone_slot_restores_and_repairs() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"kept").await.unwrap(); + + let files = files(&slots); + remove(&files[0]).unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Restore(record) if record == b"kept" + )); + + tenant.store(b"repaired").await.unwrap(); + assert_eq!(read(&files[0]).unwrap(), read(&files[1]).unwrap()); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"repaired" + )); + } + + /// Disagreeing slots are discarded, not arbitrated by sequence + /// number. + #[tokio::test] + async fn disagreement_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let a = locker(vec![slots[0].clone()]).tenant(SPEC); + a.store(b"one world").await.unwrap(); + let b = locker(vec![slots[1].clone()]).tenant(SPEC); + b.store(b"junk").await.unwrap(); + b.store(b"another").await.unwrap(); + + let tenant = locker(slots).tenant(SPEC); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Disagree) + )); + } + + /// Equal records adopt even when their sequence numbers differ, + /// and new stores outnumber the highest. + #[tokio::test] + async fn legacy_sequence_skew_is_benign() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let a = locker(vec![slots[0].clone()]).tenant(SPEC); + a.store(b"same").await.unwrap(); + let b = locker(vec![slots[1].clone()]).tenant(SPEC); + b.store(b"junk").await.unwrap(); + b.store(b"same").await.unwrap(); + + let tenant = locker(slots.clone()).tenant(SPEC); + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"same" + )); + tenant.store(b"next").await.unwrap(); + let expected = tenant.envelope(3, b"next"); + for path in files(&slots) { + assert_eq!(read(&path).unwrap(), expected); + } + } + + /// A corrupt slot is discarded even beside a valid one. + #[tokio::test] + async fn corruption_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"good").await.unwrap(); + + write(slots[0].join(SPEC.file), b"scribble").unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Corrupt { .. }) + )); + } + + /// A damaged nonce or sequence number fails the digest rather + /// than parsing. + #[tokio::test] + async fn flipped_envelope_bytes_are_corruption() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + let path = slots[0].join(SPEC.file); + for offset in [MAGIC_LEN, MAGIC_LEN + NONCE_LEN] { + tenant.store(b"good").await.unwrap(); + let mut bytes = read(&path).unwrap(); + bytes[offset] ^= 0x80; + write(&path, bytes).unwrap(); + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Corrupt { .. }) + )); + } + } + + /// Slots torn by one process resolve to its newest write, both + /// in that process and in the next. + #[tokio::test] + async fn torn_pair_resolves_to_newest_write() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + tenant.store(b"old").await.unwrap(); + write(slots[0].join(SPEC.file), tenant.envelope(2, b"new")).unwrap(); + + assert!(matches!( + tenant.load().await, + Verdict::Adopt(record) if record == b"new" + )); + + let next = locker(slots.clone()).tenant(SPEC); + assert!(matches!( + next.load().await, + Verdict::Adopt(record) if record == b"new" + )); + next.store(b"repaired").await.unwrap(); + assert_eq!( + read(slots[0].join(SPEC.file)).unwrap(), + read(slots[1].join(SPEC.file)).unwrap(), + ); + } + + /// Equal sequence numbers cannot be arbitrated, even in one life. + #[tokio::test] + async fn torn_pair_with_equal_sequence_numbers_discards() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + write(slots[0].join(SPEC.file), tenant.envelope(2, b"x")).unwrap(); + write(slots[1].join(SPEC.file), tenant.envelope(2, b"y")).unwrap(); + + assert!(matches!( + tenant.load().await, + Verdict::Discard(Discard::Disagree) + )); + } + + /// A store failing partway errs, and the survivor still loads: + /// the record was never acknowledged, so either version is sound. + #[tokio::test] + async fn partial_store_fails_loudly() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let good = slots(&dir).swap_remove(0); + let gone = Utf8PathBuf::from_path_buf(dir.path().join("gone")).unwrap(); + let tenant = locker(vec![good.clone(), gone]).tenant(SPEC); + + assert!(matches!( + tenant.store(b"half").await, + Err(StoreError::Io { .. }) + )); + assert!(matches!( + tenant.load().await, + Verdict::Restore(record) if record == b"half" + )); + } + + /// A straggling write from a dropped store future cannot land on + /// top of a newer committed record. + #[tokio::test] + async fn stragglers_cannot_clobber_newer_commits() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let slots = slots(&dir); + let tenant = locker(slots.clone()).tenant(SPEC); + let paths = files(&slots); + + let newer = tenant.envelope(7, b"newer"); + let straggler = tenant.envelope(6, b"stale"); + commit(&tenant.committed, &paths, 7, &newer).unwrap(); + assert!(matches!( + commit(&tenant.committed, &paths, 6, &straggler), + Err(StoreError::Superseded) + )); + assert_eq!(read(&paths[0]).unwrap(), newer); + } + + /// Probing proves every slot writable by writing. + #[tokio::test] + async fn probe_requires_every_slot() { + let dir = TempDir::with_prefix("sush-locker-").unwrap(); + let mut slots = slots(&dir); + locker(slots.clone()).probe().await.unwrap(); + + slots.push(Utf8PathBuf::from("/nonexistent/sush")); + assert!(matches!( + locker(slots).probe().await, + Err(StoreError::Io { .. }) + )); + } + + /// A null locker persists nothing and never fails. + #[tokio::test] + async fn null_touches_nothing() { + let tenant = Locker::null().tenant(SPEC); + tenant.store(b"lost").await.unwrap(); + assert!(matches!(tenant.load().await, Verdict::Empty)); + Locker::null().probe().await.unwrap(); + } +} diff --git a/server/src/main.rs b/server/src/main.rs index b76c43d..4528dfa 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -20,9 +20,9 @@ use x509_cert::der::DecodePem as _; use sush_api::sush_api_mod::api_description; use sush_common::targets::Cubbies; -use sush_server::bookmark::BookmarkSource; use sush_server::executor::PathIsolation; use sush_server::gossip::{isolated, lonely}; +use sush_server::locker::Locker; use sush_server::manager::JobManager; use sush_server::output::JobOutputDir; use sush_server::server::ApiServer; @@ -95,7 +95,7 @@ async fn main() -> Result<(), String> { }; // TODO: get/seed Rumors network - let gossip = isolated(seed_gossip(&BookmarkSource::null()).await); + let gossip = isolated(seed_gossip(&log, &Locker::null()).await.into_rumors()); #[cfg(feature = "test-support")] let roots = overridable_root_certs(&override_root_certs).await?; @@ -112,6 +112,7 @@ async fn main() -> Result<(), String> { cubbies, gossip, lonely(), + &Locker::null(), &roots, shutdown.clone(), ) diff --git a/server/src/manager.rs b/server/src/manager.rs index ccde5e8..989f6f0 100644 --- a/server/src/manager.rs +++ b/server/src/manager.rs @@ -23,7 +23,7 @@ use tokio::time::timeout; use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::CancellationToken; use x509_cert::Certificate; -use x509_cert::der::DecodePem as _; +use x509_cert::der::{DecodePem as _, Encode as _}; use sush_api::{JobStartParams, JobStopParams, JobWait}; use sush_common::authn::{ @@ -38,10 +38,12 @@ use sush_common::keys::{KeyError, KeyId, SshPublicKey}; use sush_common::targets::{Cubbies, SledHealth, SledVersion}; use sush_common::version::LONG_VERSION; +use crate::boundary::BoundaryStore; use crate::error::JobError; use crate::executor::PathIsolation; use crate::gossip::LinkedBaseboards; use crate::job::SocketSender; +use crate::locker::Locker; use crate::messages::v0::{CertRequest, IdentityRequest, JobRequest, Request, SessionRequest}; use crate::output::{JobOutputDir, JobOutputFileStream}; use crate::state::{GossipUniverse, MAX_CERTS, State, StateManager}; @@ -107,6 +109,7 @@ impl JobManager { cubbies: watch::Receiver, universe: watch::Receiver, linked: LinkedBaseboards, + locker: &Locker, roots: &[impl AsRef], shutdown: CancellationToken, ) -> Result { @@ -119,6 +122,7 @@ impl JobManager { cubbies, universe, linked, + locker, &roots, shutdown, ) @@ -134,6 +138,7 @@ impl JobManager { cubbies: watch::Receiver, universe: watch::Receiver, linked: LinkedBaseboards, + locker: &Locker, roots: &[Certificate], shutdown: CancellationToken, ) -> Result { @@ -141,6 +146,8 @@ impl JobManager { let (tx_req, rx_req) = mpsc::channel(16); let requests = ReceiverStream::new(rx_req); let session_sush_nonce = Arc::new(SyncMutex::new(SessionSushNonce::random())); + let boundary = Arc::new(BoundaryStore::new(&log, locker)); + boundary.load().await; let (rx_state, join_state) = StateManager::run( log.new(o!("component" => "state manager")), path_isolation, @@ -151,6 +158,7 @@ impl JobManager { universe, roots, session_sush_nonce.clone(), + boundary, shutdown, )?; Ok(Self { @@ -250,7 +258,8 @@ impl JobManager { return Err(KeyError::SelfSigned.into()); } let key_id = KeyId::try_from(&cert)?; - self.cert_request(authn, CertRequest::Import(cert)).await?; + let der = cert.to_der().map_err(KeyError::from)?; + self.cert_request(authn, CertRequest::Import(der)).await?; if wait { self.wait_for(self.wait_for_cert(key_id)).await?; } @@ -413,7 +422,7 @@ impl JobManager { last_used: Instant::now(), }, ); - let login = IdentityRequest::Login(public_key, response); + let login = IdentityRequest::Login(public_key.to_openssh()?, response); self.identity_request(key_id, login) .await .map(|()| identity) diff --git a/server/src/messages.rs b/server/src/messages.rs index e7c732e..b6885ee 100644 --- a/server/src/messages.rs +++ b/server/src/messages.rs @@ -5,26 +5,33 @@ //! Messages gossiped via rumors. use chrono::{DateTime, Utc}; +use ciborium::{Value, de::from_reader as from_cbor, ser::into_writer as into_cbor}; use rumors::Version; use serde::{Deserialize, Serialize}; use sled_hardware_types::BaseboardId; use thiserror::Error; -use x509_cert::Certificate; use sush_api::JobStartParams; use sush_common::authn::SignedLogin; use sush_common::jobs::JobOutputState; -use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob}; -use sush_common::keys::{KeyId, SshPublicKey}; +use sush_common::jobs::{Access, JobId, ProcessError, SessionId, SignedJob, SkipReason}; +use sush_common::keys::KeyId; use sush_common::version::VersionInfo; +use crate::format::{NoFormat, Versioned, Wire}; + /// Once a message schema has shipped, it is frozen, since any changes /// could break decoding of *existing* messages. Each version gets its /// own module; everything defined there and all of their dependencies /// (e.g., types shared with the HTTP API, etc.) become part of the frozen -/// version, whose serde shape on the gossip wire must not change. New -/// versions must implement `TryInto` to convert old messages into -/// compatible new ones. +/// version, whose serde shape on the gossip wire must not change. A +/// new version must convert [`From`] the old one; the +/// [`Wire`](crate::format::Wire) chain demands it. New variants must +/// keep this enum's encoding: a single-entry, text-keyed map. Older +/// sleds read an unrecognized text key as [`Unknown`](Self::Unknown) +/// and carry on, but an integer key or an array does not decode at +/// all, and a message that fails to decode aborts gossip sessions +/// instead of being ignored. /// /// Updates go sled by sled, so mixed gossip networks exist for /// the whole rollout. A message from a newer peer decodes as @@ -36,27 +43,31 @@ pub enum VersionedMessage { /// The initial message format. V0(v0::Message), - /// A message from a newer version. - Unknown(String), + Unknown { + version: String, + message: Vec, + }, } impl Serialize for VersionedMessage { fn serialize(&self, serializer: S) -> Result { - use serde::ser::Error as _; match self { Self::V0(message) => { serializer.serialize_newtype_variant("VersionedMessage", 0, "V0", message) } - Self::Unknown(version) => Err(S::Error::custom(format!( - "refusing to send a message from a newer version ({version})" - ))), + // Emit what was received. + Self::Unknown { message, .. } => { + let value: Value = from_cbor(message.as_slice()) + .expect("the bytes were encoded from a value this module decoded"); + value.serialize(serializer) + } } } } impl<'de> Deserialize<'de> for VersionedMessage { fn deserialize>(deserializer: D) -> Result { - use serde::de::{Error as _, IgnoredAny, MapAccess, Visitor}; + use serde::de::{Error as _, MapAccess, Visitor}; struct VersionVisitor; @@ -74,14 +85,22 @@ impl<'de> Deserialize<'de> for VersionedMessage { Ok(match version.as_str() { "V0" => VersionedMessage::V0(map.next_value()?), _ => { - map.next_value::()?; - VersionedMessage::Unknown(version) + let value: Value = map.next_value()?; + let whole = Value::Map(vec![(Value::Text(version.clone()), value)]); + let mut message = Vec::new(); + into_cbor(&whole, &mut message).expect("writing to a Vec cannot fail"); + VersionedMessage::Unknown { version, message } } }) } fn visit_str(self, version: &str) -> Result { - Ok(VersionedMessage::Unknown(version.to_string())) + let mut message = Vec::new(); + into_cbor(&version, &mut message).expect("writing to a Vec cannot fail"); + Ok(VersionedMessage::Unknown { + version: version.to_string(), + message, + }) } } @@ -89,39 +108,6 @@ impl<'de> Deserialize<'de> for VersionedMessage { } } -/// DER bytes for certificates, which have no serde support. -mod cert_der { - use serde::de::Visitor; - use serde::ser::Error as _; - use serde::{Deserializer, Serializer}; - use x509_cert::Certificate; - use x509_cert::der::{Decode as _, Encode as _}; - - pub fn serialize(cert: &Certificate, serializer: S) -> Result { - serializer.serialize_bytes(&cert.to_der().map_err(S::Error::custom)?) - } - - pub fn deserialize<'de, D: Deserializer<'de>>( - deserializer: D, - ) -> Result { - struct DerVisitor; - - impl<'de> Visitor<'de> for DerVisitor { - type Value = Certificate; - - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str("a DER-encoded certificate") - } - - fn visit_bytes(self, der: &[u8]) -> Result { - Certificate::from_der(der).map_err(E::custom) - } - } - - deserializer.deserialize_bytes(DerVisitor) - } -} - pub mod v0 { use super::*; @@ -138,6 +124,23 @@ pub mod v0 { } } + /// Version 0 is the initial wire format; see [`crate::format`]. + /// A later format converts up from this one infallibly, and the + /// state machine handles only the latest. + impl Versioned for Message { + const VERSION: u16 = 0; + } + + impl Wire for Message { + type Previous = NoFormat; + } + + impl From for Message { + fn from(none: NoFormat) -> Self { + match none {} + } + } + /// A request and the key that made it. /// /// The actor is attribution, not authority: it names the key whose @@ -210,7 +213,8 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[allow(clippy::large_enum_variant)] pub enum CertRequest { - Import(#[serde(with = "cert_der")] Certificate), + /// The certificate's DER bytes, decoded at use. + Import(#[serde(with = "crate::format::cbor_bytes")] Vec), Revoke(KeyId, DateTime), } @@ -236,7 +240,8 @@ pub mod v0 { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[allow(clippy::large_enum_variant)] pub enum IdentityRequest { - Login(SshPublicKey, SignedLogin), + /// OpenSSH public keys are string encoded and decoded at use. + Login(String, SignedLogin), Revoke(KeyId, DateTime), } @@ -257,6 +262,7 @@ pub mod v0 { JobOutputState, ), Error(JobId, DateTime, ProcessError), + Skipped(JobId, DateTime, SkipReason), } #[derive(Clone, Debug, Deserialize, Eq, Error, PartialEq, Serialize)] @@ -272,6 +278,10 @@ pub mod v0 { incoming_session: SessionId, incoming_version: Version, }, + #[error("Sled re-entered a burned universe")] + UniverseFlipFlop, + #[error("Sled hopped into a session its record cannot order")] + SessionHop, } } @@ -297,7 +307,7 @@ mod wire_format { #[track_caller] fn assert_wire_format(name: &str, message: VersionedMessage) { let mut bytes = Vec::new(); - ciborium::ser::into_writer(&message, &mut bytes).unwrap(); + into_cbor(&message, &mut bytes).unwrap(); let path = format!("tests/output/{name}.bin"); if env::var("EXPECTORATE").as_deref() == Ok("overwrite") { write(&path, &bytes).unwrap(); @@ -305,7 +315,7 @@ mod wire_format { let expected = read(&path).expect("missing snapshot"); assert_eq!(bytes, expected, "gossip wire format changed: {bytes:02x?}"); } - let decoded: VersionedMessage = ciborium::de::from_reader(bytes.as_slice()).unwrap(); + let decoded: VersionedMessage = from_cbor(bytes.as_slice()).unwrap(); assert_eq!(decoded, message, "wire format should round-trip"); } @@ -397,7 +407,7 @@ mod wire_format { #[test] fn identity_login_request() { use sush_common::authn::{ChallengeResponse, RequestVerifier}; - use sush_common::keys::{EncodedSignature, Signed, SshPublicKey}; + use sush_common::keys::{EncodedSignature, Signed}; // Craft deterministic evidence: nonces, then the ed25519 // basepoint as the verifier. @@ -411,7 +421,6 @@ mod wire_format { .unwrap(); let openssh = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILM+rvN+ot98qgEN796jTiQfZfG1KaT0PtFDJ13gEiGB test@sush"; - let public_key: SshPublicKey = serde_json::from_value(serde_json::json!(openssh)).unwrap(); let signed = Signed::new( response, @@ -425,7 +434,7 @@ mod wire_format { ); let msg: VersionedMessage = Message::Request(Request::identity( KeyId::from_str("zoo-zero").unwrap(), - IdentityRequest::Login(public_key, signed), + IdentityRequest::Login(openssh.to_string(), signed), )) .into(); assert_wire_format("identity-login-request", msg); @@ -452,8 +461,6 @@ mod wire_format { /// still fails. #[test] fn unknown_version_tolerated() { - use ciborium::value::Value; - let v1 = Value::Map(vec![( Value::Text("V1".to_string()), Value::Map(vec![( @@ -462,14 +469,17 @@ mod wire_format { )]), )]); let mut bytes = Vec::new(); - ciborium::ser::into_writer(&v1, &mut bytes).unwrap(); - let decoded: VersionedMessage = ciborium::de::from_reader(bytes.as_slice()).unwrap(); - assert_eq!(decoded, VersionedMessage::Unknown("V1".to_string())); + into_cbor(&v1, &mut bytes).unwrap(); + let decoded: VersionedMessage = from_cbor(bytes.as_slice()).unwrap(); + assert!(matches!(&decoded, VersionedMessage::Unknown { version, .. } if version == "V1")); + // Rumors panics when a message fails to serialize, so this + // must not fail, and it must emit what was received. let mut resent = Vec::new(); - assert!(ciborium::ser::into_writer(&decoded, &mut resent).is_err()); + into_cbor(&decoded, &mut resent).unwrap(); + assert_eq!(resent, bytes); - let corrupt: Result = ciborium::de::from_reader([0x01].as_slice()); + let corrupt: Result = from_cbor([0x01].as_slice()); assert!(corrupt.is_err()); } @@ -504,12 +514,13 @@ mod wire_format { #[test] fn cert_requests() { - use x509_cert::der::DecodePem as _; + use x509_cert::Certificate; + use x509_cert::der::{DecodePem as _, Encode as _}; let cert = Certificate::from_pem(include_str!("../../client/certs/staging.pem")).unwrap(); let msg: VersionedMessage = Message::Request(Request::cert( KeyId::from_str("zoo-zero").unwrap(), - CertRequest::Import(cert), + CertRequest::Import(cert.to_der().unwrap()), )) .into(); assert_wire_format("cert-import-request", msg); @@ -606,6 +617,66 @@ mod wire_format { assert_wire_format("concurrent-sessions-error", msg); } + #[test] + fn job_skipped_event() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Job(JobEvent::Skipped( + JobId::from_str("zoo-zero").unwrap(), + "2026-09-04T20:00:00Z".parse().unwrap(), + SkipReason::BelowFloor, + )), + ) + .into(); + assert_wire_format("job-skipped-event", msg); + } + + #[test] + fn session_ended_skip_event() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Job(JobEvent::Skipped( + JobId::from_str("zoo-zero").unwrap(), + "2026-09-04T20:00:00Z".parse().unwrap(), + SkipReason::SessionEnded, + )), + ) + .into(); + assert_wire_format("session-ended-skip-event", msg); + } + + #[test] + fn session_hop_error() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Error(Error::SessionHop), + ) + .into(); + assert_wire_format("session-hop-error", msg); + } + + #[test] + fn universe_flip_flop_error() { + let msg: VersionedMessage = Message::Event( + BaseboardId { + part_number: "913-0000019".to_string(), + serial_number: "BRM42220030".to_string(), + }, + Event::Error(Error::UniverseFlipFlop), + ) + .into(); + assert_wire_format("universe-flip-flop-error", msg); + } + #[test] fn job_start_interactive_request() { use sush_api::JobWait; diff --git a/server/src/state.rs b/server/src/state.rs index 219732c..f9fe3d9 100644 --- a/server/src/state.rs +++ b/server/src/state.rs @@ -14,7 +14,7 @@ use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use futures::{FutureExt as _, Stream, StreamExt}; use lru::LruCache; -use rumors::{CausalMessages, Peer, Rumors, Version}; +use rumors::{CausalMessages, Network, Rumors, Version}; use sled_hardware_types::BaseboardId; use slog::{Logger, debug, error, info, o, warn}; use tokio::sync::watch; @@ -22,23 +22,26 @@ use tokio::task::JoinHandle; use tokio::{select, spawn}; use tokio_util::sync::CancellationToken; use x509_cert::Certificate; -use x509_cert::der::Encode as _; +use x509_cert::der::{Decode as _, Encode as _}; use sush_api::JobStartParams; use sush_common::authn::{Identity, Nonce, RequestVerifier, SignedLogin}; use sush_common::jobs::{ - Access, JobId, JobStatus, JobStatusMap, ProcessError, Session, SessionId, SessionSushNonce, - SignedJob, + Access, JobId, JobStatus, JobStatusMap, LastJob, ProcessError, Session, SessionId, + SessionSushNonce, SignedJob, SkipReason, }; use sush_common::keys::{KeyError, KeyId, Signature, SshPublicKey}; use sush_common::targets::Cubbies; use sush_common::version::{VersionInfo, VersionMap}; -use crate::bookmark::{BookmarkSource, SushBookmark}; +use crate::bloom::Bloom; +use crate::bookmark::SushBookmark; +use crate::boundary::{Boundary, BoundaryStore, Committed, JobOutcome}; use crate::executor::{Executor, PathIsolation}; -use crate::gossip::Universe; +use crate::gossip::{Seed, Universe}; use crate::history::JobHistory; use crate::job::SocketSender; +use crate::locker::Locker; use crate::messages::v0::{ CertRequest, Error, Event, IdentityRequest, JobEvent, JobRequest, Message, Request, SessionRequest, @@ -49,6 +52,7 @@ use crate::output::JobOutputDir; pub type AttachmentPoints = BTreeMap>>; pub type Certificates = BTreeMap; pub type GossipNetwork = Rumors; +pub type GossipSeed = Seed; pub type GossipUniverse = Universe; pub type QueuedJobs = BTreeMap; pub type RunningJobs = BTreeMap<(JobId, BaseboardId), DateTime>; @@ -105,7 +109,6 @@ pub enum SessionState { Active { /// Last observed session request. frontier: Version, - /// Start of this session; identity anchor. started: Version, /// The active session. session: Box, @@ -132,10 +135,12 @@ impl SessionState { Active { session, queued_jobs, + started, .. } => Some(SessionGuard { inner: session, queued_jobs, + started, }), } } @@ -187,6 +192,8 @@ impl Default for SessionState { struct SessionGuard<'a> { inner: &'a mut Session, queued_jobs: &'a mut QueuedJobs, + /// The version of the session's start message; identity anchor. + started: &'a Version, } impl<'a> SessionGuard<'a> { @@ -206,10 +213,6 @@ impl<'a> SessionGuard<'a> { self.inner.skip_job(*job_id) } - pub fn next_queued_job(&mut self) -> Option { - self.queued_jobs.remove(&self.inner.next_job_id()) - } - #[allow(clippy::too_many_arguments)] pub fn enqueue_job( &mut self, @@ -225,10 +228,16 @@ impl<'a> SessionGuard<'a> { ) { let job_id = *job.job_id(); let targeted = job.payload().runs_on(own_baseboard, cubbies); - if history.contains(&job_id) { + // Adjudication can give the boundary job a status before its + // request replays. The queue releases jobs in order, so + // dropping that request would wedge the session behind the + // boundary job forever. A replayed request therefore always + // joins the queue; only a live duplicate is dropped. + if !replayed && history.contains(&job_id) { // Note but otherwise ignore the duplicate job. info!(log, "already started job"; "job_id" => %job_id); - } else if self.queued_jobs.len() >= MAX_QUEUED_JOBS + } else if !replayed + && self.queued_jobs.len() >= MAX_QUEUED_JOBS && !self.queued_jobs.contains_key(&job_id) { // We have no choice; drop the job on the floor. @@ -308,18 +317,63 @@ impl<'a> SessionGuard<'a> { history: &mut JobHistory, executor: &mut Executor, attachments: &mut AttachmentPoints, + past: &mut Past, + rumors: Option<&GossipNetwork>, ) { - while let Some(QueuedJob { - job: request, - params, - replayed, - }) = self.next_queued_job() - { + loop { + let next_id = self.inner.next_job_id(); + if !self.queued_jobs.contains_key(&next_id) { + break; + } + // A hop can raise the floor mid-drain, so screen on every pass. + let admission = match rumors { + None => Admission::Admit, + Some(rumors) => past.screen(rumors.network(), self.session_id(), self.started), + }; + // A hop raises the floor and screens again. + if matches!(admission, Admission::Hop) { + let Some(rumors) = rumors else { + break; + }; + warn!( + log, "raising the execution floor above a session the boundary record cannot order"; + "session_id" => %self.session_id(), "started" => ?self.started, + ); + past.hop(rumors, own_baseboard, Error::SessionHop); + continue; + } + let QueuedJob { + job: request, + params, + replayed, + } = self.queued_jobs.remove(&next_id).expect("checked above"); let (tx_attachment, rx_attachment) = watch::channel(None); let job_id = request.payload().job_id().to_owned(); if request.payload().runs_on(own_baseboard, cubbies) { - if replayed { - warn!(log, "not executing replayed job"; "job_id" => %job_id); + if past.boundary.untrusted() { + warn!( + log, "refusing job, the execution boundary is untrusted"; + "job_id" => %job_id, + ); + executor.job_refused( + job_id, + ProcessError::Io { + what: "consulting the execution boundary".to_string(), + error: "the boundary records on this sled's M.2s disagree \ + or are corrupt; an M.2 may have failed and the sled \ + needs service" + .to_string(), + }, + ); + } else if matches!(admission, Admission::Refuse) { + // Below the floor the queue skips forward in chain + // order. A live job gets the terminal skip status; + // a replayed one is history, and re-reporting it + // on every rejoin would grow the message set. + if !replayed { + warn!(log, "skipping job below the execution floor"; "job_id" => %job_id); + executor.job_skipped(job_id, SkipReason::BelowFloor); + } } else if history .get_job_status(&job_id) .map(|status| { @@ -327,8 +381,28 @@ impl<'a> SessionGuard<'a> { }) .unwrap_or(true) { - executor.job_start(certs, request.clone(), params, tx_attachment); - attachments.insert(job_id, rx_attachment); + let Some(rumors) = rumors else { + warn!(log, "not starting a job while draining"; "job_id" => %job_id); + self.job_started(request); + continue; + }; + let successor = self + .session_id() + .next_job_id(&LastJob::Some(request.clone())); + let boundary = Boundary { + network: rumors.network(), + burned: past.burned_for(rumors.network()), + executed: past.executed_for(rumors.network(), self.started), + job: Some(Committed { + session: self.session_id(), + job: job_id, + successor, + outcome: JobOutcome::Committed, + }), + }; + if executor.job_start(certs, request.clone(), params, tx_attachment, boundary) { + attachments.insert(job_id, rx_attachment); + } } } self.job_started(request); @@ -336,6 +410,159 @@ impl<'a> SessionGuard<'a> { } } +/// What [`Past::screen`] decides about a live job of the active +/// session. +enum Admission { + /// Execute normally. + Admit, + /// The session does not start above the floor; skip the job. + Refuse, + /// The boundary record cannot order this session, so a previous + /// life of this sled may have run its jobs. The sled reports an + /// error and raises its floor, as if it had just entered the + /// universe at this instant. + Hop, +} + +/// This incarnation's relationship to its own past: where it entered +/// the universe, what a previous life committed to, and what it left +/// running. +#[derive(Debug)] +pub struct Past { + /// The frontier just before this life's first send into this + /// universe: the general replay line. Messages that do not + /// strictly follow it count as replayed. + arrival: Version, + /// The frontier just past this life's first send. Own-baseboard + /// events at or below it belong to a previous life; everything + /// this life sends causally follows it. Never persisted: each + /// life computes its own, so the split is exact and independent of + /// which peer we joined through. It is not the general replay + /// line, because a neighbor's fresh traffic reaches us before our + /// first send reaches the neighbor. + birth: Version, + /// The last job this sled committed to executing, durable across + /// restarts. + boundary: Arc, + /// The boundary record as we found it on entering this universe. + /// The live store advances with our own jobs; [`Past::screen`] + /// compares sessions against the previous life's commitment, so + /// it reads this frozen copy. + committed: Option, + /// The execution floor, raised in memory at entry into a burned + /// universe, at record-less entry into a universe with history, + /// and on a session hop. Only sessions started strictly above it + /// are served; below it this sled cannot tell replay from re-run. + /// Never persisted: whatever raised it is still there after a + /// restart and raises it again. See [`entry_floor`]. + floor: Option, + /// Jobs whose replayed start events say a previous life ran them. + /// The end of such a job may replay later and remove it from the + /// set, so the survivors are known only when replay finishes. + zombies: BTreeSet, +} + +/// Whether `version` strictly dominates `mark`. Equal and concurrent +/// versions do not. +fn dominates(version: &Version, mark: &Version) -> bool { + version > mark +} + +impl Past { + pub fn new( + arrival: Version, + birth: Version, + boundary: Arc, + committed: Option, + floor: Option, + ) -> Self { + Self { + arrival, + birth, + boundary, + committed, + floor, + zombies: BTreeSet::new(), + } + } + + /// Decide whether the active session, identified by its id and + /// the version of its start message, may run a live job here. + fn screen(&self, network: Network, session: SessionId, started: &Version) -> Admission { + // We must refuse anything below our floor, because we cannot + // tell replay from re-run there. + if self + .floor + .as_ref() + .is_some_and(|floor| !dominates(started, floor)) + { + return Admission::Refuse; + } + + // If we have committed to nothing, anything is safe to admit. + let Some(committed) = &self.committed else { + return Admission::Admit; + }; + + // Jobs cannot cross universes, and anything a previous life + // ran in this universe is below the floor checked above. + if committed.network != network { + return Admission::Admit; + } + + match &committed.job { + Some(job) if job.session == session => Admission::Admit, + _ => { + if dominates(started, &committed.executed) { + Admission::Admit + } else { + Admission::Hop + } + } + } + } + + /// Report the cause of an [`Admission::Hop`] to the gossip set, + /// then set the floor at the frontier that includes the report. + fn hop(&mut self, rumors: &GossipNetwork, own_baseboard: &BaseboardId, report: Error) { + rumors.send(Message::Event(own_baseboard.clone(), Event::Error(report)).into()); + self.floor = Some(rumors.snapshot().latest().clone()); + } + + /// The burned set for a record committed in `network`; see + /// [`Boundary::burned_for`]. + fn burned_for(&self, network: Network) -> Bloom { + self.committed + .as_ref() + .map(|boundary| boundary.burned_for(network)) + .unwrap_or_default() + } + + /// The executed-session join for a record committed in `network` + /// under a session started at `started`; see + /// [`Boundary::executed_for`]. + fn executed_for(&self, network: Network, started: &Version) -> Version { + self.committed + .as_ref() + .map(|boundary| boundary.executed_for(network, started)) + .unwrap_or_else(|| started.clone()) + } + + /// Whether a message at `version` is live traffic rather than + /// replayed history: only strict causal descendants of the + /// arrival are live. + fn is_live(&self, version: &Version) -> bool { + dominates(version, &self.arrival) + } + + /// Whether this sled's own event at `version` came from a + /// previous life. The split is exact: nothing a previous life + /// sent dominates the birth, and everything this life sends does. + fn is_from_previous_life(&self, version: &Version) -> bool { + !dominates(version, &self.birth) + } +} + #[derive(Debug)] pub struct State { /// The ID of the baseboard (sled) the server is running on. @@ -361,14 +588,8 @@ pub struct State { roots: Box<[KeyId]>, /// Baseboards by cubby number, as much of it as is known. cubbies: Cubbies, - /// The causal frontier we joined this universe at, if we joined - /// rather than seeded it. - join_frontier: Option, - /// Jobs whose start event on our own baseboard arrived as replayed - /// history. A previous life started them, and no executor of ours - /// will ever stop them. A terminal event clears its job, so - /// mid-replay entries are only suspects. - zombies: BTreeSet, + /// This incarnation's relationship to its own past. + past: Past, /// Message versions from newer builds, each warned about once. unknown_versions: BTreeSet, /// Build provenance by sled. @@ -387,7 +608,7 @@ impl State { own_baseboard: BaseboardId, root_certs: &[Certificate], session_sush_nonce: Arc>, - join_frontier: Option, + past: Past, ) -> Result { let certs = root_certs .iter() @@ -414,8 +635,7 @@ impl State { unknown_versions: Default::default(), identities: LruCache::new(MAX_REGISTERED_IDENTITIES), revoked_keys: LruCache::new(MAX_REVOKED_KEYS), - join_frontier, - zombies: Default::default(), + past, }; new.validate_certs(&roots); for root in &roots { @@ -463,7 +683,8 @@ impl State { /// Jobs a previous life of this server started and left running. pub fn zombies(&self) -> BTreeSet { - self.zombies + self.past + .zombies .iter() .filter(|job_id| { self.running @@ -473,13 +694,29 @@ impl State { .collect() } - /// Whether a message at `version` is live traffic rather than - /// replayed history. Only strict causal descendants of the join - /// frontier are live. fn is_live(&self, version: &Version) -> bool { - self.join_frontier - .as_ref() - .is_none_or(|frontier| version > frontier) + self.past.is_live(version) + } + + /// The boundary record carries the boundary job's ending, so the + /// next life can tell the truth instead of guessing. The write is + /// spawned because the state machine is synchronous, and losing + /// the race to a crash only falls back to interrupted. + fn record_boundary_outcome(&self, job_id: &JobId) { + let Some(status) = self + .history + .get_job_status(job_id) + .and_then(|map| map.get(&self.own_baseboard)) + .cloned() + else { + return; + }; + if !status.is_terminal() { + return; + } + let store = self.past.boundary.clone(); + let job_id = *job_id; + spawn(async move { store.record_outcome(&job_id, &status).await }); } pub fn get_job_status(&self, job_id: &JobId) -> Option<&JobStatusMap> { @@ -541,11 +778,25 @@ impl State { self.revoked_keys.peek(key_id).is_some() } + fn skip_queued_jobs(&self, executor: &Executor) { + for (job_id, queued) in self.session.queued_jobs().into_iter().flatten() { + if !queued.replayed + && queued + .job + .payload() + .runs_on(&self.own_baseboard, &self.cubbies) + { + executor.job_skipped(*job_id, SkipReason::SessionEnded); + } + } + } + #[allow(clippy::result_large_err)] fn update( &mut self, log: &Logger, executor: &mut Executor, + rumors: Option<&GossipNetwork>, incoming_version: &Version, message: &Arc, ) -> Result<(), Error> { @@ -553,7 +804,10 @@ impl State { match message.as_ref() { V0(Message::Request(request)) => match request { Request::Cert(attributed) => match attributed.as_parts() { - (actor, CertRequest::Import(cert)) => match self.cert_import(cert) { + (actor, CertRequest::Import(der)) => match Certificate::from_der(der) + .map_err(KeyError::from) + .and_then(|cert| self.cert_import(&cert)) + { Ok(key_id) => { info!(log, "imported certificate"; "key_id" => %key_id, "actor" => %actor); self.validate_certs(&self.roots.clone()); @@ -596,10 +850,31 @@ impl State { log, "session started"; "session_id" => %session_id, "actor" => %actor, ); + self.skip_queued_jobs(executor); + let mut session = Session::started(*session_id, actor.clone()); + // The committed session resumes at its + // stored successor: every earlier chain + // position was already handled by the life + // that stored it, and the chain needs no + // replay to verify the next job. + if let Some(committed) = self + .past + .committed + .as_ref() + .and_then(|boundary| boundary.job.as_ref()) + .filter(|committed| committed.session == *session_id) + { + info!( + log, "resuming the committed session at its successor"; + "session_id" => %session_id, + "successor" => %committed.successor, + ); + session.resume_at(committed.successor); + } self.session = Active { frontier: self.session.frontier() | incoming_version.clone(), started: incoming_version.clone(), - session: Box::new(Session::started(*session_id, actor.clone())), + session: Box::new(session), queued_jobs: QueuedJobs::new(), attach_grants: BTreeMap::new(), }; @@ -623,9 +898,9 @@ impl State { incoming_session: *session_id, incoming_version: incoming_version.clone(), }; - self.session = Inactive { - frontier: &*frontier | incoming_version.clone(), - }; + let frontier = &*frontier | incoming_version.clone(); + self.skip_queued_jobs(executor); + self.session = Inactive { frontier }; return Err(error); } @@ -668,6 +943,7 @@ impl State { log, "session stopped"; "session_id" => %session_id, "actor" => %actor, ); + self.skip_queued_jobs(executor); self.session = Inactive { frontier: frontier.clone(), } @@ -749,6 +1025,8 @@ impl State { &mut self.history, executor, &mut self.attachments, + &mut self.past, + rumors, ); } else { info!( @@ -791,8 +1069,9 @@ impl State { // session's own start (since each session is // linearized by its accepting server). // - // Refused jobs targeting this sled record an error - // status so the submitter learns their fate. + // Refused jobs targeting this sled record the + // terminal skip status so the submitter learns + // their fate. let session_id = signed.payload().session_id(); let live = self.is_live(incoming_version); match self.session.active_session() { @@ -816,12 +1095,14 @@ impl State { &mut self.history, executor, &mut self.attachments, + &mut self.past, + rumors, ); } _ => { let job_id = *signed.job_id(); warn!( - log, "refusing job for inactive session"; + log, "skipping job for inactive session"; "job_id" => %job_id, "session_id" => %session_id, "actor" => %actor, @@ -830,12 +1111,7 @@ impl State { && signed.payload().runs_on(&self.own_baseboard, &self.cubbies) && !self.history.contains(&job_id) { - executor.job_refused( - job_id, - ProcessError::InvalidJob(format!( - "session `{session_id}` is not active" - )), - ); + executor.job_skipped(job_id, SkipReason::SessionEnded); } } } @@ -856,8 +1132,10 @@ impl State { } }, Request::Identity(attributed) => match attributed.as_parts() { - (actor, IdentityRequest::Login(public_key, signed)) => { - match verify_login(public_key, signed) { + (actor, IdentityRequest::Login(openssh, signed)) => { + match SshPublicKey::from_openssh(openssh) + .and_then(|public_key| verify_login(&public_key, signed)) + { Ok(registered) => { let identity = ®istered.identity; if self.revoked_keys.peek(&identity.key_id).is_some() { @@ -901,12 +1179,36 @@ impl State { // Track the active set of known-running jobs anywhere in the rack. Event::Job(job_event) => match job_event { JobEvent::Start(job_id, when) => { + // A start never displaces a terminal status. + // After an identity change, an adjudicated + // Interrupted and a late replayed start are + // concurrent, and the verdict must win in + // both arrival orders. + if self + .history + .get_job_status(job_id) + .and_then(|status| status.get(baseboard_id)) + .is_some_and(|status| status.is_terminal()) + { + info!(log, "ignoring a start for a settled job"; "job_id" => %job_id); + return Ok(()); + } info!(log, "job started"; "job_id" => %job_id, "when" => %when); self.running.insert((*job_id, baseboard_id.clone()), *when); - // A replayed start on our own baseboard is a - // previous life's. Only these can be zombies. - if *baseboard_id == self.own_baseboard && !self.is_live(incoming_version) { - self.zombies.insert(*job_id); + // A start on our own baseboard sent by a + // previous life names a job whose process + // died with that life. No executor here will + // ever end it, so the job is a zombie: unless + // replay delivers its ending, we report it + // interrupted once replay drains. The birth + // mark splits the lives exactly; the arrival + // mark would misread a previous life's start + // as this life's own whenever the join peer + // lagged behind that start. + if *baseboard_id == self.own_baseboard + && self.past.is_from_previous_life(incoming_version) + { + self.past.zombies.insert(*job_id); } self.history.set_job_status( job_id, @@ -924,7 +1226,7 @@ impl State { info!(log, "job stopped"; "job_id" => %job_id, "when" => %when, "result" => ?result); if baseboard_id == &self.own_baseboard { self.attachments.remove(job_id); - self.zombies.remove(job_id); + self.past.zombies.remove(job_id); } self.running.remove(&(*job_id, baseboard_id.clone())); self.history.transition_job_status( @@ -963,13 +1265,14 @@ impl State { ); if *baseboard_id == self.own_baseboard { executor.job_stopped(job_id); + self.record_boundary_outcome(job_id); } } JobEvent::Error(job_id, when, error) => { error!(log, "job error"; "job_id" => %job_id, "when" => %when, "error" => %error); if baseboard_id == &self.own_baseboard { self.attachments.remove(job_id); - self.zombies.remove(job_id); + self.past.zombies.remove(job_id); } self.running.remove(&(*job_id, baseboard_id.clone())); self.history.transition_job_status( @@ -995,8 +1298,36 @@ impl State { ); if *baseboard_id == self.own_baseboard { executor.job_stopped(job_id); + self.record_boundary_outcome(job_id); } } + JobEvent::Skipped(job_id, when, reason) => { + info!( + log, "job skipped"; + "job_id" => %job_id, "when" => %when, "reason" => %reason, + ); + self.running.remove(&(*job_id, baseboard_id.clone())); + self.history.transition_job_status( + job_id, + baseboard_id, + Some(incoming_version.rank()), + // A skip is the reporting sled's decision + // about itself, and never displaces a + // terminal status it already reported. + |old_status| match old_status { + None + | Some(JobStatus::Queued { .. }) + | Some(JobStatus::Started { .. }) => Some(JobStatus::Skipped { + job_id: *job_id, + time_skipped: *when, + reason: *reason, + }), + _ => None, + }, + self.session.queued_jobs(), + &self.running, + ); + } }, Event::Error(error) => { error!(log, "session error"; "error" => %error); @@ -1010,7 +1341,7 @@ impl State { } } }, - Unknown(version) => { + Unknown { version, .. } => { if self.unknown_versions.insert(version.clone()) { warn!(log, "ignoring messages from a newer peer"; "version" => version); } @@ -1039,7 +1370,7 @@ fn apply_message( message: &Arc, ) { tx_state.send_modify(|state| { - if let Err(error) = state.update(log, executor, version, message) { + if let Err(error) = state.update(log, executor, rumors, version, message) { error!(log, "state update failed"; "error" => ?error); // Re-gossiping a replayed message's error would grow the // set a little more on every rejoin by every sled. @@ -1112,30 +1443,141 @@ fn reap_zombies( } } -/// Create a fresh gossip network with this server as its only peer. +/// Adjudicate the boundary job if replay gave it no status on our +/// baseboard: announce its recorded ending, or interrupted when none +/// was recorded. No status means its fate never left this sled, and +/// no other sled can ever report it. A replayed start makes it a +/// zombie instead, and a replayed end settles it. Adjudicate only +/// after replay drains, like [`reap_zombies`], and once per universe. +/// The ruling is gossiped to every sled, but a sled that already +/// holds a terminal status for the job keeps it; only sleds with +/// none adopt the ruling, so one sled may show the real ending while +/// another shows interrupted. +fn adjudicate_boundary( + log: &Logger, + tx_state: &watch::Sender, + rumors: Option<&GossipNetwork>, + own_baseboard: &BaseboardId, + snapshot: Option<&Boundary>, + survivors: &BTreeSet, + adjudicated: &mut Option, +) { + // A floor commits to no job, so there is nothing to rule on. + let Some(Committed { job, outcome, .. }) = snapshot.and_then(|boundary| boundary.job.as_ref()) + else { + return; + }; + let job_id = *job; + // One ruling per boundary: a sled that swaps universes again + // without running a job must not re-adjudicate the same job. + if *adjudicated == Some(job_id) { + return; + } + // A survivor's events land in the new universe when it finishes, + // so it needs no verdict now and may need one at a later swap. + if survivors.contains(&job_id) { + return; + } + *adjudicated = Some(job_id); + let witnessed = { + let state = tx_state.borrow(); + state + .get_job_status(&job_id) + .is_some_and(|status| status.get(own_baseboard).is_some()) + }; + if witnessed { + return; + } + // A stopped ending announces the start and stop pair, so the + // ordinary status transitions apply on every sled; a bare stop + // with no prior start would be dropped. + let events = match outcome { + JobOutcome::Ended(JobStatus::Stopped { + time_started, + time_stopped, + result, + output, + .. + }) => vec![ + JobEvent::Start(job_id, *time_started), + JobEvent::Stop(job_id, *time_stopped, result.clone(), output.clone()), + ], + JobOutcome::Ended(JobStatus::Error { + time_error, error, .. + }) => vec![JobEvent::Error(job_id, *time_error, error.clone())], + JobOutcome::Ended(_) | JobOutcome::Committed => vec![JobEvent::Error( + job_id, + Utc::now(), + ProcessError::Interrupted, + )], + }; + if let Some(rumors) = rumors { + for event in events { + rumors.send(Message::Event(own_baseboard.clone(), Event::Job(event)).into()); + } + } + warn!(log, "adjudicated an unwitnessed job from a previous life"; "job_id" => %job_id); +} + +/// The execution floor for entering a universe, or `None` when the +/// record can order everything this sled may meet there. Two kinds +/// of entry have history the record cannot order. Re-entering a +/// burned universe is the flip-flop: the record overwrote this +/// universe's watermark when it left, and the sled reports the +/// return as an error. Entering a universe that has history while +/// holding no record leaves the sled unable to tell a first visit +/// from a return after a clean slate: the universe may hold jobs +/// this baseboard already ran, and they must not run twice. Both +/// set the floor at the frontier that includes this life's first +/// send. +/// +/// The floor must live in memory only. Its version is created by +/// sending a message, and if the sled dies before the message +/// reaches anyone, no other copy of it ever exists. A floor written +/// to disk would carry that dead version into the next life, where +/// no future session start could ever dominate it, and every one +/// would be refused until a cold boot. A floor raised fresh at each +/// entry is built from a message the living sled is actively +/// gossiping, so future sessions come to dominate it. +fn entry_floor( + log: &Logger, + snapshot: Option<&Boundary>, + arrival: &Version, + rumors: &GossipNetwork, + own_baseboard: &BaseboardId, +) -> Option { + let network = rumors.network(); + match snapshot { + // A record committed in this universe is its watermark, even + // when its burned set names this network: a sled that returns + // and commits burns its own network into the replacement + // record, and the Bloom set can never drop the stale entry. + Some(boundary) if boundary.network != network && boundary.is_burned(network) => { + warn!( + log, "re-entered a universe this sled's boundary record burned"; + "committed" => ?boundary.job, + ); + rumors.send( + Message::Event(own_baseboard.clone(), Event::Error(Error::UniverseFlipFlop)).into(), + ); + } + None if *arrival != Version::new() => { + warn!(log, "no boundary record, and this universe has history"); + } + _ => return None, + } + Some(rumors.snapshot().latest().clone()) +} + +/// Grow a fresh gossip seed over sush's message type. /// /// A peer that seeds its own network has no one to gossip with, so jobs run /// only on the server that accepted them, and no server learns about any other /// server's sessions. This stands in for joining the rack's network over -/// sprockets on the bootstrap network. -/// -/// A pristine seed's bookmark touches no storage, and identities -/// recorded there are reclaimed only after a migration returns us to -/// their universe. Bad storage would abort every session at the -/// persist gate, before the seed could even learn to migrate. Probe -/// first and shed on failure. -pub async fn seed_gossip(bookmarks: &BookmarkSource) -> GossipNetwork { - let handle = match bookmarks.probe().await { - Ok(()) => bookmarks.next_handle(), - Err(_) => bookmarks.shed_handle(), - }; - match Peer::seed().bookmark(handle).await { - Ok(peer) => peer.into_rumors(), - Err(unbookmarked) => match unbookmarked.peer.bookmark(bookmarks.shed_handle()).await { - Ok(peer) => peer.into_rumors(), - Err(_) => unreachable!("a shed bookmark never touches storage"), - }, - } +/// sprockets on the bootstrap network. Storage semantics are +/// [`Seed::grow`]'s. +pub async fn seed_gossip(log: &Logger, locker: &Locker) -> GossipSeed { + Seed::grow(log, locker).await } #[derive(Debug)] @@ -1165,6 +1607,7 @@ impl StateManager { mut universe: watch::Receiver, roots: &[Certificate], session_sush_nonce: Arc>, + store: Arc, shutdown: CancellationToken, ) -> Result<(watch::Receiver, JoinHandle<()>), KeyError> where @@ -1178,18 +1621,29 @@ impl StateManager { // in the face of arbitrary *causal* reorderings. // `borrow_and_update` marks the value seen, so a migration // that landed before we subscribed does not replay as a swap. - let Universe { - rumors: initial, - frontier, - } = universe.borrow_and_update().clone(); + let Universe { rumors: initial } = universe.borrow_and_update().clone(); let mut causal_messages = initial.causal_messages(); - // We report our current state through a watch channel. + // Decisions read a snapshot of the boundary, never the live + // store: the launcher advances the store concurrently, and a + // job committed after the snapshot is this incarnation's, not + // the past's. + let boundary = store.boundary(); + let boundary_store = store.clone(); + + // We report our current state through a watch channel. The + // placeholder birth is replaced before any message applies. let mut initial_state = State::new( own_baseboard.clone(), roots, session_sush_nonce.clone(), - frontier.clone(), + Past::new( + Version::new(), + Version::new(), + store.clone(), + boundary.clone(), + None, + ), )?; initial_state.cubbies = cubbies.borrow_and_update().clone(); let (tx_state, rx_state) = watch::channel(initial_state); @@ -1200,6 +1654,7 @@ impl StateManager { log.new(o!("component" => "executor")), path_isolation, output_dir, + store, shutdown.child_token(), ); @@ -1214,26 +1669,54 @@ impl StateManager { spawn(async move { info!(log, "managing state"); - // Replay bookkeeping. `frontier` classifies incoming - // messages (at or concurrent with it means replayed - // history); `survivors` are jobs this incarnation itself - // runs across a universe swap; `reaped` are zombies - // already declared interrupted. - let mut frontier = frontier; - let mut survivors: BTreeSet = BTreeSet::new(); - let mut reaped: BTreeSet = BTreeSet::new(); - - // Announce our build. - if let Some((rumors, _)) = &gossip { - rumors.send( - Message::Event( - own_baseboard.clone(), - Event::Version(VersionInfo::current()), - ) - .into(), + // Before any message is processed: announce our build + // (the send that starts this life's causal presence), + // raise the floor in case the record burned the + // initial universe, and take the birth mark that + // splits this life's traffic from replayed history. + let boundary = match &gossip { + Some((rumors, _)) => { + let arrival = rumors.snapshot().latest().clone(); + rumors.send( + Message::Event( + own_baseboard.clone(), + Event::Version(VersionInfo::current()), + ) + .into(), + ); + let floor = + entry_floor(&log, boundary.as_ref(), &arrival, rumors, &own_baseboard); + let birth = rumors.snapshot().latest().clone(); + tx_state.send_modify(|state| { + state.past = Past::new( + arrival, + birth, + boundary_store.clone(), + boundary.clone(), + floor.clone(), + ); + }); + boundary + } + None => boundary, + }; + if let Some(Committed { session, job, .. }) = + boundary.as_ref().and_then(|boundary| boundary.job.as_ref()) + { + info!( + log, "inherited an execution boundary; the committed session resumes at its stored successor"; + "session_id" => %session, "job_id" => %job, ); } + // Replay bookkeeping. `survivors` are jobs this + // incarnation itself runs across a universe swap; + // `reaped` are zombies already declared interrupted. + let mut boundary = boundary; + let mut survivors: BTreeSet = BTreeSet::new(); + let mut reaped: BTreeSet = BTreeSet::new(); + let mut adjudicated: Option = None; + // These flip both to `true` once our two input streams (local // requests and local events from the executor) terminate or // we're shutting down. At this point, we must drop `gossip` @@ -1293,8 +1776,7 @@ impl StateManager { break; } Some((version, message)) => { - // Past the join frontier means the message is live. - let live = frontier.as_ref().is_none_or(|f| version > f); + let live = tx_state.borrow().is_live(&version); apply_message( &log, &tx_state, @@ -1322,6 +1804,15 @@ impl StateManager { &survivors, &mut reaped, ); + adjudicate_boundary( + &log, + &tx_state, + gossip.as_ref().map(|(rumors, _)| rumors), + &own_baseboard, + boundary.as_ref(), + &survivors, + &mut adjudicated, + ); } }, }, @@ -1356,29 +1847,45 @@ impl StateManager { causal_messages = fresh.rumors.causal_messages(); // A reaped zombie may still show as running (its // error event races the swap); it is no survivor. + // A job the executor is still launching is a + // survivor too: its start event may not have + // applied yet, but its events land in the new + // universe like any running job's. survivors = tx_state.borrow().own_running_jobs(); + survivors.extend(executor.in_flight()); survivors.retain(|job_id| !reaped.contains(job_id)); reaped = BTreeSet::new(); - frontier = fresh.frontier.clone(); + *rumors = fresh.rumors; + let arrival = rumors.snapshot().latest().clone(); + rumors.send( + Message::Event( + own_baseboard.clone(), + Event::Version(VersionInfo::current()), + ) + .into(), + ); + let boundary_store = tx_state.borrow().past.boundary.clone(); + boundary = boundary_store.boundary(); + let floor = + entry_floor(&log, boundary.as_ref(), &arrival, rumors, &own_baseboard); + let birth = rumors.snapshot().latest().clone(); // TODO: re-inject local job state (policy pending). tx_state.send_modify(|state| { *state = State::new( own_baseboard.clone(), &roots, state.session_sush_nonce.clone(), - fresh.frontier.clone(), + Past::new( + arrival, + birth, + boundary_store.clone(), + boundary.clone(), + floor.clone(), + ), ) .expect("roots validated at startup"); state.cubbies = cubbies.borrow().clone(); }); - *rumors = fresh.rumors; - rumors.send( - Message::Event( - own_baseboard.clone(), - Event::Version(VersionInfo::current()), - ) - .into(), - ); // The set received at join is already local: // drain it, then reap. drain_ready( @@ -1397,6 +1904,15 @@ impl StateManager { &survivors, &mut reaped, ); + adjudicate_boundary( + &log, + &tx_state, + Some(&*rumors), + &own_baseboard, + boundary.as_ref(), + &survivors, + &mut adjudicated, + ); } } }), @@ -1578,3 +2094,103 @@ pub fn cert_chain(certs: &Certificates, key_id: &KeyId) -> Result Network { + serde_json::from_str(&format!("[{seed:?}{}]", ", 0".repeat(15))).unwrap() + } + + fn past(committed: Option, floor: Option) -> Past { + let log = Logger::root(slog::Discard, o!()); + Past::new( + Version::new(), + Version::new(), + Arc::new(BoundaryStore::new(&log, &Locker::null())), + committed, + floor, + ) + } + + /// The admission rules: a sled with no record admits, a record + /// from a foreign universe admits, the committed session admits + /// outright, a session started strictly above the executed join + /// admits, and everything else hops. + #[test] + fn admission_rules() { + let started: Version = "(1, 1, (0, 0, 2))".parse().unwrap(); + let older: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); + let newer: Version = "(2, 1, (0, 0, 3))".parse().unwrap(); + let concurrent: Version = "(1, 2, (0, 0, 1))".parse().unwrap(); + let session = SessionId::random(); + let job = JobId::random(); + let committed = Boundary { + network: network(1), + burned: Bloom::new(), + executed: started.clone(), + job: Some(Committed { + session, + job, + successor: JobId::random(), + outcome: JobOutcome::Committed, + }), + }; + + let recordless = past(None, None); + assert!(matches!( + recordless.screen(network(1), session, &started), + Admission::Admit + )); + + let past = past(Some(committed), None); + assert!(matches!( + past.screen(network(2), session, &started), + Admission::Admit + )); + // The committed session admits outright: it resumes at the + // stored successor when it activates, and chain position + // keeps every earlier job from popping. + assert!(matches!( + past.screen(network(1), session, &started), + Admission::Admit + )); + assert!(matches!( + past.screen(network(1), SessionId::random(), &newer), + Admission::Admit + )); + for unordered in [&older, &started, &concurrent] { + assert!(matches!( + past.screen(network(1), SessionId::random(), unordered), + Admission::Hop + )); + } + } + + /// A floor refuses every session not started strictly above it, + /// in every universe: the floor belongs to this life, not to any + /// record. + #[test] + fn floors_refuse_below() { + let floor: Version = "(1, 1, (0, 0, 2))".parse().unwrap(); + let at: Version = floor.clone(); + let below: Version = "(1, 0, (0, 0, 2))".parse().unwrap(); + let above: Version = "(2, 1, (0, 0, 3))".parse().unwrap(); + let concurrent: Version = "(1, 2, (0, 0, 1))".parse().unwrap(); + let past = past(None, Some(floor.clone())); + + for started in [&at, &below, &concurrent] { + for net in [network(1), network(2)] { + assert!(matches!( + past.screen(net, SessionId::random(), started), + Admission::Refuse + )); + } + } + assert!(matches!( + past.screen(network(1), SessionId::random(), &above), + Admission::Admit + )); + } +} diff --git a/server/tests/distributed.rs b/server/tests/distributed.rs index 9b54af2..27c0bc1 100644 --- a/server/tests/distributed.rs +++ b/server/tests/distributed.rs @@ -7,26 +7,33 @@ mod common; use std::collections::BTreeSet; +use std::fs::{read, read_to_string, write}; use std::net::SocketAddrV6; +use std::slice::from_ref; +use std::time::Duration; use camino::Utf8PathBuf; +use function_name::named; use sled_hardware_types::BaseboardId; use slog::Logger; use tempfile::TempDir; use tokio::sync::watch; +use tokio::time::sleep; use tokio_util::sync::CancellationToken; use chrono::Utc; use sush_api::{JobStartParams, JobWait}; use sush_common::jobs::{ - JobId, JobOutputState, JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, + JobId, JobMode, JobOutputState, JobStartRequest, JobStatus, ProcessError, Session, SessionId, + SessionSignerNonce, SignedJob, SkipReason, }; -use sush_common::keys::pem_cert_chain; -use sush_common::targets::{Cubbies, SledHealth}; +use sush_common::keys::{EphemeralKey, Signer as _, pem_cert_chain}; +use sush_common::targets::{Cubbies, SledHealth, SledId, Target}; use sush_common::version::VersionInfo; -use sush_server::bookmark::BookmarkSource; +use sush_server::bookmark::BOOKMARK; use sush_server::executor::PathIsolation; use sush_server::gossip::spawn_gossip; +use sush_server::locker::Locker; use sush_server::messages::v0::{Event, JobEvent, Message}; use sush_server::output::JobOutputDir; use sush_server::state::GossipUniverse; @@ -54,25 +61,18 @@ impl Sled { root_pem: &Utf8PathBuf, shutdown: &CancellationToken, ) -> Sled { - Self::start_with_bookmarks( - log, - dir, - identity, - root_pem, - BookmarkSource::null(), - shutdown, - ) - .await + Self::start_with_locker(log, dir, identity, root_pem, Locker::null(), shutdown).await } - async fn start_with_bookmarks( + async fn start_with_locker( log: &Logger, dir: &Utf8PathBuf, identity: usize, root_pem: &Utf8PathBuf, - bookmarks: BookmarkSource, + locker: Locker, shutdown: &CancellationToken, ) -> Sled { + let seed = seed_gossip(log, &locker).await; let (peers, peers_rx) = watch::channel(BTreeSet::new()); let (addr, universe, linked) = spawn_gossip( log, @@ -81,8 +81,7 @@ impl Sled { corpus(dir), localhost(), peers_rx, - seed_gossip(&bookmarks).await, - bookmarks, + seed, shutdown.clone(), ) .await @@ -100,7 +99,8 @@ impl Sled { cubbies, universe.clone(), linked, - std::slice::from_ref(root_pem), + &locker, + from_ref(root_pem), shutdown.clone(), ) .await @@ -116,18 +116,19 @@ impl Sled { } } +#[named] #[tokio::test] async fn jobs_gossip_between_sleds() { let (_tmp, dir) = pki("sush-distributed-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("jobs_gossip_between_sleds"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let b = Sled::start(&log, &dir, 2, &root_pem, &shutdown).await; @@ -227,18 +228,19 @@ async fn jobs_gossip_between_sleds() { shutdown.cancel(); } +#[named] #[tokio::test] async fn rejoining_replays_without_reexecuting() { let (_tmp, dir) = pki("sush-replay-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("rejoining_replays_without_reexecuting"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // Sled A runs a whole job before B exists. @@ -293,6 +295,7 @@ async fn rejoining_replays_without_reexecuting() { // Live traffic still executes everywhere: a fresh session's job runs // on both sleds. + sees(&a, &b).await; let successor_nonce = SessionSignerNonce::random(); let successor = SessionId::compute( a.mgr.own_baseboard(), @@ -342,18 +345,19 @@ async fn rejoining_replays_without_reexecuting() { shutdown.cancel(); } +#[named] #[tokio::test] async fn interrupted_jobs_get_stopped() { let (_tmp, dir) = pki("sush-interrupted-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("interrupted_jobs_get_stopped"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; @@ -421,18 +425,19 @@ async fn interrupted_jobs_get_stopped() { shutdown.cancel(); } +#[named] #[tokio::test] async fn stragglers_do_not_interrupt_live_jobs() { let (_tmp, dir) = pki("sush-straggler-", 3); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("stragglers_do_not_interrupt_live_jobs"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // A and C converge; C then holds a message A never sees. @@ -476,6 +481,7 @@ async fn stragglers_do_not_interrupt_live_jobs() { a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() }) .await; + sees(&a, &b).await; let authn_a = fake_identity(&mut root).await; let signer_nonce = SessionSignerNonce::random(); let session_id = SessionId::compute( @@ -530,18 +536,734 @@ async fn stragglers_do_not_interrupt_live_jobs() { shutdown.cancel(); } +/// Wait until `anchor` has applied `joiner`'s build announcement. A +/// session started on `anchor` afterward causally follows everything +/// `joiner` held at entry, so it clears the joiner's entry floor. +async fn sees(anchor: &Sled, joiner: &Sled) { + eventually("the anchor sees the joiner", 60, async || { + anchor + .mgr + .versions() + .iter() + .any(|row| row.baseboard == joiner.baseboard) + }) + .await; +} + +/// Sign a job aimed at one sled, so a retry cannot legitimately run +/// anywhere else. +async fn sign_job_for( + root: &mut EphemeralKey, + job_id: JobId, + session_id: SessionId, + command: &str, + sled: &BaseboardId, +) -> SignedJob { + root.sign(JobStartRequest::new( + job_id, + session_id, + command, + JobMode::Batch, + Target::Sleds(vec![SledId::Baseboard(sled.clone())]), + )) + .await + .unwrap() +} + +#[named] +#[tokio::test] +async fn lost_suffix_never_reruns() { + let (_tmp, dir) = pki("sush-lost-", 2); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + // Sled A anchors the session and survives throughout. + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + // Sled B keeps its boundary in a locker. It joins before the + // session starts: a record-less sled raises its floor at entry, + // and serves only sessions started after it arrived. + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + sees(&a, &b).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); + eventually("the session gossips to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_id) + }) + .await; + a.peers.send(BTreeSet::new()).unwrap(); + b.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + + // Two jobs run on B through its front door. No one else hears of + // them. The first leaves a footprint we can count. + let footprint = boundary_dir.path().join("footprint"); + let j1_id = session.next_job_id(); + let j1 = sign_job_for( + &mut root, + j1_id, + session_id, + &format!("echo run >> {}", footprint.display()), + &b.baseboard, + ) + .await; + b.mgr + .job_start( + &authn_b, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j1.clone()); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j2, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); + + // B dies with both jobs unwitnessed and rejoins. Its boundary + // proves the rack is missing part of the session's history. + b_shutdown.cancel(); + drop(b); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + + // The recorded job finished before the crash, so B adjudicates + // its true ending on its sole authority. + eventually("the boundary job is adjudicated", 120, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // A retry of the first job's preserved artifact stays queued + // instead of running: the stored successor resumes the chain past + // it, so its position never pops, and the footprint file still + // shows one run. + b.mgr + .job_start(&authn_b, j1, JobStartParams::default()) + .await + .unwrap(); + eventually("the retry queues", 60, async || { + b.mgr.job_status(&authn_b, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Queued { .. })) + }) + }) + .await; + sleep(Duration::from_secs(1)).await; + assert!( + b.mgr.job_status(&authn_b, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Queued { .. })) + }), + "a job below the stored successor must stay queued" + ); + assert_eq!(read_to_string(&footprint).unwrap(), "run\n"); + + // A new session serves immediately. + let signer_nonce = SessionSignerNonce::random(); + let session2_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session2 = Session::new(session2_id); + a.mgr + .session_start(&authn_a, session2_id, signer_nonce, true) + .await + .unwrap(); + eventually("the new session gossips to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j3_id = session2.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session2_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j3, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + b.mgr.job_status(&authn_b, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "a new session must not be held" + ); + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn session_resumes_at_stored_successor() { + let (_tmp, dir) = pki("sush-resume-", 3); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + // Three sleds converge, then the session starts, so everyone + // serves it. B keeps its boundary in a locker. + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + let c = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; + let authn_c = fake_identity(&mut root).await; + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + let authn_b = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([b.addr, c.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + c.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + let network = a.universe.borrow().rumors.network(); + b.universe.borrow().rumors.network() == network + && c.universe.borrow().rumors.network() == network + }) + .await; + sees(&a, &b).await; + sees(&a, &c).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); + eventually("the session gossips to B and C", 120, async || { + [(&b.mgr, &authn_b), (&c.mgr, &authn_c)] + .iter() + .all(|(mgr, authn)| { + mgr.session(authn) + .is_some_and(|s| s.session_id() == session_id) + }) + }) + .await; + + // C falls behind: it hears nothing of what follows. + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + c.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + + // B runs two jobs submitted through its own API; only A + // witnesses them. + let j1_id = session.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j1); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j2.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session.job_started(j2); + eventually("A witnesses the runs", 60, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // B dies and rejoins through lagging C alone. C holds none of the + // jobs, but B's record stores the successor of its last + // commitment, so the session resumes there with no witness at + // all: the next job runs immediately. + b_shutdown.cancel(); + drop(b); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_b = fake_identity(&mut root).await; + b.peers.send(BTreeSet::from([c.addr])).unwrap(); + c.peers.send(BTreeSet::from([b.addr])).unwrap(); + eventually("the session replays to B", 120, async || { + b.mgr + .session(&authn_b) + .is_some_and(|s| s.session_id() == session_id) + }) + .await; + let j3_id = session.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session_id, "true", &b.baseboard).await; + b.mgr + .job_start( + &authn_b, + j3, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + b.mgr.job_status(&authn_b, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "the resumed session must serve its next job without a witness" + ); + + // A returns, and the resumed chain reconciles rack-wide. + a.peers.send(BTreeSet::from([b.addr, c.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr, c.addr])).unwrap(); + c.peers.send(BTreeSet::from([a.addr, b.addr])).unwrap(); + eventually("the resumed run reaches A", 120, async || { + a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn universe_flip_flop_raises_floor() { + let (_tmp, dir) = pki("sush-flipflop-", 3); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + // Two universes that never meet: A anchors one session, D another. + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + let d = Sled::start(&log, &dir, 3, &root_pem, &shutdown).await; + let authn_d = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session2_id = SessionId::compute( + d.mgr.own_baseboard(), + d.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session2 = Session::new(session2_id); + d.mgr + .session_start(&authn_d, session2_id, signer_nonce, true) + .await + .unwrap(); + + // X joins A's universe and runs a job there. + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == x.universe.borrow().rumors.network() + }) + .await; + sees(&a, &x).await; + let signer_nonce = SessionSignerNonce::random(); + let session1_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session1 = Session::new(session1_id); + a.mgr + .session_start(&authn_a, session1_id, signer_nonce, true) + .await + .unwrap(); + eventually("session one gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session1_id) + }) + .await; + let j1_id = session1.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session1_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j1.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session1.job_started(j1); + // A must witness the job: X's replayed chain in its third life + // can only be rebuilt from what A holds. + eventually("A witnesses the first job", 60, async || { + a.mgr.job_status(&authn_a, &j1_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // X dies and rejoins D's universe instead, running a job there. + // That commit burns A's universe out of X's record. + x_shutdown.cancel(); + drop(x); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + d.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([d.addr])).unwrap(); + eventually("session two gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j2_id = session2.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session2_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j2.clone(), + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + session2.job_started(j2); + // D must witness the job: X's replayed chain in its fourth life + // can only be rebuilt from what D holds. + eventually("D witnesses the second job", 60, async || { + d.mgr.job_status(&authn_d, &j2_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }) + }) + .await; + + // X dies again and flip-flops back to A's universe. Its record + // burned that universe, so X raises its floor: session one's next + // job is refused, and nothing re-runs. + x_shutdown.cancel(); + drop(x); + d.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let x_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &x_shutdown).await; + let authn_x = fake_identity(&mut root).await; + a.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("session one replays to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session1_id) + }) + .await; + let j3_id = session1.next_job_id(); + let j3 = sign_job_for(&mut root, j3_id, session1_id, "true", &x.baseboard).await; + x.mgr + .job_start(&authn_x, j3, JobStartParams::default()) + .await + .unwrap(); + eventually("the floor skips the old session", 120, async || { + a.mgr.job_status(&authn_a, &j3_id).await.is_ok_and(|map| { + map.get(&x.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Skipped { + reason: SkipReason::BelowFloor, + .. + } + ) + }) + }) + }) + .await; + + // A witnessed the refusal, so a session started now begins above + // X's floor, and serves X again. + let signer_nonce = SessionSignerNonce::random(); + let session3_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session3 = Session::new(session3_id); + a.mgr + .session_start(&authn_a, session3_id, signer_nonce, true) + .await + .unwrap(); + eventually("session three gossips to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session3_id) + }) + .await; + let j4_id = session3.next_job_id(); + let j4 = sign_job_for(&mut root, j4_id, session3_id, "true", &x.baseboard).await; + x.mgr + .job_start( + &authn_x, + j4, + JobStartParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + x.mgr.job_status(&authn_x, &j4_id).await.is_ok_and(|map| { + map.get(&x.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { result: Ok(0), .. })) + }), + "a session started above the floor must serve" + ); + + // X flops back to D's universe a second time. The job X served in + // A's universe displaced D's watermark, so that write must have + // burned D: session two's next job is skipped there, never re-run. + x_shutdown.cancel(); + drop(x); + a.peers.send(BTreeSet::new()).unwrap(); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let x = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + let authn_x = fake_identity(&mut root).await; + d.peers.send(BTreeSet::from([x.addr])).unwrap(); + x.peers.send(BTreeSet::from([d.addr])).unwrap(); + eventually("session two replays to X", 120, async || { + x.mgr + .session(&authn_x) + .is_some_and(|s| s.session_id() == session2_id) + }) + .await; + let j5_id = session2.next_job_id(); + let j5 = sign_job_for(&mut root, j5_id, session2_id, "true", &x.baseboard).await; + x.mgr + .job_start(&authn_x, j5, JobStartParams::default()) + .await + .unwrap(); + eventually("the floor skips session two as well", 120, async || { + x.mgr.job_status(&authn_x, &j5_id).await.is_ok_and(|map| { + map.get(&x.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Skipped { + reason: SkipReason::BelowFloor, + .. + } + ) + }) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] +#[tokio::test] +async fn witnessed_session_survives_restart() { + let (_tmp, dir) = pki("sush-witness-", 2); + let mut root = common::ephemeral_root(); + let root_pem = dir.join("job-root.pem"); + write( + &root_pem, + pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), + ) + .unwrap(); + + let log = test_logger(function_name!()); + let shutdown = CancellationToken::new(); + + let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; + let authn_a = fake_identity(&mut root).await; + + let boundary_dir = TempDir::with_prefix("sush-boundary-").unwrap(); + let slot = Utf8PathBuf::from_path_buf(boundary_dir.path().to_path_buf()).unwrap(); + let b_shutdown = CancellationToken::new(); + let locker = Locker::new(&log, vec![slot.clone()]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &b_shutdown).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe convergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + sees(&a, &b).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let mut session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); + + // A job runs on B and its request is witnessed by A, so replay + // reaches B's boundary when it returns. The job must be live + // traffic on B: a replayed job never executes. + let j1_id = session.next_job_id(); + let j1 = sign_job_for(&mut root, j1_id, session_id, "true", &b.baseboard).await; + a.mgr + .job_start(&authn_a, j1.clone(), JobStartParams::default()) + .await + .unwrap(); + eventually("B's result reaches A", 120, async || { + a.mgr.job_status(&authn_a, &j1_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) + }) + }) + .await; + + // B restarts. The session's history is intact, so the session + // keeps working on B. + b_shutdown.cancel(); + drop(b); + sleep(Duration::from_millis(500)).await; + let locker = Locker::new(&log, vec![slot]); + let b = Sled::start_with_locker(&log, &dir, 2, &root_pem, locker, &shutdown).await; + a.peers.send(BTreeSet::from([b.addr])).unwrap(); + b.peers.send(BTreeSet::from([a.addr])).unwrap(); + eventually("universe reconvergence", 120, async || { + a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() + }) + .await; + + session.job_started(j1); + let j2_id = session.next_job_id(); + let j2 = sign_job_for(&mut root, j2_id, session_id, "true", &b.baseboard).await; + a.mgr + .job_start(&authn_a, j2, JobStartParams::default()) + .await + .unwrap(); + eventually("the session's next job runs on B", 120, async || { + a.mgr.job_status(&authn_a, &j2_id).await.is_ok_and(|map| { + map.get(&b.baseboard) + .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) + }) + }) + .await; + + shutdown.cancel(); +} + +#[named] #[tokio::test] async fn bookmarks_survive_restart() { let (_tmp, dir) = pki("sush-bookmark-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("bookmarks_survive_restart"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); // Sled A holds session history, so it wins every dominance contest. @@ -560,24 +1282,25 @@ async fn bookmarks_survive_restart() { // Sled 2 keeps its identity in a bookmark; joining records it. let bookmark_dir = TempDir::with_prefix("sush-bookmark-").unwrap(); - let slot = Utf8PathBuf::from_path_buf(bookmark_dir.path().join("bookmark")).unwrap(); + let slot = Utf8PathBuf::from_path_buf(bookmark_dir.path().to_path_buf()).unwrap(); + let record = slot.join(BOOKMARK.file); let b_shutdown = CancellationToken::new(); - let b = Sled::start_with_bookmarks( + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![slot.clone()]), + Locker::new(&log, vec![slot.clone()]), &b_shutdown, ) .await; a.peers.send(BTreeSet::from([b.addr])).unwrap(); b.peers.send(BTreeSet::from([a.addr])).unwrap(); eventually("the joining sled records its identity", 120, async || { - slot.as_std_path().exists() + record.as_std_path().exists() }) .await; - let before = std::fs::read(&slot).unwrap(); + let before = read(&record).unwrap(); // The next incarnation reads the record back, rejoins, and // advances it, reclaiming the previous life's identity. @@ -587,13 +1310,13 @@ async fn bookmarks_survive_restart() { // Let the dead incarnation's tasks quiesce, as a real reboot // would. Two live sources over one slot are the store's one // forbidden misuse. - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - let b = Sled::start_with_bookmarks( + sleep(Duration::from_millis(500)).await; + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![slot.clone()]), + Locker::new(&log, vec![slot.clone()]), &shutdown, ) .await; @@ -606,50 +1329,40 @@ async fn bookmarks_survive_restart() { eventually( "the record advances past the previous life", 120, - async || std::fs::read(&slot).unwrap() != before, + async || read(&record).unwrap() != before, ) .await; shutdown.cancel(); } +#[named] #[tokio::test] async fn gossip_survives_bookmark_failure() { let (_tmp, dir) = pki("sush-nobookmark-", 2); let mut root = common::ephemeral_root(); let root_pem = dir.join("job-root.pem"); - std::fs::write( + write( &root_pem, pem_cert_chain(vec![root.cert().to_owned()]).unwrap(), ) .unwrap(); - let log = test_logger("gossip_survives_bookmark_failure"); + let log = test_logger(function_name!()); let shutdown = CancellationToken::new(); let a = Sled::start(&log, &dir, 1, &root_pem, &shutdown).await; let authn_a = fake_identity(&mut root).await; - let signer_nonce = SessionSignerNonce::random(); - let session_id = SessionId::compute( - a.mgr.own_baseboard(), - a.mgr.session_sush_nonce(), - signer_nonce, - ); - let session = Session::new(session_id); - a.mgr - .session_start(&authn_a, session_id, signer_nonce, true) - .await - .unwrap(); // Sled 2's bookmark points into a directory that does not exist. // It sheds the bookmark and gossips anyway, stranding identities // rather than the rack. - let b = Sled::start_with_bookmarks( + let b = Sled::start_with_locker( &log, &dir, 2, &root_pem, - BookmarkSource::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush/bookmark")]), + Locker::new(&log, vec![Utf8PathBuf::from("/nonexistent/sush")]), &shutdown, ) .await; @@ -659,8 +1372,22 @@ async fn gossip_survives_bookmark_failure() { a.universe.borrow().rumors.network() == b.universe.borrow().rumors.network() }) .await; + sees(&a, &b).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = SessionId::compute( + a.mgr.own_baseboard(), + a.mgr.session_sush_nonce(), + signer_nonce, + ); + let session = Session::new(session_id); + a.mgr + .session_start(&authn_a, session_id, signer_nonce, true) + .await + .unwrap(); - // Live jobs still run on the degraded sled. + // The degraded sled refuses live jobs rather than run one it + // cannot record, and gossips the refusal. The healthy sled still + // runs it. let job_id = session.next_job_id(); let job = sign_job(&mut root, job_id, session_id, "true").await; a.mgr @@ -674,9 +1401,23 @@ async fn gossip_survives_bookmark_failure() { ) .await .unwrap(); - eventually("the job runs on the degraded sled", 120, async || { + eventually("the degraded sled refuses the job", 120, async || { a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { - map.get(&b.baseboard) + map.get(&b.baseboard).is_some_and(|s| { + matches!( + s, + JobStatus::Error { + error: ProcessError::Io { .. }, + .. + } + ) + }) + }) + }) + .await; + eventually("the healthy sled runs the job", 120, async || { + a.mgr.job_status(&authn_a, &job_id).await.is_ok_and(|map| { + map.get(&a.baseboard) .is_some_and(|s| matches!(s, JobStatus::Stopped { .. })) }) }) diff --git a/server/tests/gossip.rs b/server/tests/gossip.rs index 77784a2..e9ffe2b 100644 --- a/server/tests/gossip.rs +++ b/server/tests/gossip.rs @@ -11,14 +11,16 @@ use std::collections::BTreeSet; use std::net::SocketAddrV6; use camino::Utf8PathBuf; -use rumors::{Network, Peer, Rumors}; +use function_name::named; +use rumors::{Network, Rumors}; use slog::Logger; use tokio::sync::watch; use tokio_util::sync::CancellationToken; use sush_common::jobs::BaseboardId; -use sush_server::bookmark::{BookmarkSource, SushBookmark}; -use sush_server::gossip::{LinkedBaseboards, Universe, spawn_gossip}; +use sush_server::bookmark::SushBookmark; +use sush_server::gossip::{LinkedBaseboards, Seed, Universe, spawn_gossip}; +use sush_server::locker::Locker; use common::{ baseboard, corpus, eventually, gossip_config, localhost, pki, sprockets_config, test_logger, @@ -36,13 +38,8 @@ struct Node { impl Node { async fn start(log: &Logger, dir: &Utf8PathBuf, identity: usize) -> Node { let shutdown = CancellationToken::new(); - let bookmarks = BookmarkSource::null(); - let seed: Rumors = Peer::seed() - .bookmark(bookmarks.next_handle()) - .await - .expect("a pristine seed never touches its bookmark") - .into_rumors(); - let initial = seed.network(); + let seed: Seed = Seed::grow(log, &Locker::null()).await; + let initial = seed.rumors().network(); let (peers, peers_rx) = watch::channel(BTreeSet::new()); let (addr, universe, linked) = spawn_gossip( log, @@ -52,7 +49,6 @@ impl Node { localhost(), peers_rx, seed, - bookmarks, shutdown.clone(), ) .await @@ -109,10 +105,11 @@ fn converged(nodes: &[&Node]) -> Option { nodes.iter().all(|n| n.network() == first).then_some(first) } +#[named] #[tokio::test] async fn cold_start_converges() { let (_tmp, dir) = pki("sush-gossip-", 3); - let log = test_logger("cold_start_converges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; let c = Node::start(&log, &dir, 3).await; @@ -132,10 +129,11 @@ async fn cold_start_converges() { .await; } +#[named] #[tokio::test] async fn staggered_start_converges() { let (_tmp, dir) = pki("sush-gossip-", 3); - let log = test_logger("staggered_start_converges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; mesh(&[&a, &b]); @@ -160,10 +158,11 @@ async fn staggered_start_converges() { .await; } +#[named] #[tokio::test] async fn node_replacement_reconverges() { let (_tmp, dir) = pki("sush-gossip-", 4); - let log = test_logger("node_replacement_reconverges"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; let c = Node::start(&log, &dir, 3).await; @@ -189,10 +188,11 @@ async fn node_replacement_reconverges() { .await; } +#[named] #[tokio::test] async fn linked_follows_live_links() { let (_tmp, dir) = pki("sush-gossip-", 2); - let log = test_logger("linked_follows_live_links"); + let log = test_logger(function_name!()); let a = Node::start(&log, &dir, 1).await; let b = Node::start(&log, &dir, 2).await; assert!(a.linked().is_empty()); diff --git a/server/tests/output/boundary-record-v0.bin b/server/tests/output/boundary-record-v0.bin new file mode 100644 index 0000000..8915286 Binary files /dev/null and b/server/tests/output/boundary-record-v0.bin differ diff --git a/server/tests/output/job-skipped-event.bin b/server/tests/output/job-skipped-event.bin new file mode 100644 index 0000000..074aa94 Binary files /dev/null and b/server/tests/output/job-skipped-event.bin differ diff --git a/server/tests/output/session-ended-skip-event.bin b/server/tests/output/session-ended-skip-event.bin new file mode 100644 index 0000000..505c789 Binary files /dev/null and b/server/tests/output/session-ended-skip-event.bin differ diff --git a/server/tests/output/session-hop-error.bin b/server/tests/output/session-hop-error.bin new file mode 100644 index 0000000..e872b2a --- /dev/null +++ b/server/tests/output/session-hop-error.bin @@ -0,0 +1 @@ +bV0eEventkpart_numberk913-0000019mserial_numberkBRM42220030eErrorjSessionHop \ No newline at end of file diff --git a/server/tests/output/universe-flip-flop-error.bin b/server/tests/output/universe-flip-flop-error.bin new file mode 100644 index 0000000..2b75214 --- /dev/null +++ b/server/tests/output/universe-flip-flop-error.bin @@ -0,0 +1 @@ +bV0eEventkpart_numberk913-0000019mserial_numberkBRM42220030eErrorpUniverseFlipFlop \ No newline at end of file diff --git a/sush.json b/sush.json index cb04437..870c37f 100644 --- a/sush.json +++ b/sush.json @@ -1455,6 +1455,36 @@ "Stopped" ], "additionalProperties": false + }, + { + "description": "The reporting sled decided it will never run this job. A skip is a decision, not a failure: the job may have run on other sleds, and the operator decides whether to resubmit.", + "type": "object", + "properties": { + "Skipped": { + "type": "object", + "properties": { + "job_id": { + "$ref": "#/components/schemas/JobId" + }, + "reason": { + "$ref": "#/components/schemas/SkipReason" + }, + "time_skipped": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "job_id", + "reason", + "time_skipped" + ] + } + }, + "required": [ + "Skipped" + ], + "additionalProperties": false } ] }, @@ -1493,6 +1523,18 @@ "Burned" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "Resumed": { + "$ref": "#/components/schemas/JobId" + } + }, + "required": [ + "Resumed" + ], + "additionalProperties": false } ] }, @@ -1717,6 +1759,31 @@ "signature" ] }, + "SkipReason": { + "description": "Why a sled will never run a job.", + "oneOf": [ + { + "type": "string", + "enum": [ + "session-ended" + ] + }, + { + "description": "The job's session sits at or below the sled's execution floor, where the sled cannot tell replay from re-run.", + "type": "string", + "enum": [ + "below-floor" + ] + }, + { + "description": "The job's chain position precedes the sled's recorded commitment: a previous life already handled it.", + "type": "string", + "enum": [ + "already-handled" + ] + } + ] + }, "SledHealth": { "description": "One sled's gossip link health, as the answering sled sees it. A silent death can lag `Linked` at TCP's pace.", "oneOf": [ diff --git a/tests/src/manager_tests.rs b/tests/src/manager_tests.rs index 7dbfa4e..4961c23 100644 --- a/tests/src/manager_tests.rs +++ b/tests/src/manager_tests.rs @@ -23,6 +23,7 @@ use tokio::fs::{metadata, read, write}; use tokio::sync::watch; use tokio::time::{sleep, timeout}; use tokio_util::sync::CancellationToken; +use x509_cert::der::Encode as _; use x509_cert::time::Validity; use sush_api::{JobStartParams, JobStopParams, JobWait}; @@ -30,21 +31,21 @@ use sush_client::context::Authz; use sush_common::authn::{Challenge, ChallengeResponse, Credentials, Identity, Nonce, RequestKey}; use sush_common::jobs::{ Access, JobId, JobLimits, JobMode, JobOutputState, JobOutputStream::*, JobStartRequest, - JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, SignedJob, + JobStatus, ProcessError, Session, SessionId, SessionSignerNonce, SignedJob, SkipReason, }; use sush_common::keys::{EphemeralKey, KeyError, KeyId, KeyType, Signer as _, pem_cert_chain}; use sush_common::targets::{Cubbies, Target}; -use sush_server::bookmark::BookmarkSource; use sush_server::gossip::{Universe, isolated, lonely}; use sush_server::io::BATCH_OUTPUT_BUFFER_SIZE; +use sush_server::locker::Locker; use sush_server::messages::v0::{CertRequest, IdentityRequest, Message, Request, SessionRequest}; use sush_server::output::{JobOutputDir, OutputDirs}; -use sush_server::{JobError, JobManager, seed_gossip}; +use sush_server::{JobError, JobManager}; use crate::test_utils::{ IntoBytes as _, SignJobRequest as _, ephemeral_test_root, ephemeral_test_subject, fake_identity, manager_and_test_root, manager_login, manager_test_root_and_peer, no_cubbies, - test_baseboard_id, test_logger, + null_gossip, test_baseboard_id, test_logger, }; use sush_server::executor::PathIsolation; @@ -424,6 +425,33 @@ async fn job_stop() { ); } +/// Queue a job behind a hole in the job chain: the executor only runs +/// the job whose id the chain expects next, and it never sees the +/// hole's, so the queued job cannot start. +async fn queue_job_behind_hole( + mgr: &JobManager, + root: &mut EphemeralKey, + authn: &Identity, + session: &mut Session, +) -> JobId { + let session_id = session.session_id(); + let hole_id = session.next_job_id(); + let hole = root + .sign_job_request(hole_id, session_id, "true", false) + .await; + session.job_started(hole.into_signed()); + let job_id = session.next_job_id(); + let job = root + .sign_job_request(job_id, session_id, "false", false) + .await; + mgr.job_start(authn, job.clone().into_signed(), JobStartParams::default()) + .await + .expect("should be able to queue the job"); + session.job_started(job.into_signed()); + mgr.wait_for_job_status(&job_id).await.unwrap(); + job_id +} + #[named] #[tokio::test] async fn cancel_queued_job() { @@ -456,28 +484,7 @@ async fn cancel_queued_job() { .expect("should be able to start job A"); session.job_started(job_a.into_signed()); - // Queue job B behind a hole in the job chain, so it cannot start - // before we cancel it: the executor only runs the job whose id the - // chain expects next, and it never sees this one. - let hole_id = session.next_job_id(); - let hole = root - .sign_job_request(hole_id, session_id, "true", false) - .await; - session.job_started(hole.into_signed()); - let command_b = "false"; - let job_id_b = session.next_job_id(); - let job_b = root - .sign_job_request(job_id_b, session_id, command_b, false) - .await; - mgr.job_start( - &authn, - job_b.clone().into_signed(), - JobStartParams::default(), - ) - .await - .expect("should be able to queue job B"); - session.job_started(job_b.into_signed()); - mgr.wait_for_job_status(&job_id_b).await.unwrap(); + let job_id_b = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; assert!(matches!( &mgr.job_status(&authn, &job_id_b).await.unwrap()[mgr.own_baseboard()], JobStatus::Queued { job_id: jid, time_queued, .. } if *jid == job_id_b && *time_queued <= Utc::now() @@ -512,6 +519,109 @@ async fn cancel_queued_job() { .expect("should be able to stop job A"); } +async fn wait_for_session_ended_skip(mgr: &JobManager, authn: &Identity, job_id: &JobId) { + timeout(Duration::from_secs(30), async { + loop { + if let Ok(map) = mgr.job_status(authn, job_id).await + && matches!( + map.get(mgr.own_baseboard()), + Some(JobStatus::Skipped { + reason: SkipReason::SessionEnded, + .. + }) + ) + { + break; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("job skipped for ended session"); +} + +#[named] +#[tokio::test] +async fn session_stop_skips_queued_jobs() { + let log = test_logger(function_name!()); + let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; + let authn = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = + SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let mut session = Session::new(session_id); + mgr.session_start(&authn, session_id, signer_nonce, true) + .await + .unwrap(); + + let job_id_a = session.next_job_id(); + let job_a = root + .sign_job_request(job_id_a, session_id, "sleep 10", false) + .await; + mgr.job_start( + &authn, + job_a.clone().into_signed(), + JobStartParams { + wait: JobWait::Start, + ..Default::default() + }, + ) + .await + .expect("should be able to start job A"); + session.job_started(job_a.into_signed()); + + let job_id_b = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; + + mgr.session_stop(&authn, session_id) + .await + .expect("should be able to stop the session"); + wait_for_session_ended_skip(&mgr, &authn, &job_id_b).await; + + assert!(matches!( + &mgr.job_status(&authn, &job_id_a).await.unwrap()[mgr.own_baseboard()], + JobStatus::Started { .. } + )); + + mgr.job_stop( + &authn, + &job_id_a, + JobStopParams { + wait: JobWait::Stop, + ..Default::default() + }, + ) + .await + .expect("should be able to stop job A"); +} + +#[named] +#[tokio::test] +async fn superseding_session_skips_queued_jobs() { + let log = test_logger(function_name!()); + let (mgr, mut root, _dir, _shutdown) = manager_and_test_root(log).await; + let authn = fake_identity(&mut root).await; + let signer_nonce = SessionSignerNonce::random(); + let session_id = + SessionId::compute(mgr.own_baseboard(), mgr.session_sush_nonce(), signer_nonce); + let mut session = Session::new(session_id); + mgr.session_start(&authn, session_id, signer_nonce, true) + .await + .unwrap(); + + let job_id = queue_job_behind_hole(&mgr, &mut root, &authn, &mut session).await; + + let new_signer_nonce = SessionSignerNonce::random(); + let new_session_id = SessionId::compute( + mgr.own_baseboard(), + mgr.session_sush_nonce(), + new_signer_nonce, + ); + mgr.session_start(&authn, new_session_id, new_signer_nonce, true) + .await + .expect("should be able to start a superseding session"); + wait_for_session_ended_skip(&mgr, &authn, &job_id).await; +} + #[named] #[tokio::test] async fn job_output_perms() { @@ -579,8 +689,9 @@ async fn cubby_targets() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), cubbies_rx, - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -685,8 +796,9 @@ async fn root_certs_from_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[path], CancellationToken::new(), ) @@ -736,8 +848,9 @@ async fn bad_root_cert_files() { JobOutputDir::fixed(dir.path()), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[path], CancellationToken::new(), ) @@ -767,8 +880,9 @@ async fn job_output_dir_moves() { JobOutputDir::new(rx_dirs), test_baseboard_id(), no_cubbies(), - isolated(seed_gossip(&BookmarkSource::null()).await), + isolated(null_gossip().await), lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -856,13 +970,12 @@ async fn job_output_dir_moves() { #[tokio::test] async fn universe_swap() { // A universe migration resets the state machine. Sessions and history - // die with the old universe, and the manager keeps serving. + // die with the old universe, the boundary carries the last committed + // job across, and the manager keeps serving. let log = test_logger(function_name!()); let dir = TempDir::with_prefix("sush-").unwrap(); let mut root = ephemeral_test_root(); - let (universe, universe_rx) = watch::channel(Universe::genesis( - seed_gossip(&BookmarkSource::null()).await, - )); + let (universe, universe_rx) = watch::channel(Universe::genesis(null_gossip().await)); let mgr = JobManager::with_root_certs( log, PathIsolation::InsecureDisable, @@ -871,6 +984,7 @@ async fn universe_swap() { no_cubbies(), universe_rx, lonely(), + &Locker::null(), &[root.cert().to_owned()], CancellationToken::new(), ) @@ -909,9 +1023,7 @@ async fn universe_swap() { // Migrate. The session and the job's history are gone. universe - .send(Universe::genesis( - seed_gossip(&BookmarkSource::null()).await, - )) + .send(Universe::genesis(null_gossip().await)) .unwrap(); timeout(Duration::from_secs(30), async { while mgr.session(&authn).is_some() { @@ -920,6 +1032,30 @@ async fn universe_swap() { }) .await .expect("state reset"); + // The new universe cannot name the job, so the boundary + // adjudicates its recorded ending on this sled's sole authority. + timeout(Duration::from_secs(30), async { + loop { + if let Ok(map) = mgr.job_status(&authn, &job_id).await + && matches!( + map.get(mgr.own_baseboard()), + Some(JobStatus::Stopped { result: Ok(0), .. }) + ) + { + break; + } + sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("boundary adjudication"); + + // A further swap does not re-adjudicate the same boundary: one + // ruling per boundary. + universe + .send(Universe::genesis(null_gossip().await)) + .unwrap(); + sleep(Duration::from_millis(200)).await; assert!(matches!( mgr.job_status(&authn, &job_id).await, Err(JobError::JobNotFound(_)) @@ -1010,7 +1146,7 @@ async fn cert_chain() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let gossip = isolated(seed_gossip(&BookmarkSource::null()).await); + let gossip = isolated(null_gossip().await); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( log, @@ -1020,6 +1156,7 @@ async fn cert_chain() { no_cubbies(), gossip, lonely(), + &Locker::null(), &roots, shutdown, ) @@ -1220,7 +1357,7 @@ async fn revocation_tombstones() { peer.send( Message::Request(Request::cert( authn.key_id.clone(), - CertRequest::Import(doomed.cert().clone()), + CertRequest::Import(doomed.cert().to_der().unwrap()), )) .into(), ); @@ -1418,7 +1555,7 @@ async fn gossiped_identities() { peer.send( Message::Request(Request::identity( root_key_id.clone(), - IdentityRequest::Login(root_pk.clone(), signed_by_liar), + IdentityRequest::Login(root_pk.to_openssh().unwrap(), signed_by_liar), )) .into(), ); @@ -1433,7 +1570,7 @@ async fn gossiped_identities() { peer.send( Message::Request(Request::identity( root_key_id.clone(), - IdentityRequest::Login(root_pk.clone(), signed), + IdentityRequest::Login(root_pk.to_openssh().unwrap(), signed), )) .into(), ); @@ -1784,7 +1921,7 @@ async fn hostile_imports_cannot_displace() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( @@ -1795,6 +1932,7 @@ async fn hostile_imports_cannot_displace() { no_cubbies(), isolated(seed), lonely(), + &Locker::null(), from_ref(&root_cert), shutdown, ) @@ -1816,7 +1954,7 @@ async fn hostile_imports_cannot_displace() { let import = |key: &EphemeralKey| { Message::Request(Request::cert( key.key_id().clone(), - CertRequest::Import(key.cert().clone()), + CertRequest::Import(key.cert().to_der().unwrap()), )) .into() }; @@ -1845,7 +1983,7 @@ async fn hostile_imports_cannot_displace() { peer.send( Message::Request(Request::cert( child.key_id().clone(), - CertRequest::Import(conflict), + CertRequest::Import(conflict.to_der().unwrap()), )) .into(), ); @@ -1930,7 +2068,7 @@ async fn homonym_issuer_resolves_to_true_parent() { part_number: "test part".to_string(), serial_number: "0000".to_string(), }; - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let shutdown = CancellationToken::new(); let mgr = JobManager::with_root_certs( @@ -1941,6 +2079,7 @@ async fn homonym_issuer_resolves_to_true_parent() { no_cubbies(), isolated(seed), lonely(), + &Locker::null(), from_ref(&root_cert), shutdown, ) @@ -1954,7 +2093,7 @@ async fn homonym_issuer_resolves_to_true_parent() { peer.send( Message::Request(Request::cert( homonym.key_id().clone(), - CertRequest::Import(homonym.cert().clone()), + CertRequest::Import(homonym.cert().to_der().unwrap()), )) .into(), ); diff --git a/tests/src/test_utils.rs b/tests/src/test_utils.rs index 54c1bb9..6abf424 100644 --- a/tests/src/test_utils.rs +++ b/tests/src/test_utils.rs @@ -29,9 +29,9 @@ use sush_common::codephrases::Codephrase; use sush_common::jobs::{JobId, JobMode, JobStartRequest, SessionId, VerifiedJob}; use sush_common::keys::{EphemeralKey, KeyType, Signer}; use sush_common::targets::{Cubbies, Target}; -use sush_server::bookmark::BookmarkSource; use sush_server::executor::PathIsolation; use sush_server::gossip::{isolated, lonely}; +use sush_server::locker::Locker; use sush_server::output::{JobOutputDir, JobOutputFileStream}; use sush_server::state::GossipNetwork; use sush_server::{JobError, JobManager, seed_gossip}; @@ -228,7 +228,7 @@ pub async fn manager_test_root_and_peer( CancellationToken, ) { let dir = TempDir::with_prefix("sush-").unwrap(); - let seed = seed_gossip(&BookmarkSource::null()).await; + let seed = null_gossip().await; let peer = seed.clone(); let gossip = isolated(seed); let shutdown = CancellationToken::new(); @@ -241,6 +241,7 @@ pub async fn manager_test_root_and_peer( no_cubbies(), gossip, lonely(), + &Locker::null(), &[root.cert().to_owned()], shutdown.clone(), ) @@ -254,6 +255,12 @@ pub fn no_cubbies() -> watch::Receiver { watch::channel(Cubbies::new()).1 } +/// A seed over storage that persists nothing, silently. +pub async fn null_gossip() -> GossipNetwork { + let log = Logger::root(slog::Discard, slog::o!()); + seed_gossip(&log, &Locker::null()).await.into_rumors() +} + pub async fn authz( client: &Client, response: ResponseValue,