From b03f0072c907aa619332758274e70dfb888d95c7 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 20 Oct 2026 15:13:36 -0400 Subject: [PATCH] feat(backend-kubernetes): launch thread-bound Agent Sandbox sessions Add an opt-in sandbox mode that creates one agents.x-k8s.io/v1beta1 Sandbox per agent, channel, and canonical thread root. A matching running Sandbox is reused; a changed, replaced, or completed one requires explicit recovery instead of a silent empty replacement. The provider requires the identity policy and an absolute Pod deadline, binds the launch Secret to the first Pod UID, configures the harness for owner-only bound thread sessions with activity logging, and supports Codex device account setup. Sandbox Pods use the bare-Pod termination grace, which covers the drain and the stop checkpoint. Because the workspace does not outlive the Pod, deploy refuses Sandbox mode unless BUZZ_CHECKPOINT_BUCKET and BUZZ_CHECKPOINT_PREFIX enable the stop checkpoint, or BUZZ_SANDBOX_CHECKPOINT=disabled opts out. The env guard now accepts the harness's strict owner respond-to mode. Signed-off-by: Tee8z --- Cargo.lock | 1 + crates/buzz-acp/TESTING.md | 14 + crates/buzz-backend-kubernetes/Cargo.toml | 1 + crates/buzz-backend-kubernetes/src/config.rs | 23 + crates/buzz-backend-kubernetes/src/env.rs | 10 +- crates/buzz-backend-kubernetes/src/main.rs | 13 +- crates/buzz-backend-kubernetes/src/sandbox.rs | 532 ++++++++++++++++++ .../src/sandbox_tests.rs | 370 ++++++++++++ .../agents/ui/ProviderConfigFields.test.mjs | 17 + .../agents/ui/ProviderConfigFields.tsx | 5 +- docs/remote-agents.md | 39 ++ 11 files changed, 1021 insertions(+), 4 deletions(-) create mode 100644 crates/buzz-backend-kubernetes/src/sandbox.rs create mode 100644 crates/buzz-backend-kubernetes/src/sandbox_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 89d55e68f2a..a7ede930456 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,6 +1103,7 @@ dependencies = [ "sha2 0.11.0", "tokio", "tower 0.5.3", + "uuid", ] [[package]] diff --git a/crates/buzz-acp/TESTING.md b/crates/buzz-acp/TESTING.md index 6d63ff3adb4..5135d4aa1d1 100644 --- a/crates/buzz-acp/TESTING.md +++ b/crates/buzz-acp/TESTING.md @@ -36,6 +36,20 @@ The image repository's `buzz-agent/test-checkpoint-storage.py` also stops the packaged harness during startup while an adapter writes files. It checks that recovery preserves the final edit and rejected uploads make shutdown fail. +Run the controller lifecycle test against the isolated local kind cluster: + +```sh +BUZZ_SANDBOX_TEST_KUBECONFIG=/absolute/kind.kubeconfig \ +BUZZ_SANDBOX_TEST_IMAGE=registry.example/uid-guard-fixture@sha256: \ + cargo test -p buzz-backend-kubernetes live_controller_reuses_scopes \ + -- --ignored --nocapture +``` + +The test requires context `kind-remote-agent-check`, the Agent Sandbox controller, +and a loaded fixture image that checks the Pod UID binding before sleeping. +It verifies concurrent reuse, separate threads, owner and relay refusals, and +refusal after the original Pod is replaced. It does not enroll real accounts. + Managed agent sessions may already export harness options. Clear them when running the package suite: three CLI parsing tests assert the unset defaults, and inherited values would change the inputs those tests exercise. Running the diff --git a/crates/buzz-backend-kubernetes/Cargo.toml b/crates/buzz-backend-kubernetes/Cargo.toml index bc7854c560c..d8b3d2bd907 100644 --- a/crates/buzz-backend-kubernetes/Cargo.toml +++ b/crates/buzz-backend-kubernetes/Cargo.toml @@ -22,6 +22,7 @@ sha2 = { workspace = true } hex = { workspace = true } rand = { workspace = true } chrono = { workspace = true } +uuid = { workspace = true } http = "1" http-body-util = "0.1" diff --git a/crates/buzz-backend-kubernetes/src/config.rs b/crates/buzz-backend-kubernetes/src/config.rs index 3937576ce6a..935f2a287cb 100644 --- a/crates/buzz-backend-kubernetes/src/config.rs +++ b/crates/buzz-backend-kubernetes/src/config.rs @@ -416,6 +416,28 @@ pub fn config_schema() -> serde_json::Value { "title": "Service account", "description": "Scheduling/RBAC identity only. No API token is mounted." }, + "sandbox": { + "oneOf": [ + {"const": false}, + { + "type": "object", + "additionalProperties": false, + "required": ["channel_id", "thread_root"], + "properties": { + "channel_id": { + "type": "string", + "format": "uuid", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" + }, + "thread_root": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + } + ], + "x-buzz-ui-type": "boolean", + "default": false, + "title": "Workspace per thread", + "description": "Use Agent Sandbox. Mention the agent in a channel to create or reuse that thread's workspace. Requires an absolute Pod deadline and BUZZ_CHECKPOINT_BUCKET/BUZZ_CHECKPOINT_PREFIX (or BUZZ_SANDBOX_CHECKPOINT=disabled) in the agent environment. Desktop stores a boolean setting and binds the mentioned thread at launch; deploy accepts only {\"channel_id\": \"\", \"thread_root\": \"\"} and refuses true. false or absent disables Sandbox." + }, "identity_policy": { "type": "object", "description": "Optional approved agent and owner public keys, cryptographically checked before cluster access. Also enables local --check-identity preflight.", @@ -806,6 +828,7 @@ mod tests { "memory_request", "namespace", "pod_options", + "sandbox", "service_account" ] ); diff --git a/crates/buzz-backend-kubernetes/src/env.rs b/crates/buzz-backend-kubernetes/src/env.rs index 5fc27ab9055..31c324cdffe 100644 --- a/crates/buzz-backend-kubernetes/src/env.rs +++ b/crates/buzz-backend-kubernetes/src/env.rs @@ -93,7 +93,13 @@ const RESPOND_TO_ALLOWLIST: &str = "allowlist"; /// rejects `nobody` on purpose (`managed_agents/types.rs:871-880`), but the /// harness starts fine with it. This guard exists to cover non-desktop callers, /// so inheriting a desktop-only narrowing would refuse a launch that works. -const RESPOND_TO_MODES: [&str; 4] = ["owner-only", RESPOND_TO_ALLOWLIST, "anyone", "nobody"]; +const RESPOND_TO_MODES: [&str; 5] = [ + "owner-only", + "owner", + RESPOND_TO_ALLOWLIST, + "anyone", + "nobody", +]; /// Refuse a respond-to gate the harness will reject at config parse. /// @@ -804,7 +810,7 @@ mod tests { /// for. Without this test, refusing `nobody` would pass everything above. #[test] fn every_mode_the_harness_accepts_is_deployable() { - for mode in ["owner-only", "allowlist", "anyone", "nobody"] { + for mode in ["owner-only", "owner", "allowlist", "anyone", "nobody"] { let agent = payload_json(serde_json::json!({ "respond_to": mode, "respond_to_allowlist": [pubkey('a')], diff --git a/crates/buzz-backend-kubernetes/src/main.rs b/crates/buzz-backend-kubernetes/src/main.rs index a416e2e99e6..292deb5587a 100644 --- a/crates/buzz-backend-kubernetes/src/main.rs +++ b/crates/buzz-backend-kubernetes/src/main.rs @@ -21,6 +21,7 @@ mod naming; mod observe; mod pod; mod reconcile; +mod sandbox; mod wire; use std::io::Read; @@ -135,9 +136,10 @@ fn refuse_relay_mesh(raw: &serde_json::Value) -> Option { /// Run one deploy to a terminal outcome. async fn deploy_agent(request: &wire::DeployRequest) -> Result { let cfg = config::parse(&request.provider_config)?; + let sandbox = sandbox::Options::parse(&request.provider_config)?; // Identity before any cluster contact: a malformed nsec is a refusal, not // a failed connection (§Deploy State Machine step 0). - let identity = identity::check(request, false)?; + let identity = identity::check(request, sandbox.is_some())?; // One generation for this operation's first attempt; the reconciler mints // its own per attempt and restamps the correlator to match. @@ -150,6 +152,15 @@ async fn deploy_agent(request: &wire::DeployRequest) -> Result { )?; let client = client::connect(cfg.context.as_deref()).await?; + if let Some(scope) = sandbox { + let owner = request + .agent + .launch + .as_ref() + .and_then(|launch| launch.owner_pubkey.as_deref()) + .ok_or("Sandbox requires a verified launch owner")?; + return sandbox::deploy(client, &identity, &cfg, &scope, owner, env).await; + } let substrate = cluster::Cluster::new(client, &cfg.namespace); reconcile::deploy(&substrate, &identity, &cfg, env).await } diff --git a/crates/buzz-backend-kubernetes/src/sandbox.rs b/crates/buzz-backend-kubernetes/src/sandbox.rs new file mode 100644 index 00000000000..0293a546bd0 --- /dev/null +++ b/crates/buzz-backend-kubernetes/src/sandbox.rs @@ -0,0 +1,532 @@ +//! Agent Sandbox integration. The upstream controller owns the Pod; this +//! provider owns its launch contract and binds credentials to the first Pod UID. + +use std::collections::BTreeMap; +use std::time::Duration; + +use k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector, Pod, Secret}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; +use kube::api::{Api, ListParams, PostParams}; +use kube::core::{ApiResource, DynamicObject, GroupVersionKind}; +use kube::{Client, ResourceExt}; +use serde::Deserialize; +use serde_json::json; +use sha2::{Digest, Sha256}; + +use crate::{config::ProviderConfig, intent::Fingerprint, naming, pod}; + +const INITIAL_POD: &str = "buzz.block.xyz/initial-pod-uid"; +const GENERATION: &str = "buzz.block.xyz/generation"; +const OWNER: &str = "buzz.block.xyz/owner-pubkey"; +const CHANNEL: &str = "buzz.block.xyz/channel-id"; +const THREAD: &str = "buzz.block.xyz/thread-root"; +const SESSION: &str = "buzz.block.xyz/session-id"; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Options { + pub channel_id: String, + pub thread_root: String, +} + +impl Options { + fn name(&self, identity: &naming::AgentIdentity) -> String { + let scope = Sha256::digest(format!("{}:{}", self.channel_id, self.thread_root).as_bytes()); + format!("{}-{}", identity.pod_name(), &hex::encode(scope)[..12]) + } + + pub fn parse(value: &serde_json::Value) -> Result, String> { + let Some(value) = value.get("sandbox") else { + return Ok(None); + }; + if value == &serde_json::Value::Bool(false) { + return Ok(None); + } + let options: Self = serde_json::from_value(value.clone()) + .map_err(|_| "sandbox requires channel_id and canonical thread_root".to_string())?; + if uuid::Uuid::parse_str(&options.channel_id) + .map(|id| id.to_string()) + .ok() + .as_deref() + != Some(&options.channel_id) + { + return Err("sandbox channel_id must be a canonical UUID".into()); + } + if options.thread_root.len() != 64 + || !options + .thread_root + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + { + return Err("sandbox thread_root must be a lowercase 64-character event ID".into()); + } + Ok(Some(options)) + } +} + +fn resource() -> ApiResource { + ApiResource::from_gvk(&GroupVersionKind::gvk( + "agents.x-k8s.io", + "v1beta1", + "Sandbox", + )) +} + +fn api(client: Client, namespace: &str) -> Api { + Api::namespaced_with(client, namespace, &resource()) +} + +fn annotation<'a>(object: &'a DynamicObject, key: &str) -> Option<&'a str> { + object + .metadata + .annotations + .as_ref()? + .get(key) + .map(String::as_str) +} + +fn build( + identity: &naming::AgentIdentity, + cfg: &ProviderConfig, + scope: &Options, + owner: &str, + generation: &str, + fingerprint: &Fingerprint, +) -> Result { + let name = scope.name(identity); + let mut child = pod::build_pod(identity, cfg, generation, fingerprint); + child.metadata.name = Some(name.clone()); + // Older bare-Pod providers must refuse these controller-owned resources. + child + .labels_mut() + .insert(naming::LABEL_BINDING_VERSION.into(), "2".into()); + let annotations = child.metadata.annotations.get_or_insert_with(BTreeMap::new); + annotations.extend([ + (OWNER.into(), owner.into()), + (CHANNEL.into(), scope.channel_id.clone()), + (THREAD.into(), scope.thread_root.clone()), + (SESSION.into(), generation.into()), + (GENERATION.into(), generation.into()), + ]); + // The bare-Pod grace (`TERMINATION_GRACE_SECONDS`) already covers the + // drain and the harness's 120s checkpoint; keep it so the template + // matches the fingerprinted intent. + let spec = child.spec.as_mut().ok_or("agent Pod has no spec")?; + spec.containers[0] + .env + .get_or_insert_with(Vec::new) + .push(EnvVar { + name: "BUZZ_SANDBOX_POD_UID".into(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + api_version: Some("v1".into()), + field_path: "metadata.uid".into(), + }), + ..Default::default() + }), + ..Default::default() + }); + let mut sandbox = DynamicObject::new(&name, &resource()); + sandbox.metadata = child.metadata.clone(); + sandbox.data = json!({"spec": { + "operatingMode": "Running", + "service": false, + "podTemplate": { + "metadata": {"labels": child.metadata.labels, "annotations": child.metadata.annotations}, + "spec": child.spec, + }, + }}); + Ok(sandbox) +} + +fn verify( + sandbox: &DynamicObject, + identity: &naming::AgentIdentity, + scope: &Options, + owner: &str, + fingerprint: &Fingerprint, +) -> Result<(), String> { + for (key, expected) in [ + (naming::ANNOTATION_PUBKEY_FULL, identity.pubkey_hex()), + (naming::ANNOTATION_CREATE_INTENT, fingerprint.as_str()), + (OWNER, owner), + (CHANNEL, scope.channel_id.as_str()), + (THREAD, scope.thread_root.as_str()), + ] { + if annotation(sandbox, key) != Some(expected) { + return Err("Sandbox identity, thread, or launch configuration differs; explicit recovery is required".into()); + } + } + if sandbox + .labels() + .get(naming::LABEL_MANAGED_BY) + .map(String::as_str) + != Some(naming::MANAGED_BY) + || sandbox.metadata.deletion_timestamp.is_some() + || sandbox.data["spec"]["operatingMode"] != "Running" + { + return Err( + "Sandbox is not an active managed session; explicit recovery is required".into(), + ); + } + Ok(()) +} + +fn owned_pod(pod: &Pod, sandbox: &DynamicObject) -> bool { + sandbox.metadata.uid.as_ref().is_some_and(|uid| { + pod.owner_references().iter().any(|r| { + &r.uid == uid + && r.kind == "Sandbox" + && r.api_version == "agents.x-k8s.io/v1beta1" + && r.controller == Some(true) + }) + }) +} + +/// Opt-out for launches that knowingly run without a stop checkpoint. +const CHECKPOINT_OPT_OUT: &str = "BUZZ_SANDBOX_CHECKPOINT"; + +/// Status reason and code only: an apiserver message can echo object data, +/// and these calls handle the launch Secret. +fn api_failure(error: &kube::Error) -> String { + match error { + kube::Error::Api(status) => format!("{} ({})", status.reason, status.code), + other => other.to_string(), + } +} + +/// Configure the harness for one owner thread. The workspace is an +/// `emptyDir` on a `restartPolicy: Never` Pod, so the stop checkpoint is the +/// only copy that outlives the process; refuse to launch without its storage +/// unless the caller opts out explicitly. +fn apply_session_env(scope: &Options, env: &mut BTreeMap) -> Result<(), String> { + let opted_out = env.get(CHECKPOINT_OPT_OUT).map(String::as_str) == Some("disabled"); + // A blank value is as unusable as a missing one: the save would fail at stop. + let configured = |key: &str| env.get(key).is_some_and(|value| !value.trim().is_empty()); + if !opted_out && configured("BUZZ_CHECKPOINT_BUCKET") && configured("BUZZ_CHECKPOINT_PREFIX") { + env.insert("BUZZ_ACP_CHECKPOINT_ON_STOP".into(), "true".into()); + } else if !opted_out { + return Err(format!( + "Sandbox requires BUZZ_CHECKPOINT_BUCKET and BUZZ_CHECKPOINT_PREFIX in the agent \ + environment, or {CHECKPOINT_OPT_OUT}=disabled to launch without a stop checkpoint" + )); + } + env.insert("BUZZ_ACP_ACTIVITY_LOG".into(), "true".into()); + env.insert("BUZZ_ACP_RESPOND_TO".into(), "owner".into()); + env.insert("BUZZ_ACP_ALLOWED_RESPOND_TO".into(), "owner".into()); + env.insert( + "BUZZ_ACP_BOUND_SESSION".into(), + json!({"channel_id":scope.channel_id,"thread_root":scope.thread_root}).to_string(), + ); + env.insert("BUZZ_ACP_SESSION_POLICY".into(), "thread".into()); + env.insert("BUZZ_ACP_SUBSCRIBE".into(), "mentions".into()); + env.insert("BUZZ_ACP_CHANNELS".into(), scope.channel_id.clone()); + env.insert("BUZZ_ACP_HEARTBEAT_INTERVAL".into(), "0".into()); + Ok(()) +} + +fn configure_setup( + cfg: &mut ProviderConfig, + identity: &naming::AgentIdentity, + env: &mut BTreeMap, +) -> Result<(), String> { + let Some(mode) = env.get("BUZZ_SETUP_MODE") else { + return Ok(()); + }; + if mode != "codex-device-v1" + || env.get("BUZZ_ACP_AGENT_COMMAND").map(String::as_str) != Some("codex-acp") + || env.get("BUZZ_ACP_PERMISSION_MODE").map(String::as_str) != Some("agent-full-access") + { + return Err( + "Sandbox account setup requires codex-device-v1, codex-acp, and agent-full-access" + .into(), + ); + } + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| "could not establish account setup time")? + .as_secs(); + let (_, digest) = cfg + .image + .as_str() + .rsplit_once('@') + .ok_or("image digest is missing")?; + // The image acknowledges this exact Pod/agent/image generation. The Pod UID + // comes from the downward API, overriding any caller-supplied environment. + cfg.pod_options.setup_pending = true; + env.remove("BUZZ_SETUP_POD_UID"); + env.extend([ + ( + "BUZZ_SETUP_AGENT_PUBKEY".into(), + identity.pubkey_hex().into(), + ), + ("BUZZ_SETUP_IMAGE_DIGEST".into(), digest.into()), + ( + "BUZZ_SETUP_NONCE".into(), + uuid::Uuid::new_v4().simple().to_string(), + ), + ("BUZZ_SETUP_DEADLINE".into(), (now + 900).to_string()), + ("CODEX_HOME".into(), "/home/agent/.codex".into()), + ( + "CODEX_CONFIG".into(), + r#"{"cli_auth_credentials_store":"file"}"#.into(), + ), + ("NO_BROWSER".into(), "1".into()), + ]); + Ok(()) +} + +/// Reuses a running, identically scoped Sandbox. Terminal or replaced Pods are +/// never redeployed implicitly. Namespace creation remains Terraform's concern. +pub async fn deploy( + client: Client, + identity: &naming::AgentIdentity, + cfg: &ProviderConfig, + scope: &Options, + owner: &str, + mut env: BTreeMap, +) -> Result { + let mut effective_cfg = cfg.clone(); + configure_setup(&mut effective_cfg, identity, &mut env)?; + let cfg = &effective_cfg; + if cfg.pod_options.active_deadline_seconds.is_none() { + return Err("Sandbox requires pod_options.active_deadline_seconds".into()); + } + apply_session_env(scope, &mut env)?; + // Include the UID binding key in the fingerprint without hashing its value. + env.insert("BUZZ_SANDBOX_INITIAL_POD_UID".into(), String::new()); + let fingerprint = pod::intent_template(cfg, env.keys().cloned()).fingerprint(); + let sandboxes = api(client.clone(), &cfg.namespace); + let pods: Api = Api::namespaced(client.clone(), &cfg.namespace); + let secrets: Api = Api::namespaced(client, &cfg.namespace); + let name = scope.name(identity); + let mut sandbox = match sandboxes + .get_opt(&name) + .await + .map_err(|e| format!("read Sandbox: {e}"))? + { + Some(existing) => existing, + None => { + if pods + .get_opt(&name) + .await + .map_err(|e| format!("check Pod collision: {e}"))? + .is_some() + || !pods + .list(&ListParams::default().labels(&identity.selector())) + .await + .map_err(|e| format!("check existing agent consumers: {e}"))? + .items + .iter() + .all(|pod| { + pod.labels() + .get(naming::LABEL_BINDING_VERSION) + .map(String::as_str) + == Some("2") + && pod.annotations().get(OWNER).map(String::as_str) == Some(owner) + && pod.owner_references().iter().any(|r| { + r.api_version == "agents.x-k8s.io/v1beta1" + && r.kind == "Sandbox" + && r.controller == Some(true) + }) + }) + { + return Err( + "agent already has a Pod; stop and recover it explicitly before using Sandbox" + .into(), + ); + } + let generation = uuid::Uuid::new_v4().simple().to_string(); + let desired = build(identity, cfg, scope, owner, &generation, &fingerprint)?; + match sandboxes.create(&PostParams::default(), &desired).await { + Ok(created) => created, + Err(kube::Error::Api(e)) if e.reason == "AlreadyExists" => sandboxes + .get(&name) + .await + .map_err(|e| format!("read concurrent Sandbox: {e}"))?, + Err(e) => { + return Err(format!( + "create Sandbox (install the Agent Sandbox v1beta1 CRD/controller first): {e}" + )) + } + } + } + }; + verify(&sandbox, identity, scope, owner, &fingerprint)?; + let generation = annotation(&sandbox, GENERATION) + .ok_or("Sandbox generation is missing")? + .to_string(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(120); + loop { + if let Some(child) = pods + .get_opt(&name) + .await + .map_err(|e| format!("read Sandbox Pod: {e}"))? + { + if !owned_pod(&child, &sandbox) { + return Err("Pod is not owned by this Sandbox".into()); + } + let uid = child.metadata.uid.as_deref().ok_or("Pod UID is missing")?; + if child.metadata.deletion_timestamp.is_some() + || child + .status + .as_ref() + .and_then(|s| s.phase.as_deref()) + .is_some_and(|p| matches!(p, "Failed" | "Succeeded")) + { + return Err( + "Sandbox session ended; save or recover its workspace explicitly".into(), + ); + } + if let Some(initial_uid) = annotation(&sandbox, INITIAL_POD) { + if uid != initial_uid { + return Err("Sandbox Pod was replaced; credentials remain bound to the original session, explicit recovery is required".into()); + } + let secret = secrets + .get_opt(&identity.secret_name(&generation)) + .await + .map_err(|e| { + format!("could not check Sandbox launch Secret: {}", api_failure(&e)) + })?; + let Some(secret) = secret else { + if tokio::time::Instant::now() >= deadline { + return Err("Sandbox credential binding is incomplete; explicit recovery is required".into()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + continue; + }; + if secret.immutable != Some(true) + || !secret + .owner_references() + .iter() + .any(|owner| owner.uid == uid && owner.kind == "Pod") + || secret + .annotations() + .get(naming::ANNOTATION_PUBKEY_FULL) + .map(String::as_str) + != Some(identity.pubkey_hex()) + { + return Err("Sandbox credentials belong to a different Pod or agent".into()); + } + let saved_relay = secret + .data + .as_ref() + .and_then(|data| data.get("BUZZ_RELAY_URL")) + .map(|value| value.0.as_slice()); + if saved_relay != env.get("BUZZ_RELAY_URL").map(|value| value.as_bytes()) { + return Err("Sandbox belongs to a different relay; use a new thread or explicit recovery".into()); + } + } else { + // resourceVersion makes this an atomic first-Pod binding. A + // concurrent deploy must retry and observe the winner's binding. + sandbox + .annotations_mut() + .insert(INITIAL_POD.into(), uid.into()); + sandbox = match sandboxes + .replace(&name, &PostParams::default(), &sandbox) + .await + { + Ok(bound) => bound, + Err(kube::Error::Api(error)) if error.reason == "Conflict" => { + sandbox = sandboxes + .get(&name) + .await + .map_err(|e| format!("refresh Sandbox binding: {e}"))?; + verify(&sandbox, identity, scope, owner, &fingerprint)?; + if annotation(&sandbox, GENERATION) != Some(&generation) { + return Err( + "Sandbox generation changed; explicit recovery is required".into(), + ); + } + if tokio::time::Instant::now() >= deadline { + return Err( + "Sandbox kept changing during credential binding; retry deploy" + .into(), + ); + } + continue; + } + Err(error) => return Err(format!("bind Sandbox to its first Pod: {error}")), + }; + env.insert("BUZZ_MANAGED_AGENT_START_NONCE".into(), generation.clone()); + env.insert("BUZZ_SANDBOX_INITIAL_POD_UID".into(), uid.into()); + let mut secret = pod::build_secret(identity, &cfg.namespace, &generation, env); + secret + .labels_mut() + .insert(naming::LABEL_BINDING_VERSION.into(), "2".into()); + secret.metadata.owner_references = Some(vec![OwnerReference { + api_version: "v1".into(), + kind: "Pod".into(), + name: name.clone(), + uid: uid.into(), + controller: Some(true), + block_owner_deletion: Some(false), + }]); + secrets + .create(&PostParams::default(), &secret) + .await + .map_err(|e| { + format!( + "could not create bound Sandbox credentials ({}); explicit recovery is required", + api_failure(&e) + ) + })?; + } + return wait_for_start(&pods, &sandbox, &name, uid).await; + } + if annotation(&sandbox, INITIAL_POD).is_some() { + return Err("Sandbox lost its original Pod; explicit recovery is required".into()); + } + if tokio::time::Instant::now() >= deadline { + return Err( + "Sandbox controller has not created its Pod; inspect the Sandbox and retry".into(), + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +async fn wait_for_start( + pods: &Api, + sandbox: &DynamicObject, + name: &str, + uid: &str, +) -> Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(120); + loop { + let child = pods + .get_opt(name) + .await + .map_err(|e| format!("observe Sandbox startup: {e}"))? + .ok_or("Sandbox lost its Pod during startup; explicit recovery is required")?; + if child.metadata.uid.as_deref() != Some(uid) + || !owned_pod(&child, sandbox) + || child.metadata.deletion_timestamp.is_some() + { + return Err("Sandbox Pod changed during startup; explicit recovery is required".into()); + } + let state = child + .status + .as_ref() + .and_then(|s| s.container_statuses.as_ref()) + .and_then(|containers| containers.iter().find(|c| c.name == "agent")) + .and_then(|c| c.state.as_ref()); + if state.is_some_and(|s| s.terminated.is_some()) { + return Err("Sandbox process ended during startup; inspect its retained logs".into()); + } + if state.is_some_and(|s| s.running.is_some()) { + return Ok(name.into()); + } + if tokio::time::Instant::now() >= deadline { + return Err( + "Sandbox startup timed out; inspect scheduling and image status before retrying" + .into(), + ); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} + +#[cfg(test)] +#[path = "sandbox_tests.rs"] +mod tests; diff --git a/crates/buzz-backend-kubernetes/src/sandbox_tests.rs b/crates/buzz-backend-kubernetes/src/sandbox_tests.rs new file mode 100644 index 00000000000..01755e580fa --- /dev/null +++ b/crates/buzz-backend-kubernetes/src/sandbox_tests.rs @@ -0,0 +1,370 @@ +use super::*; +use nostr::nips::nip19::ToBech32; + +#[test] +fn custom_image_is_inside_a_real_sandbox_with_a_downward_uid() { + let identity = naming::AgentIdentity::from_nsec( + &nostr::Keys::generate().secret_key().to_bech32().unwrap(), + ) + .unwrap(); + let cfg = crate::config::parse(&json!({"namespace":"agents", "image":format!("example.com/agent@sha256:{}", "a".repeat(64)), + "pod_options":{"active_deadline_seconds":3600,"workspace_size_limit":"50Gi"}})).unwrap(); + let scope = Options { + channel_id: uuid::Uuid::new_v4().to_string(), + thread_root: "c".repeat(64), + }; + let fp = pod::intent_template(&cfg, vec![]).fingerprint(); + let mut sandbox = build(&identity, &cfg, &scope, "owner", "generation", &fp).unwrap(); + assert_eq!( + sandbox.types.as_ref().unwrap().api_version, + "agents.x-k8s.io/v1beta1" + ); + let spec = &sandbox.data["spec"]["podTemplate"]["spec"]; + assert_eq!(spec["containers"][0]["image"], cfg.image.as_str()); + assert_eq!( + spec["containers"][0]["env"][0]["valueFrom"]["fieldRef"]["fieldPath"], + "metadata.uid" + ); + assert_eq!(spec["automountServiceAccountToken"], false); + assert_eq!(spec["restartPolicy"], "Never"); + // One grace for both paths: it covers the checkpoint, and intent + // fingerprints assume it. + assert_eq!( + spec["terminationGracePeriodSeconds"], + crate::config::TERMINATION_GRACE_SECONDS + ); + assert_eq!(spec["volumes"][0]["emptyDir"]["sizeLimit"], "50Gi"); + verify(&sandbox, &identity, &scope, "owner", &fp).unwrap(); + sandbox + .annotations_mut() + .insert(OWNER.into(), "different-owner".into()); + assert!(verify(&sandbox, &identity, &scope, "owner", &fp).is_err()); +} + +#[test] +fn options_parse_accepts_only_a_canonical_thread_scope() { + let channel = "0f8fad5b-d9cb-469f-a165-70867728950e"; + let root = "a".repeat(64); + let parse = |sandbox: serde_json::Value| Options::parse(&json!({ "sandbox": sandbox })); + assert!(Options::parse(&json!({})).unwrap().is_none()); + assert!(parse(json!(false)).unwrap().is_none()); + let scope = parse(json!({"channel_id": channel, "thread_root": root})) + .unwrap() + .unwrap(); + assert_eq!( + (scope.channel_id.as_str(), scope.thread_root.as_str()), + (channel, root.as_str()) + ); + for (sandbox, error) in [ + (json!(true), "requires channel_id"), + (json!({"channel_id": channel}), "requires channel_id"), + ( + json!({"channel_id": channel, "thread_root": root, "extra": 1}), + "requires channel_id", + ), + ( + json!({"channel_id": channel.to_uppercase(), "thread_root": root}), + "canonical UUID", + ), + ( + json!({"channel_id": channel.replace('-', ""), "thread_root": root}), + "canonical UUID", + ), + ( + json!({"channel_id": channel, "thread_root": "A".repeat(64)}), + "lowercase 64-character", + ), + ( + json!({"channel_id": channel, "thread_root": "a".repeat(63)}), + "lowercase 64-character", + ), + ( + json!({"channel_id": channel, "thread_root": "g".repeat(64)}), + "lowercase 64-character", + ), + ] { + let refused = parse(sandbox.clone()).unwrap_err(); + assert!(refused.contains(error), "{sandbox}: {refused}"); + } +} + +fn identity() -> naming::AgentIdentity { + naming::AgentIdentity::from_nsec(&nostr::Keys::generate().secret_key().to_bech32().unwrap()) + .unwrap() +} + +#[test] +fn account_setup_refuses_any_other_mode_runtime_or_permission() { + let cfg = crate::config::parse(&json!({"namespace":"agents", + "image":format!("example.com/agent@sha256:{}", "a".repeat(64))})) + .unwrap(); + for (key, value) in [ + ("BUZZ_SETUP_MODE", "codex-device-v2"), + ("BUZZ_ACP_AGENT_COMMAND", "claude-agent-acp"), + ("BUZZ_ACP_PERMISSION_MODE", "default"), + ] { + let mut cfg = cfg.clone(); + let mut env = fixture_env(); + env.insert(key.into(), value.into()); + let before = env.clone(); + assert!(configure_setup(&mut cfg, &identity(), &mut env) + .unwrap_err() + .contains("codex-device-v1")); + assert_eq!(env, before, "a refused setup must not change the launch"); + assert!(!cfg.pod_options.setup_pending); + } + let mut cfg = cfg.clone(); + let mut env = fixture_env(); + configure_setup(&mut cfg, &identity(), &mut env).unwrap(); + assert!(cfg.pod_options.setup_pending); + assert!(!env.contains_key("BUZZ_SETUP_POD_UID")); +} + +#[test] +fn session_env_requires_checkpoint_storage_unless_disabled() { + let scope = Options { + channel_id: uuid::Uuid::new_v4().to_string(), + thread_root: "c".repeat(64), + }; + let mut env = fixture_env(); + apply_session_env(&scope, &mut env).unwrap(); + assert_eq!(env["BUZZ_ACP_CHECKPOINT_ON_STOP"], "true"); + assert_eq!(env["BUZZ_ACP_RESPOND_TO"], "owner"); + + for missing in ["BUZZ_CHECKPOINT_BUCKET", "BUZZ_CHECKPOINT_PREFIX"] { + let mut env = fixture_env(); + env.remove(missing); + let refused = apply_session_env(&scope, &mut env).unwrap_err(); + assert!( + refused.contains("BUZZ_SANDBOX_CHECKPOINT=disabled"), + "{refused}" + ); + // A blank value would let the launch through and fail the stop save. + for blank in ["", " "] { + let mut env = fixture_env(); + env.insert(missing.into(), blank.into()); + assert!( + apply_session_env(&scope, &mut env).is_err(), + "{missing}={blank:?} must be refused" + ); + assert!(!env.contains_key("BUZZ_ACP_CHECKPOINT_ON_STOP")); + } + } + + let mut env = fixture_env(); + env.remove("BUZZ_CHECKPOINT_BUCKET"); + env.insert(CHECKPOINT_OPT_OUT.into(), "disabled".into()); + apply_session_env(&scope, &mut env).unwrap(); + assert!(!env.contains_key("BUZZ_ACP_CHECKPOINT_ON_STOP")); + // The opt-out also wins over configured storage. + let mut env = fixture_env(); + env.insert(CHECKPOINT_OPT_OUT.into(), "disabled".into()); + apply_session_env(&scope, &mut env).unwrap(); + assert!(!env.contains_key("BUZZ_ACP_CHECKPOINT_ON_STOP")); +} + +fn fixture_env() -> BTreeMap { + BTreeMap::from([ + ("BUZZ_RELAY_URL".into(), "wss://fixture.invalid".into()), + ("BUZZ_CHECKPOINT_BUCKET".into(), "fixture-bucket".into()), + ("BUZZ_CHECKPOINT_PREFIX".into(), "fixture/".into()), + ("BUZZ_SETUP_MODE".into(), "codex-device-v1".into()), + ("BUZZ_ACP_AGENT_COMMAND".into(), "codex-acp".into()), + ( + "BUZZ_ACP_PERMISSION_MODE".into(), + "agent-full-access".into(), + ), + ( + "BUZZ_SETUP_AGENT_PUBKEY".into(), + "stale-caller-value".into(), + ), + ("BUZZ_SETUP_POD_UID".into(), "stale-caller-value".into()), + ]) +} + +#[tokio::test] +#[ignore = "requires the isolated kind-remote-agent-check cluster and fixture image"] +async fn live_controller_reuses_scopes_and_refuses_lost_workspaces() { + use k8s_openapi::api::core::v1::Namespace; + use kube::api::DeleteParams; + use kube::config::{KubeConfigOptions, Kubeconfig}; + + let config = + Kubeconfig::read_from(std::env::var("BUZZ_SANDBOX_TEST_KUBECONFIG").unwrap()).unwrap(); + let config = kube::Config::from_custom_kubeconfig( + config, + &KubeConfigOptions { + context: Some("kind-remote-agent-check".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!( + config.cluster_url.host(), + Some("127.0.0.1"), + "never use a shared cluster for this test" + ); + let _ = rustls::crypto::ring::default_provider().install_default(); + let client = Client::try_from(config).unwrap(); + let namespace = format!( + "buzz-agent-staging-test-{}", + &uuid::Uuid::new_v4().simple().to_string()[..8] + ); + let namespaces: Api = Api::all(client.clone()); + namespaces.create(&PostParams::default(), &serde_json::from_value(json!({ + "metadata":{"name":namespace,"labels":{"pod-security.kubernetes.io/enforce":"restricted"}} + })).unwrap()).await.unwrap(); + let cfg = crate::config::parse(&json!({ + "namespace":namespace, "image":std::env::var("BUZZ_SANDBOX_TEST_IMAGE").unwrap(), + "cpu_request":"10m", "memory_request":"16Mi", "cpu_limit":"100m", "memory_limit":"32Mi", + "pod_options":{"active_deadline_seconds":600} + })) + .unwrap(); + let identity = naming::AgentIdentity::from_nsec( + &nostr::Keys::generate().secret_key().to_bech32().unwrap(), + ) + .unwrap(); + let scope = Options { + channel_id: uuid::Uuid::new_v4().to_string(), + thread_root: "a".repeat(64), + }; + let (first, concurrent) = tokio::join!( + deploy( + client.clone(), + &identity, + &cfg, + &scope, + "owner", + fixture_env() + ), + deploy( + client.clone(), + &identity, + &cfg, + &scope, + "owner", + fixture_env() + ), + ); + let first = first.unwrap(); + assert_eq!( + concurrent.unwrap(), + first, + "concurrent mentions reuse one Sandbox" + ); + let secrets: Api = Api::namespaced(client.clone(), &namespace); + let before = secrets.list(&ListParams::default()).await.unwrap(); + assert_eq!(before.items.len(), 1); + let data = before.items[0].data.as_ref().unwrap(); + assert_eq!( + data["BUZZ_SETUP_AGENT_PUBKEY"].0, + identity.pubkey_hex().as_bytes() + ); + assert_eq!(data["BUZZ_SETUP_NONCE"].0.len(), 32); + assert_eq!(data["BUZZ_MANAGED_AGENT_START_NONCE"].0.len(), 32); + assert!(!data.contains_key("BUZZ_SETUP_POD_UID")); + assert_eq!( + deploy( + client.clone(), + &identity, + &cfg, + &scope, + "owner", + fixture_env() + ) + .await + .unwrap(), + first + ); + let after = secrets.list(&ListParams::default()).await.unwrap(); + assert_eq!(after.items.len(), 1); + assert_eq!( + after.items[0].data, before.items[0].data, + "retry must preserve setup binding and deadline" + ); + assert!(deploy( + client.clone(), + &identity, + &cfg, + &scope, + "other-owner", + fixture_env() + ) + .await + .is_err()); + let mut other_relay = fixture_env(); + other_relay.insert("BUZZ_RELAY_URL".into(), "wss://other.invalid".into()); + assert!(deploy( + client.clone(), + &identity, + &cfg, + &scope, + "owner", + other_relay + ) + .await + .unwrap_err() + .contains("different relay")); + let second_scope = Options { + channel_id: scope.channel_id.clone(), + thread_root: "b".repeat(64), + }; + let second = deploy( + client.clone(), + &identity, + &cfg, + &second_scope, + "owner", + fixture_env(), + ) + .await + .unwrap(); + assert_ne!(first, second); + let pods: Api = Api::namespaced(client.clone(), &namespace); + let original = pods.get(&first).await.unwrap(); + let pod_uid = original.metadata.uid.unwrap(); + pods.delete( + &first, + &DeleteParams { + grace_period_seconds: Some(0), + ..Default::default() + }, + ) + .await + .unwrap(); + let mut replacement = None; + for _ in 0..30 { + if let Some(pod) = pods.get_opt(&first).await.unwrap() { + if pod.metadata.uid.as_deref() != Some(&pod_uid) { + replacement = Some(pod); + break; + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert!( + replacement.is_some(), + "upstream controller recreates the Pod" + ); + let error = deploy( + client.clone(), + &identity, + &cfg, + &scope, + "owner", + fixture_env(), + ) + .await + .unwrap_err(); + assert!(error.contains("replaced"), "{error}"); + let sandboxes = api(client, &namespace) + .list(&ListParams::default()) + .await + .unwrap(); + assert_eq!(sandboxes.items.len(), 2, "one Sandbox per canonical thread"); + namespaces + .delete(&namespace, &DeleteParams::default()) + .await + .unwrap(); +} diff --git a/desktop/src/features/agents/ui/ProviderConfigFields.test.mjs b/desktop/src/features/agents/ui/ProviderConfigFields.test.mjs index 0dfd80d4444..26f6dc82e90 100644 --- a/desktop/src/features/agents/ui/ProviderConfigFields.test.mjs +++ b/desktop/src/features/agents/ui/ProviderConfigFields.test.mjs @@ -12,6 +12,23 @@ const schema = { }; describe("coerceConfigValues", () => { + it("keeps deferred thread scope settings boolean before launch binds them", () => { + const deferred = { + properties: { + sandbox: { + oneOf: [{ const: false }, { type: "object" }], + "x-buzz-ui-type": "boolean", + default: false, + }, + }, + }; + for (const enabled of [true, false]) { + assert.deepEqual( + coerceConfigValues({ sandbox: String(enabled) }, deferred), + { sandbox: enabled }, + ); + } + }); it("omits cleared numeric fields without losing explicit zero", () => { assert.deepEqual( coerceConfigValues( diff --git a/desktop/src/features/agents/ui/ProviderConfigFields.tsx b/desktop/src/features/agents/ui/ProviderConfigFields.tsx index 031b0b7277d..64d8880a795 100644 --- a/desktop/src/features/agents/ui/ProviderConfigFields.tsx +++ b/desktop/src/features/agents/ui/ProviderConfigFields.tsx @@ -13,7 +13,10 @@ export function coerceConfigValues( const result: Record = {}; for (const [key, value] of Object.entries(config)) { const prop = properties[key] as Record | undefined; - const schemaType = prop?.type; + // A deferred Sandbox scope is a boolean setting until launch binds the + // published thread. Its provider wire schema still requires the object. + const schemaType = + prop?.["x-buzz-ui-type"] === "boolean" ? "boolean" : prop?.type; if (schemaType === "integer" || schemaType === "number") { if (value === "") continue; const num = Number(value); diff --git a/docs/remote-agents.md b/docs/remote-agents.md index b997e576ebc..5ed7e8fc854 100644 --- a/docs/remote-agents.md +++ b/docs/remote-agents.md @@ -416,6 +416,10 @@ string/number/boolean coercion, `required` gating. A provider MAY compute defaults freshly per call (the Kubernetes binding generates a random namespace default this way — §K8s Namespace). The schema's fields are subject to I2 validation when the user's values come back in `deploy`. +For a deferred thread scope, `x-buzz-ui-type: "boolean"` tells the desktop to +store an enable/disable setting. Launch replaces that setting with the bound +scope object. The JSON Schema still describes the provider's accepted wire +values: `false` or an object with `channel_id` and `thread_root`. ### `deploy` @@ -1426,6 +1430,41 @@ Launchers must also treat a success response without `"preflight": "identity"` a Launchers must check the `info` schema for required options before credential handoff, since older providers ignore unknown fields. +### Agent Sandbox workspaces + +Desktop mention routing is added by the following stack layer. This layer accepts scoped requests from direct provider clients. +Direct provider clients pass `sandbox={"channel_id":"","thread_root":""}` and an approved `identity_policy`. + +Install the `agents.x-k8s.io/v1beta1` Sandbox CRD and controller before launching. +Set a digest-pinned image and `pod_options.active_deadline_seconds`. +The image must reject a mismatch between `BUZZ_SANDBOX_POD_UID` and `BUZZ_SANDBOX_INITIAL_POD_UID` before reading account credentials. + +The controller owns the Pod. Concurrent mentions reuse its workspace when the agent, owner, relay, channel, and thread match. +The provider binds the launch Secret to the first Pod UID. +A replaced or completed Pod requires explicit recovery; a later mention cannot silently create an empty replacement session. +The workspace is an `emptyDir` on a `restartPolicy: Never` Pod. When the harness exits, the stop checkpoint is the only copy of its work. +Deleting a Sandbox also removes its Pod and any workspace that remains. Recover or checkpoint authorized work before deletion. + +Sandbox ingress accepts only its owner's signed messages for the assigned channel thread. +The provider enables JSONL activity and disables autonomous heartbeat work. +The agent environment must set `BUZZ_CHECKPOINT_BUCKET` and `BUZZ_CHECKPOINT_PREFIX`; the deployment's launch environment supplies them. Shutdown then invokes `buzz-agent-checkpoint save` after sessions drain, and a failed save makes the harness exit non-zero. +Deploy refuses Sandbox mode when either value is missing. +To launch without a stop checkpoint, for example in a disposable test cluster, set `BUZZ_SANDBOX_CHECKPOINT=disabled`. Such a workspace is lost when its process exits. +Sandbox Pods use the same `terminationGracePeriodSeconds` as bare Pods (300s), which covers the drain and the 120s checkpoint. +The image supplies that helper and account setup. Infrastructure supplies storage, resource quotas, and log collection. + +Use channel thread mentions to launch these workspaces. A generic Start action has no thread binding and is refused. +Remote setup success still requires an authenticated runtime response; a running setup gate alone does not prove account readiness. +For Codex account enrollment, set `BUZZ_SETUP_MODE=codex-device-v1`, select `codex-acp`, and use `agent-full-access`. +The provider supplies the agent/image binding, downward Pod UID, one-use nonce, and a 15-minute setup deadline. +Retries preserve the original binding and deadline. The image must accept the Sandbox's 32-character generation. + +**Limit: interrupted credential binding.** Deploy first records the Pod UID on the Sandbox, then creates the launch Secret. +If the Secret create fails after the UID is recorded, the deploy error names the Kubernetes reason and code. +Later deploys for that thread wait 120s for the Secret and then fail with "Sandbox credential binding is incomplete; explicit recovery is required". +To recover, delete the Sandbox. This discards the workspace; the container never started, so it contains no agent work. Then mention the agent in the thread again. +A later layer of this stack makes the binding resumable, so a retried deploy creates the missing Secret for the same Pod. + ### Distribution Its own release workflow (macOS arm64/x64 + Linux musl; the sprig workflow's