diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 457c19578..c51708f84 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -324,6 +324,58 @@ dependencies = [ "serde_cbor", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -1714,6 +1766,7 @@ dependencies = [ "nsm-qvl", "or-panic", "parity-scale-codec", + "pem", "rand 0.8.6", "rmp-serde", "rsa", @@ -1731,6 +1784,7 @@ dependencies = [ "tpm-types", "tpm2", "tracing", + "x509-parser", ] [[package]] @@ -1787,6 +1841,7 @@ dependencies = [ "clap", "cmd_lib", "dstack-api-auth", + "dstack-attest", "dstack-build-info", "dstack-gateway-rpc", "dstack-guest-agent-rpc", @@ -2062,13 +2117,30 @@ name = "dstack-tee-simulator" version = "0.6.0" dependencies = [ "anyhow", + "aws-nitro-enclaves-nsm-api", "cc-eventlog", "clap", "dcap-qvl", + "dstack-mr", + "dstack-types", + "fs-err", "fuser", + "hex", "libc", + "libloading", + "mock-attestation", + "nsm-qvl", + "pem", + "reqwest", "sd-notify", + "serde_cbor", + "serde_json", + "sev-snp-qvl", "sha2 0.10.9", + "tempfile", + "tokio", + "tpm-qvl", + "tpm-types", "tracing", "tracing-subscriber", ] @@ -4085,6 +4157,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -4209,6 +4287,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "mock-attestation" +version = "0.6.0" +dependencies = [ + "anyhow", + "axum", + "chrono", + "ciborium", + "clap", + "dcap-qvl", + "dstack-types", + "fs-err", + "hex", + "nsm-qvl", + "p256", + "p384", + "parity-scale-codec", + "rand 0.8.6", + "rcgen", + "reqwest", + "serde", + "serde_json", + "sev", + "sev-snp-qvl", + "sha2 0.10.9", + "tempfile", + "time", + "tokio", + "tpm-qvl", + "tpm-types", + "urlencoding", + "yasna", +] + [[package]] name = "moka" version = "0.12.15" @@ -6636,6 +6748,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_qs" version = "0.13.0" @@ -6757,9 +6880,13 @@ dependencies = [ "anyhow", "hex", "moka", + "pem", "reqwest", + "rustls-pki-types", + "rustls-webpki", "sev", "tokio", + "x509-parser", ] [[package]] @@ -7605,6 +7732,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -7708,6 +7836,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 3dd8f8179..68da4a425 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -68,6 +68,7 @@ members = [ "crates/dstack-auth", "crates/api-auth", "crates/build-info", + "crates/mock-attestation", ] resolver = "2" diff --git a/dstack/cert-client/src/lib.rs b/dstack/cert-client/src/lib.rs index 8b5b329a1..e55689b52 100644 --- a/dstack/cert-client/src/lib.rs +++ b/dstack/cert-client/src/lib.rs @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::sync::Arc; + use anyhow::{Context, Result}; use dstack_kms_rpc::{kms_client::KmsClient, SignCertRequest}; use dstack_types::{AppKeys, KeyProvider}; use ra_rpc::client::{RaClient, RaClientConfig}; -use ra_tls::cert::{generate_ra_cert, CaCert, CertSigningRequestV2}; +use ra_tls::{ + attestation::AttestationVerifier, + cert::{generate_ra_cert, CaCert, CertSigningRequestV2}, +}; pub enum CertRequestClient { Local { @@ -54,7 +59,7 @@ impl CertRequestClient { pub async fn create( keys: &AppKeys, - pccs_url: Option<&str>, + attestation_verifier: Arc, vm_config: String, ) -> Result { match &keys.key_provider { @@ -79,7 +84,7 @@ impl CertRequestClient { .tls_client_key(client_cert.key_pem) .tls_ca_cert(keys.ca_cert.clone()) .tls_built_in_root_certs(false) - .maybe_pccs_url(pccs_url.map(|s| s.to_string())) + .attestation_verifier(attestation_verifier) .build() .into_client() .context("Failed to create RA client")?; diff --git a/dstack/crates/mock-attestation/Cargo.toml b/dstack/crates/mock-attestation/Cargo.toml new file mode 100644 index 000000000..65091e6ae --- /dev/null +++ b/dstack/crates/mock-attestation/Cargo.toml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "mock-attestation" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +description = "Development-only cryptographically valid attestation evidence generator" + +[[bin]] +name = "dstack-mock-attestation" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +fs-err.workspace = true +rcgen.workspace = true +ciborium.workspace = true +p384 = { workspace = true, features = ["ecdsa", "pkcs8"] } +p256 = { workspace = true, features = ["ecdsa", "pkcs8"] } +sha2.workspace = true +sev.workspace = true +sev-snp-qvl.workspace = true +tpm-types.workspace = true +tpm-qvl.workspace = true +dstack-types.workspace = true +dcap-qvl.workspace = true +scale.workspace = true +yasna.workspace = true +chrono.workspace = true +rand.workspace = true +tokio = { workspace = true, features = ["full"] } +axum = "0.8" +hex.workspace = true +urlencoding = "2" +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +time.workspace = true + +[dev-dependencies] +tempfile.workspace = true +nsm-qvl.workspace = true diff --git a/dstack/crates/mock-attestation/README.md b/dstack/crates/mock-attestation/README.md new file mode 100644 index 000000000..9aafe6ddd --- /dev/null +++ b/dstack/crates/mock-attestation/README.md @@ -0,0 +1,79 @@ +# mock-attestation + +Development-only, cryptographically valid attestation evidence for CI and the +`dstack-tee-simulator` dev image. + +The crate derives a development PKI from a 32-byte seed and dynamically signs evidence for: + +- Intel TDX/DCAP, including TCB Info, QE Identity, PCK certificates and CRLs; +- AMD SEV-SNP, including ARK, ASK, VCEK and reports; +- TPM, including AK certificates, quotes, AIA and CRLs; +- AWS NSM, including the certificate bundle and COSE Sign1 document. + +All evidence is verified in the test suite by the production QVLs. These keys +are test credentials and must never be copied into a production image. + +## CLI + +Generate matching public roots and `sys-config-fragment.json`: + +```console +dstack-mock-attestation generate \ + --output ./mock-roots \ + --collateral-base-url http://HOST_REACHABLE_FROM_VERIFIER:8088 +``` + +Merge the generated fragment into the VM sys-config, then start the host-side +Mock PCCS/KDS/AIA service from that same config: + +```console +dstack-mock-attestation serve \ + --listen 127.0.0.1:8088 \ + --sys-config ./sys-config.json \ + --output ./active-mock-roots +``` + +The server exposes only production-shaped public collateral endpoints. Evidence +is generated inside the development guest by `dstack-tee-simulator`; the HTTP +service deliberately provides no unauthenticated signing endpoint. + +## Dev image + +Select the platform in `.sys-config.json`; omission defaults to TDX: + +```json +{ + "tee_simulator": { + "platform": "dstack-amd-sev-snp", + "mock_attestation_seed": "<64 hex characters>", + "collateral_base_url": "http://HOST_REACHABLE_FROM_VERIFIER:8088" + } +} +``` + +Valid values mirror the supported attestation modes: `dstack-tdx`, +`dstack-gcp-tdx`, `dstack-amd-sev-snp`, `dstack-nitro-enclave`, and +`dstack-aws-nitro-tpm`. The simulator exposes +the production guest ABI for the selected platform (TSM configfs, vTPM, or an +NSM CUSE character device); attester libraries contain no mock HTTP or +environment-variable path. The guest only reads the +seed from `.sys-config.json`; it never writes credentials or roots back into +`/dstack/.host-shared`. CI retains the generated public roots and mounts them +into verifier/KMS/gateway. The independently running host collateral service +reconstructs the same hierarchy from the seed. Configure it under +`[attestation.urls]`: TDX uses `pccs`, and SEV-SNP uses `amd_kds`. + +Every verifier process must also explicitly set +`attestation.insecure_allow_external_trust_anchors = true`. Merely mounting and +configuring a mock root is rejected at startup while this flag remains false. + +The seed adds only 64 hex bytes (the generated fragment is well below 1 KiB), +so the existing 32 KiB sys-config copy limit does not need to be enlarged. + +## Required negative tests + +The crate tests ensure every platform rejects a different root and modified +signed bytes. SEV-SNP additionally performs expected `report_data` comparison +inside its QVL API. TDX/TPM/NSM expose their authenticated binding field for +the caller; the production `dstack-attest` layer performs the final equality +check. diff --git a/dstack/crates/mock-attestation/src/lib.rs b/dstack/crates/mock-attestation/src/lib.rs new file mode 100644 index 000000000..54497aa1f --- /dev/null +++ b/dstack/crates/mock-attestation/src/lib.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! Development-only attestation PKI and evidence generation. +//! +//! Private keys produced by this crate are test material and must never be +//! installed in a production image. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use p256::pkcs8::EncodePrivateKey; +use rcgen::KeyPair; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256, Sha384}; + +pub mod nsm; +pub mod server; + +/// Fail closed when authenticated evidence is not bound to the caller's +/// challenge/report data. +pub fn ensure_report_data(actual: &[u8], expected: &[u8]) -> Result<()> { + anyhow::ensure!(actual == expected, "mock attestation report_data mismatch"); + Ok(()) +} +pub mod sev_snp; +pub mod tdx; +pub mod tpm; + +pub const MOCK_SEED_LEN: usize = 32; + +pub fn parse_seed(value: &str) -> Result<[u8; MOCK_SEED_LEN]> { + let bytes = hex::decode(value).context("mock attestation seed must be hex")?; + bytes + .try_into() + .map_err(|_| anyhow::anyhow!("mock attestation seed must be 32 bytes")) +} + +pub fn random_seed() -> [u8; MOCK_SEED_LEN] { + rand::random() +} + +pub(crate) fn p256_key(seed: &[u8; 32], label: &str) -> Result { + for counter in 0u32.. { + let bytes = Sha256::new() + .chain_update(b"dstack-mock-p256-v1") + .chain_update(seed) + .chain_update(label) + .chain_update(counter.to_be_bytes()) + .finalize(); + if let Ok(key) = p256::ecdsa::SigningKey::from_slice(&bytes) { + return Ok(KeyPair::from_pem(&key.to_pkcs8_pem(Default::default())?)?); + } + } + unreachable!() +} + +pub(crate) fn p384_key(seed: &[u8; 32], label: &str) -> Result { + for counter in 0u32.. { + let bytes = Sha384::new() + .chain_update(b"dstack-mock-p384-v1") + .chain_update(seed) + .chain_update(label) + .chain_update(counter.to_be_bytes()) + .finalize(); + if let Ok(key) = p384::ecdsa::SigningKey::from_slice(&bytes) { + return Ok(KeyPair::from_pem(&key.to_pkcs8_pem(Default::default())?)?); + } + } + unreachable!() +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssetManifest { + pub version: u32, + pub tdx_root_ca: PathBuf, + pub tpm_root_ca: PathBuf, + pub nsm_root_ca: PathBuf, + pub sev_snp_milan_root_ca: PathBuf, + pub sev_snp_genoa_root_ca: PathBuf, + pub sev_snp_turin_root_ca: PathBuf, +} + +impl AssetManifest { + pub fn write_to(&self, output: &Path) -> Result<()> { + fs_err::write( + output.join("manifest.json"), + serde_json::to_vec_pretty(self).context("failed to serialize asset manifest")?, + )?; + Ok(()) + } +} + +pub fn generate_assets(output: &Path) -> Result { + generate_assets_from_seed(output, random_seed(), "http://127.0.0.1:8088") +} + +pub fn generate_assets_from_seed( + output: &Path, + seed: [u8; 32], + base_url: &str, +) -> Result { + fs_err::create_dir_all(output)?; + let state = server::MockCollateralState::from_seed(seed, base_url)?; + state.write_roots(output)?; + let tdx = PathBuf::from("tdx-root-ca.pem"); + let tpm = PathBuf::from("tpm-root-ca.pem"); + let nsm = PathBuf::from("nsm-root-ca.pem"); + let milan = PathBuf::from("sev-snp-root-ca.pem"); + let genoa = milan.clone(); + let turin = milan.clone(); + let manifest = AssetManifest { + version: 1, + tdx_root_ca: tdx, + tpm_root_ca: tpm, + nsm_root_ca: nsm, + sev_snp_milan_root_ca: milan, + sev_snp_genoa_root_ca: genoa, + sev_snp_turin_root_ca: turin, + }; + manifest.write_to(output)?; + fs_err::write( + output.join("sys-config-fragment.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "tee_simulator": { "platform": "dstack-tdx", "mock_attestation_seed": hex::encode(seed), "collateral_base_url": base_url } + }))?, + )?; + Ok(manifest) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generates_all_platform_roots() { + let dir = tempfile::tempdir().unwrap(); + let manifest = generate_assets(dir.path()).unwrap(); + for path in [ + manifest.tdx_root_ca, + manifest.tpm_root_ca, + manifest.nsm_root_ca, + manifest.sev_snp_milan_root_ca, + manifest.sev_snp_genoa_root_ca, + manifest.sev_snp_turin_root_ca, + ] { + assert!(dir.path().join(path).exists()); + } + } +} diff --git a/dstack/crates/mock-attestation/src/main.rs b/dstack/crates/mock-attestation/src/main.rs new file mode 100644 index 000000000..457746508 --- /dev/null +++ b/dstack/crates/mock-attestation/src/main.rs @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::path::PathBuf; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command(about = "Generate development-only mock attestation assets")] +struct Args { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Generate PKI roots and a manifest for every supported platform. + Generate { + #[arg(long)] + output: PathBuf, + #[arg(long, default_value = "http://127.0.0.1:8088")] + collateral_base_url: String, + }, + /// Serve mock PCCS, AMD KDS and certificate collateral endpoints. + Serve { + #[arg(long, default_value = "127.0.0.1:8088")] + listen: std::net::SocketAddr, + /// Write the roots matching this server instance into this directory. + #[arg(long)] + output: Option, + /// Read the seed and collateral URL from a sys-config JSON file. + #[arg(long)] + sys_config: Option, + }, +} + +#[tokio::main] +async fn main() -> Result<()> { + match Args::parse().command { + Command::Generate { + output, + collateral_base_url, + } => { + mock_attestation::generate_assets_from_seed( + &output, + mock_attestation::random_seed(), + &collateral_base_url, + )?; + } + Command::Serve { + listen, + output, + sys_config, + } => { + let state = if let Some(path) = sys_config { + let config: dstack_types::SysConfig = serde_json::from_slice(&fs_err::read(path)?)?; + let config = config + .tee_simulator + .ok_or_else(|| anyhow::anyhow!("tee_simulator config missing"))?; + let seed = mock_attestation::parse_seed( + config + .mock_attestation_seed + .as_deref() + .ok_or_else(|| anyhow::anyhow!("mock_attestation_seed missing"))?, + )?; + let url = config + .collateral_base_url + .as_deref() + .unwrap_or("http://127.0.0.1:8088"); + std::sync::Arc::new(mock_attestation::server::MockCollateralState::from_seed( + seed, url, + )?) + } else { + std::sync::Arc::new(mock_attestation::server::MockCollateralState::new()?) + }; + if let Some(output) = output { + state.write_roots(&output)?; + } + mock_attestation::server::serve(listen, state).await?; + } + } + Ok(()) +} diff --git a/dstack/crates/mock-attestation/src/nsm.rs b/dstack/crates/mock-attestation/src/nsm.rs new file mode 100644 index 000000000..d8325c6e9 --- /dev/null +++ b/dstack/crates/mock-attestation/src/nsm.rs @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use p384::ecdsa::{signature::hazmat::PrehashSigner, Signature, SigningKey}; +use p384::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertifiedKey, DnType, IsCa, KeyPair, + KeyUsagePurpose, +}; +use serde::Serialize; +use sha2::{Digest, Sha384}; +use time::{Duration, OffsetDateTime}; + +/// In-memory NSM test hierarchy. A fresh leaf is issued so validity follows CI +/// wall-clock time rather than a checked-in fixture. +pub struct NsmGenerator { + root: Certificate, + root_key: KeyPair, + leaf: Certificate, + leaf_key: KeyPair, +} + +#[derive(Serialize)] +struct Document { + module_id: String, + digest: String, + timestamp: u64, + pcrs: BTreeMap>, + certificate: Vec, + cabundle: Vec>, + public_key: Option>, + user_data: Option>, + nonce: Option>, +} + +impl NsmGenerator { + pub fn new() -> Result { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result { + let CertifiedKey { + cert: root, + key_pair: root_key, + } = make_root(&seed)?; + let leaf_key = crate::p384_key(&seed, "nsm-leaf")?; + let mut params = CertificateParams::new(Vec::::new())?; + params + .distinguished_name + .push(DnType::CommonName, "mock.nsm.dstack"); + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + let (not_before, not_after) = validity(); + params.not_before = not_before; + params.not_after = not_after; + let leaf = params.signed_by(&leaf_key, &root, &root_key)?; + Ok(Self { + root, + root_key, + leaf, + leaf_key, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn attest(&self, report_data: &[u8]) -> Result> { + let pcrs = (0..=2) + .map(|index| (index, vec![index as u8; 48])) + .collect(); + self.attest_with_pcrs(report_data, pcrs) + } + + pub fn attest_with_pcrs( + &self, + report_data: &[u8], + pcrs: BTreeMap>, + ) -> Result> { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("system clock before UNIX epoch")? + .as_millis() as u64; + let document = Document { + module_id: "mock-nsm".into(), + digest: "SHA384".into(), + timestamp, + pcrs, + certificate: self.leaf.der().to_vec(), + cabundle: vec![self.root.der().to_vec()], + public_key: None, + user_data: Some(report_data.to_vec()), + nonce: None, + }; + let mut payload = Vec::new(); + ciborium::into_writer(&document, &mut payload)?; + + let mut protected = Vec::new(); + let protected_map = BTreeMap::from([(1i64, ciborium::Value::Integer((-35).into()))]); + ciborium::into_writer(&protected_map, &mut protected)?; + let sig_structure = ciborium::Value::Array(vec![ + ciborium::Value::Text("Signature1".into()), + ciborium::Value::Bytes(protected.clone()), + ciborium::Value::Bytes(Vec::new()), + ciborium::Value::Bytes(payload.clone()), + ]); + let mut to_sign = Vec::new(); + ciborium::into_writer(&sig_structure, &mut to_sign)?; + let digest = Sha384::digest(to_sign); + let signing_key = SigningKey::from_pkcs8_pem(&self.leaf_key.serialize_pem())?; + let signature: Signature = signing_key.sign_prehash(&digest)?; + + let cose = ciborium::Value::Array(vec![ + ciborium::Value::Bytes(protected), + ciborium::Value::Map(Vec::new()), + ciborium::Value::Bytes(payload), + ciborium::Value::Bytes(signature.to_bytes().to_vec()), + ]); + let mut encoded = Vec::new(); + ciborium::into_writer(&cose, &mut encoded)?; + Ok(encoded) + } +} + +fn make_root(seed: &[u8; 32]) -> Result { + let key_pair = crate::p384_key(seed, "nsm-root")?; + let mut params = CertificateParams::new(Vec::::new())?; + params + .distinguished_name + .push(DnType::CommonName, "Mock AWS Nitro Enclaves Root CA"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let (not_before, not_after) = validity(); + params.not_before = not_before; + params.not_after = not_after; + let cert = params.self_signed(&key_pair)?; + Ok(CertifiedKey { cert, key_pair }) +} + +fn validity() -> (OffsetDateTime, OffsetDateTime) { + let now = OffsetDateTime::now_utc(); + (now - Duration::days(1), now + Duration::days(30)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_document_passes_real_qvl_and_negative_cases_fail() { + let generator = NsmGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(&report_data).unwrap(); + let verifier = nsm_qvl::QuoteVerifier::new(generator.root_ca_pem()); + let verified = verifier.verify(&evidence, None, None).unwrap(); + assert_eq!(verified.user_data.as_deref(), Some(report_data.as_slice())); + + let wrong = NsmGenerator::new().unwrap(); + assert!(nsm_qvl::QuoteVerifier::new(wrong.root_ca_pem()) + .verify(&evidence, None, None) + .is_err()); + + let mut tampered = evidence; + *tampered.last_mut().unwrap() ^= 1; + assert!(verifier.verify(&tampered, None, None).is_err()); + assert!( + crate::ensure_report_data(verified.user_data.as_deref().unwrap(), &[0x24; 64]).is_err() + ); + } +} diff --git a/dstack/crates/mock-attestation/src/server.rs b/dstack/crates/mock-attestation/src/server.rs new file mode 100644 index 000000000..19480b3e0 --- /dev/null +++ b/dstack/crates/mock-attestation/src/server.rs @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::net::SocketAddr; +use std::sync::Arc; + +use anyhow::Result; +use axum::{ + body::Body, + extract::State, + http::{header::CONTENT_TYPE, HeaderValue, Response, StatusCode}, + response::IntoResponse, + routing::get, + Router, +}; +use serde_json::json; + +use crate::{nsm::NsmGenerator, sev_snp::SevSnpGenerator, tdx::TdxGenerator, tpm::TpmGenerator}; + +pub struct MockCollateralState { + pub tdx: Arc, + pub sev_snp: Arc, + pub tpm: Arc, + pub nsm: Arc, +} + +impl MockCollateralState { + pub fn new() -> Result { + Self::with_base_url("http://127.0.0.1:8088") + } + + pub fn with_base_url(base_url: &str) -> Result { + Self::from_seed(rand::random(), base_url) + } + + pub fn from_seed(seed: [u8; 32], base_url: &str) -> Result { + Ok(Self { + tdx: Arc::new(TdxGenerator::from_seed(seed)?), + sev_snp: Arc::new(SevSnpGenerator::from_seed(seed)?), + tpm: Arc::new(TpmGenerator::from_seed(seed, base_url)?), + nsm: Arc::new(NsmGenerator::from_seed(seed)?), + }) + } + + pub fn write_roots(&self, output: &std::path::Path) -> Result<()> { + fs_err::create_dir_all(output)?; + fs_err::write(output.join("tdx-root-ca.pem"), self.tdx.root_ca_pem())?; + fs_err::write( + output.join("sev-snp-root-ca.pem"), + self.sev_snp.root_ca_pem(), + )?; + fs_err::write(output.join("tpm-root-ca.pem"), self.tpm.root_ca_pem())?; + fs_err::write(output.join("nsm-root-ca.pem"), self.nsm.root_ca_pem())?; + Ok(()) + } +} + +pub fn router(state: Arc) -> Router { + Router::new() + .route("/sgx/certification/v4/pckcrl", get(pck_crl)) + .route("/sgx/certification/v4/rootcacrl", get(pccs_root_crl)) + .route("/tdx/certification/v4/tcb", get(tcb_info)) + .route("/tdx/certification/v4/qe/identity", get(qe_identity)) + .route("/vcek/v1/Milan/cert_chain", get(sev_ca_chain)) + .route("/vcek/v1/Milan/{chip_id}", get(sev_vcek)) + .route("/tpm/aia/root.pem", get(tpm_root)) + .route("/tpm/aia/intermediate.der", get(tpm_intermediate)) + .route("/tpm/crl/root.crl", get(tpm_root_crl)) + .route("/tpm/crl/intermediate.crl", get(tpm_intermediate_crl)) + .with_state(state) +} + +pub async fn serve(addr: SocketAddr, state: Arc) -> Result<()> { + let listener = tokio::net::TcpListener::bind(addr).await?; + serve_listener(listener, state).await +} + +pub async fn serve_listener( + listener: tokio::net::TcpListener, + state: Arc, +) -> Result<()> { + axum::serve(listener, router(state)).await?; + Ok(()) +} + +async fn pccs_root_crl(State(state): State>) -> impl IntoResponse { + binary(hex::encode(state.tdx.root_crl_der()).into_bytes(), None) +} + +async fn pck_crl(State(state): State>) -> impl IntoResponse { + binary( + state.tdx.root_crl_der(), + Some(("SGX-PCK-CRL-Issuer-Chain", state.tdx.root_ca_pem())), + ) +} + +async fn tcb_info(State(state): State>) -> impl IntoResponse { + let Ok(collateral) = state.tdx.sample_collateral() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(tcb_info) = serde_json::from_str::(&collateral.tcb_info) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + json_response( + json!({"tcbInfo": tcb_info, "signature": hex::encode(collateral.tcb_info_signature)}), + Some(( + "SGX-TCB-Info-Issuer-Chain", + collateral.tcb_info_issuer_chain, + )), + ) +} + +async fn qe_identity(State(state): State>) -> impl IntoResponse { + let Ok(collateral) = state.tdx.sample_collateral() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(qe_identity) = serde_json::from_str::(&collateral.qe_identity) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + json_response( + json!({"enclaveIdentity": qe_identity, "signature": hex::encode(collateral.qe_identity_signature)}), + Some(( + "SGX-Enclave-Identity-Issuer-Chain", + collateral.qe_identity_issuer_chain, + )), + ) +} + +async fn sev_ca_chain(State(state): State>) -> impl IntoResponse { + binary(state.sev_snp.ca_chain_pem(), None) +} + +async fn sev_vcek(State(state): State>) -> impl IntoResponse { + binary(state.sev_snp.vcek_der(), None) +} + +async fn tpm_root(State(state): State>) -> impl IntoResponse { + binary(state.tpm.root_ca_der(), None) +} +async fn tpm_intermediate(State(state): State>) -> impl IntoResponse { + binary(state.tpm.intermediate_der(), None) +} +async fn tpm_root_crl(State(state): State>) -> impl IntoResponse { + binary(state.tpm.root_crl_der(), None) +} +async fn tpm_intermediate_crl(State(state): State>) -> impl IntoResponse { + binary(state.tpm.intermediate_crl_der(), None) +} + +fn json_response(value: serde_json::Value, header: Option<(&str, String)>) -> Response { + let mut response = Response::new(Body::from(value.to_string())); + *response.status_mut() = StatusCode::OK; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + if let Some((name, value)) = header { + let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(value) = HeaderValue::from_str(&urlencoding::encode(&value)) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + response.headers_mut().insert(name, value); + } + response +} + +fn binary(value: Vec, header: Option<(&str, String)>) -> Response { + let mut response = Response::new(Body::from(value)); + *response.status_mut() = StatusCode::OK; + if let Some((name, value)) = header { + let Ok(name) = axum::http::HeaderName::from_bytes(name.as_bytes()) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Ok(value) = HeaderValue::from_str(&urlencoding::encode(&value)) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + response.headers_mut().insert(name, value); + } + response +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn independent_host_and_guest_from_sys_config_seed_interoperate() { + let seed = [0x5a; 32]; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let url = format!("http://{addr}"); + // These states model separate host and CVM processes. No certificates, + // keys, or Arc state are exchanged after construction. + let host = Arc::new(MockCollateralState::from_seed(seed, &url).unwrap()); + let guest = MockCollateralState::from_seed(seed, &url).unwrap(); + let task = tokio::spawn(serve_listener(listener, host.clone())); + + let report_data = [0x61; 64]; + let tdx = guest.tdx.attest(report_data).unwrap(); + let collateral = dcap_qvl::collateral::CollateralClient::with_default_http(&url) + .unwrap() + .fetch(&tdx.quote) + .await + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let tdx_verifier = dcap_qvl::verify::QuoteVerifier::new(host.tdx.root_ca_der()); + tdx_verifier.verify(&tdx.quote, &collateral, now).unwrap(); + let mut tampered = tdx.quote.clone(); + tampered[200] ^= 1; + assert!(tdx_verifier.verify(&tampered, &collateral, now).is_err()); + + let sev = guest.sev_snp.attest(report_data).unwrap(); + let sev_verifier = sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + host.sev_snp.root_ca_pem().into_bytes(), + ); + sev_verifier + .verify(&sev.report, &sev.cert_chain, &report_data) + .unwrap(); + assert!(sev_verifier + .verify(&sev.report, &sev.cert_chain, &[0x62; 64]) + .is_err()); + + let qualifying = [0x63; 32]; + let tpm = guest.tpm.attest(&qualifying).unwrap(); + let tpm_verifier = tpm_qvl::QuoteVerifier::new(host.tpm.root_ca_pem()); + tpm_verifier.verify(&tpm, &host.tpm.collateral()).unwrap(); + assert!(crate::ensure_report_data(&qualifying, &[0x64; 32]).is_err()); + + let nsm = guest.nsm.attest(&report_data).unwrap(); + let verified = nsm_qvl::QuoteVerifier::new(host.nsm.root_ca_pem()) + .verify(&nsm, None, None) + .unwrap(); + assert_eq!(verified.user_data.as_deref(), Some(report_data.as_slice())); + + let wrong = MockCollateralState::from_seed([0xa5; 32], &url).unwrap(); + assert!(nsm_qvl::QuoteVerifier::new(wrong.nsm.root_ca_pem()) + .verify(&nsm, None, None) + .is_err()); + task.abort(); + } + + #[tokio::test] + async fn mock_pccs_and_kds_drive_real_qvls() { + let state = Arc::new(MockCollateralState::new().unwrap()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(serve_listener(listener, state.clone())); + let base = format!("http://{addr}"); + + let tdx_evidence = state.tdx.attest([0x42; 64]).unwrap(); + let collateral = dcap_qvl::collateral::CollateralClient::with_default_http(&base) + .unwrap() + .fetch(&tdx_evidence.quote) + .await + .unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(state.tdx.root_ca_der()) + .verify(&tdx_evidence.quote, &collateral, now) + .unwrap(); + + let sev_evidence = state.sev_snp.attest([0x43; 64]).unwrap(); + let kds = sev_snp_qvl::AmdKdsClient::with_base_url(format!("{base}/vcek/v1")).unwrap(); + sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + state.sev_snp.root_ca_pem().into_bytes(), + ) + .fetch_and_verify(&kds, &sev_evidence.report, &[], &[0x43; 64]) + .await + .unwrap(); + task.abort(); + } + + #[tokio::test] + async fn tpm_aia_and_crl_service_builds_verifiable_collateral() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let state = + Arc::new(MockCollateralState::with_base_url(&format!("http://{addr}")).unwrap()); + let task = tokio::spawn(serve_listener(listener, state.clone())); + let quote = state.tpm.attest(&[0x42; 32]).unwrap(); + let root = state.tpm.root_ca_pem(); + let q = quote.clone(); + let collateral = tokio::task::spawn_blocking(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime + .block_on(tpm_qvl::get_collateral(&q, &root)) + .unwrap() + }) + .await + .unwrap(); + tpm_qvl::QuoteVerifier::new(state.tpm.root_ca_pem()) + .verify("e, &collateral) + .unwrap(); + task.abort(); + } +} diff --git a/dstack/crates/mock-attestation/src/sev_snp.rs b/dstack/crates/mock-attestation/src/sev_snp.rs new file mode 100644 index 000000000..720078198 --- /dev/null +++ b/dstack/crates/mock-attestation/src/sev_snp.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use p384::ecdsa::{signature::DigestSigner, Signature as P384Signature, SigningKey}; +use p384::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertifiedKey, DnType, IsCa, KeyPair, + KeyUsagePurpose, +}; +use sev::firmware::guest::AttestationReport; +use sha2::{Digest, Sha384}; +use time::{Duration, OffsetDateTime}; + +pub struct SevSnpGenerator { + root: Certificate, + root_key: KeyPair, + ask: Certificate, + vcek: Certificate, + vcek_key: KeyPair, +} + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct SevSnpEvidence { + pub report: Vec, + pub cert_chain: Vec>, +} + +impl SevSnpGenerator { + pub fn new() -> Result { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result { + let CertifiedKey { + cert: root, + key_pair: root_key, + } = make_ca("Mock AMD Milan ARK", "sev-root", &seed, None)?; + let CertifiedKey { + cert: ask, + key_pair: ask_key, + } = make_ca( + "Mock AMD Milan ASK", + "sev-ask", + &seed, + Some((&root, &root_key)), + )?; + let vcek_key = crate::p384_key(&seed, "sev-vcek")?; + let mut params = base_params("Mock AMD VCEK")?; + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + let vcek = params.signed_by(&vcek_key, &ask, &ask_key)?; + Ok(Self { + root, + root_key, + ask, + vcek, + vcek_key, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn ca_chain_pem(&self) -> Vec { + format!("{}{}", self.ask.pem(), self.root.pem()).into_bytes() + } + + pub fn vcek_der(&self) -> Vec { + self.vcek.der().to_vec() + } + + pub fn attest(&self, report_data: [u8; 64]) -> Result { + self.attest_with_host_data(report_data, [0x22; 32]) + } + + pub fn attest_with_host_data( + &self, + report_data: [u8; 64], + host_data: [u8; 32], + ) -> Result { + self.attest_with_measurement(report_data, host_data, [0x33; 48]) + } + + pub fn attest_with_measurement( + &self, + report_data: [u8; 64], + host_data: [u8; 32], + measurement: [u8; 48], + ) -> Result { + let mut encoded = Vec::new(); + AttestationReport::default().write_bytes(&mut encoded)?; + encoded[0..4].copy_from_slice(&2u32.to_le_bytes()); + encoded[52..56].copy_from_slice(&1u32.to_le_bytes()); + encoded[0x50..0x90].copy_from_slice(&report_data); + encoded[0x90..0xc0].copy_from_slice(&measurement); + encoded[0xc0..0xe0].copy_from_slice(&host_data); + encoded[0x1a0..0x1e0].fill(0x33); + let signing_key = SigningKey::from_pkcs8_pem(&self.vcek_key.serialize_pem())?; + let signature: P384Signature = + signing_key.sign_digest(Sha384::new_with_prefix(&encoded[..0x2a0])); + write_amd_signature(&mut encoded[0x2a0..], &signature); + Ok(SevSnpEvidence { + report: encoded, + cert_chain: vec![self.ask.pem().into_bytes(), self.vcek.pem().into_bytes()], + }) + } +} + +fn make_ca( + name: &str, + label: &str, + seed: &[u8; 32], + issuer: Option<(&Certificate, &KeyPair)>, +) -> Result { + let key_pair = crate::p384_key(seed, label)?; + let mut params = base_params(name)?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let cert = match issuer { + Some((issuer, issuer_key)) => params.signed_by(&key_pair, issuer, issuer_key)?, + None => params.self_signed(&key_pair)?, + }; + Ok(CertifiedKey { cert, key_pair }) +} + +fn base_params(name: &str) -> Result { + let mut params = CertificateParams::new(Vec::::new())?; + params.distinguished_name.push(DnType::CommonName, name); + let now = OffsetDateTime::now_utc(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(30); + Ok(params) +} + +fn write_amd_signature(output: &mut [u8], signature: &P384Signature) { + let bytes = signature.to_bytes(); + output[..144].fill(0); + for (dst, src) in output[..48].iter_mut().zip(bytes[..48].iter().rev()) { + *dst = *src; + } + for (dst, src) in output[72..120].iter_mut().zip(bytes[48..].iter().rev()) { + *dst = *src; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_report_passes_real_qvl_and_negative_cases_fail() { + let generator = SevSnpGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(report_data).unwrap(); + let verifier = sev_snp_qvl::QuoteVerifier::new( + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + generator.root_ca_pem().into_bytes(), + ); + verifier + .verify(&evidence.report, &evidence.cert_chain, &report_data) + .unwrap(); + + let wrong = SevSnpGenerator::new().unwrap(); + let wrong_verifier = sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + wrong.root_ca_pem().into_bytes(), + ); + assert!(wrong_verifier + .verify(&evidence.report, &evidence.cert_chain, &report_data) + .is_err()); + let mut tampered = evidence.report.clone(); + tampered[0x100] ^= 1; + assert!(verifier + .verify(&tampered, &evidence.cert_chain, &report_data) + .is_err()); + assert!(verifier + .verify(&evidence.report, &evidence.cert_chain, &[0x24; 64]) + .is_err()); + } +} diff --git a/dstack/crates/mock-attestation/src/tdx.rs b/dstack/crates/mock-attestation/src/tdx.rs new file mode 100644 index 000000000..9ae50a513 --- /dev/null +++ b/dstack/crates/mock-attestation/src/tdx.rs @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use dcap_qvl::quote::{ + AuthData, AuthDataV4, CertificationData, Data, EnclaveReport, Header, + QEReportCertificationData, Quote, Report, TDReport10, +}; +use dcap_qvl::QuoteCollateralV3; +use p256::ecdsa::{signature::Signer, Signature, SigningKey}; +use p256::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertificateRevocationListParams, + CertifiedKey, CustomExtension, DnType, ExtendedKeyUsagePurpose, IsCa, KeyIdMethod, KeyPair, + KeyUsagePurpose, SerialNumber, +}; +use scale::Encode; +use serde_json::json; +use sha2::{Digest, Sha256}; +use time::{Duration, OffsetDateTime}; + +const INTEL_QE_VENDOR_ID: [u8; 16] = [ + 0x93, 0x9a, 0x72, 0x33, 0xf7, 0x9c, 0x4c, 0xa9, 0x94, 0x0a, 0x0d, 0xb3, 0x95, 0x7f, 0x06, 0x07, +]; + +pub struct TdxGenerator { + root: Certificate, + root_key: KeyPair, + pck: Certificate, + pck_key: KeyPair, + tcb_signer: Certificate, + tcb_signer_key: KeyPair, + qe_signer: Certificate, + qe_signer_key: KeyPair, + root_crl: Vec, +} + +pub struct TdxEvidence { + pub quote: Vec, + pub collateral: QuoteCollateralV3, +} + +impl TdxGenerator { + pub fn new() -> Result { + Self::from_seed(rand::random()) + } + + pub fn from_seed(seed: [u8; 32]) -> Result { + let CertifiedKey { + cert: root, + key_pair: root_key, + } = make_root(&seed)?; + let (pck, pck_key) = make_leaf( + "Mock Intel SGX PCK Certificate", + "tdx-pck", + &seed, + &root, + &root_key, + true, + )?; + let (tcb_signer, tcb_signer_key) = make_leaf( + "Mock Intel SGX TCB Signing", + "tdx-tcb", + &seed, + &root, + &root_key, + false, + )?; + let (qe_signer, qe_signer_key) = make_leaf( + "Mock Intel SGX QE Identity Signing", + "tdx-qe", + &seed, + &root, + &root_key, + false, + )?; + let now = OffsetDateTime::now_utc(); + let root_crl = CertificateRevocationListParams { + this_update: now - Duration::days(1), + next_update: now + Duration::days(30), + crl_number: SerialNumber::from(1u64), + issuing_distribution_point: None, + revoked_certs: Vec::new(), + key_identifier_method: KeyIdMethod::Sha256, + } + .signed_by(&root, &root_key)? + .der() + .to_vec(); + Ok(Self { + root, + root_key, + pck, + pck_key, + tcb_signer, + tcb_signer_key, + qe_signer, + qe_signer_key, + root_crl, + }) + } + + pub fn root_ca_der(&self) -> Vec { + self.root.der().to_vec() + } + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn sample_collateral(&self) -> Result { + self.collateral() + } + + pub fn root_crl_der(&self) -> Vec { + self.root_crl.clone() + } + + pub fn attest(&self, report_data: [u8; 64]) -> Result { + self.attest_with_rtmrs( + report_data, + [[0x20; 48], [0x21; 48], [0x22; 48], [0x23; 48]], + ) + } + + pub fn attest_with_rtmrs( + &self, + report_data: [u8; 64], + rtmrs: [[u8; 48]; 4], + ) -> Result { + self.attest_with_measurements(report_data, [0x11; 48], rtmrs) + } + + pub fn attest_with_measurements( + &self, + report_data: [u8; 64], + mrtd: [u8; 48], + rtmrs: [[u8; 48]; 4], + ) -> Result { + let auth_key = SigningKey::random(&mut rand::thread_rng()); + let auth_pub = auth_key.verifying_key().to_encoded_point(false); + let auth_pub: [u8; 64] = auth_pub.as_bytes()[1..] + .try_into() + .map_err(|_| anyhow::anyhow!("invalid P-256 public key length"))?; + let qe_auth = vec![0u8; 32]; + let mut qe_hash_input = Vec::from(auth_pub); + qe_hash_input.extend_from_slice(&qe_auth); + let mut qe_report = EnclaveReport { + cpu_svn: [0; 16], + misc_select: 0, + reserved1: [0; 28], + attributes: [0; 16], + mr_enclave: [0; 32], + reserved2: [0; 32], + mr_signer: [0x44; 32], + reserved3: [0; 96], + isv_prod_id: 1, + isv_svn: 1, + reserved4: [0; 60], + report_data: [0; 64], + }; + qe_report.report_data[..32].copy_from_slice(&Sha256::digest(qe_hash_input)); + let qe_report_bytes: [u8; 384] = qe_report + .encode() + .try_into() + .map_err(|bytes: Vec| anyhow::anyhow!("invalid QE report size {}", bytes.len()))?; + let pck_key = signing_key(&self.pck_key)?; + let qe_sig: Signature = pck_key.sign(&qe_report_bytes); + + let pck_chain = format!("{}{}", self.pck.pem(), self.root.pem()).into_bytes(); + let qe_certification = QEReportCertificationData { + qe_report: qe_report_bytes, + qe_report_signature: qe_sig.to_bytes().into(), + qe_auth_data: Data::new(qe_auth), + certification_data: CertificationData { + cert_type: 5, + body: Data::new(pck_chain), + }, + }; + let td_report = TDReport10 { + tee_tcb_svn: [0; 16], + mr_seam: [0; 48], + mr_signer_seam: [0; 48], + seam_attributes: [0; 8], + td_attributes: [0, 0, 0, 0x10, 0, 0, 0, 0], + xfam: [0; 8], + mr_td: mrtd, + mr_config_id: [0; 48], + mr_owner: [0; 48], + mr_owner_config: [0; 48], + rt_mr0: rtmrs[0], + rt_mr1: rtmrs[1], + rt_mr2: rtmrs[2], + rt_mr3: rtmrs[3], + report_data, + }; + let header = Header { + version: 4, + attestation_key_type: 2, + tee_type: 0x81, + qe_svn: 1, + pce_svn: 0, + qe_vendor_id: INTEL_QE_VENDOR_ID, + user_data: [0; 20], + }; + let mut auth = AuthDataV4 { + ecdsa_signature: [0; 64], + ecdsa_attestation_key: auth_pub, + certification_data: CertificationData { + cert_type: 6, + body: Data::new(qe_certification.encode()), + }, + qe_report_data: qe_certification, + }; + let mut quote = Quote { + header, + report: Report::TD10(td_report), + auth_data: AuthData::V4(auth.clone()), + }; + let raw = quote.encode(); + let quote_sig: Signature = auth_key.sign(&raw[..quote.signed_length()]); + auth.ecdsa_signature = quote_sig.to_bytes().into(); + quote.auth_data = AuthData::V4(auth); + + Ok(TdxEvidence { + quote: quote.encode(), + collateral: self.collateral()?, + }) + } + + fn collateral(&self) -> Result { + let now = chrono::Utc::now(); + let issue = + (now - chrono::Duration::days(1)).to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let next = + (now + chrono::Duration::days(30)).to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let tcb_info = json!({ + "id":"TDX", "version":3, "issueDate":issue, "nextUpdate":next, + "fmspc":"000000000000", "pceId":"0000", "tcbType":0, "tcbEvaluationDataNumber":1, + "tdxModule":{"mrsigner":"00".repeat(48),"attributes":"00".repeat(8),"attributesMask":"00".repeat(8)}, + "tcbLevels":[{"tcb":{"sgxtcbcomponents":vec![json!({"svn":0});16],"pcesvn":0,"tdxtcbcomponents":vec![json!({"svn":0});16]},"tcbDate":issue,"tcbStatus":"UpToDate"}] + }).to_string(); + let qe_identity = json!({ + "id":"TD_QE", "version":2, "issueDate":issue, "nextUpdate":next, + "tcbEvaluationDataNumber":1, "miscselect":"00000000", "miscselectMask":"00000000", + "attributes":"00".repeat(16), "attributesMask":"00".repeat(16), "mrsigner":"44".repeat(32), + "isvprodid":1, "tcbLevels":[{"tcb":{"isvsvn":1},"tcbDate":issue,"tcbStatus":"UpToDate"}] + }).to_string(); + Ok(QuoteCollateralV3 { + pck_crl_issuer_chain: self.root.pem(), + root_ca_crl: self.root_crl.clone(), + pck_crl: self.root_crl.clone(), + tcb_info_issuer_chain: format!("{}{}", self.tcb_signer.pem(), self.root.pem()), + tcb_info_signature: sign_raw(&self.tcb_signer_key, tcb_info.as_bytes())?, + tcb_info, + qe_identity_issuer_chain: format!("{}{}", self.qe_signer.pem(), self.root.pem()), + qe_identity_signature: sign_raw(&self.qe_signer_key, qe_identity.as_bytes())?, + qe_identity, + pck_certificate_chain: None, + }) + } +} + +fn make_root(seed: &[u8; 32]) -> Result { + let key_pair = crate::p256_key(seed, "tdx-root")?; + let mut params = cert_params("Mock Intel SGX Root CA")?; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + let cert = params.self_signed(&key_pair)?; + Ok(CertifiedKey { cert, key_pair }) +} + +fn make_leaf( + name: &str, + label: &str, + seed: &[u8; 32], + root: &Certificate, + root_key: &KeyPair, + pck: bool, +) -> Result<(Certificate, KeyPair)> { + let key = crate::p256_key(seed, label)?; + let mut params = cert_params(name)?; + params.key_usages.push(KeyUsagePurpose::DigitalSignature); + params + .extended_key_usages + .push(ExtendedKeyUsagePurpose::ServerAuth); + if pck { + params.custom_extensions.push(pck_extension()); + } + let cert = params.signed_by(&key, root, root_key)?; + Ok((cert, key)) +} + +fn cert_params(name: &str) -> Result { + let mut params = CertificateParams::new(vec!["mock.dstack.invalid".into()])?; + params.distinguished_name.push(DnType::CommonName, name); + params.serial_number = Some(SerialNumber::from(42u64)); + let now = OffsetDateTime::now_utc(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(30); + Ok(params) +} + +fn pck_extension() -> CustomExtension { + fn oid(writer: yasna::DERWriter, oid: &[u64]) { + writer.write_oid(&yasna::models::ObjectIdentifier::from_slice(oid)); + } + let der = yasna::construct_der(|writer| { + writer.write_sequence(|writer| { + let entries: Vec<(&[u64], Vec, u8)> = vec![ + (&[1, 2, 840, 113741, 1, 13, 1, 1], vec![0; 16], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 3], vec![0; 2], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 4], vec![0; 6], 0), + (&[1, 2, 840, 113741, 1, 13, 1, 5], vec![0], 2), + ]; + for (entry_oid, value, kind) in entries { + writer.next().write_sequence(|w| { + oid(w.next(), entry_oid); + if kind == 2 { + w.next().write_enum(0) + } else { + w.next().write_bytes(&value) + } + }); + } + writer.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2]); + w.next().write_sequence(|w| { + w.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2, 17]); + w.next().write_u8(0); + }); + w.next().write_sequence(|w| { + oid(w.next(), &[1, 2, 840, 113741, 1, 13, 1, 2, 18]); + w.next().write_bytes(&[0; 16]); + }); + }); + }); + }) + }); + CustomExtension::from_oid_content(&[1, 2, 840, 113741, 1, 13, 1], der) +} + +fn signing_key(key: &KeyPair) -> Result { + Ok(SigningKey::from_pkcs8_pem(&key.serialize_pem())?) +} +fn sign_raw(key: &KeyPair, message: &[u8]) -> Result> { + let sig: Signature = signing_key(key)?.sign(message); + Ok(sig.to_bytes().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_quote_passes_real_qvl_and_negative_cases_fail() { + let generator = TdxGenerator::new().unwrap(); + let report_data = [0x42; 64]; + let evidence = generator.attest(report_data).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let verifier = dcap_qvl::verify::QuoteVerifier::new(generator.root_ca_der()); + let verified = verifier + .verify(&evidence.quote, &evidence.collateral, now) + .unwrap(); + assert_eq!(verified.report.as_td10().unwrap().report_data, report_data); + assert!( + dcap_qvl::verify::QuoteVerifier::new(TdxGenerator::new().unwrap().root_ca_der()) + .verify(&evidence.quote, &evidence.collateral, now) + .is_err() + ); + let mut tampered = evidence.quote.clone(); + tampered[100] ^= 1; + assert!(verifier + .verify(&tampered, &evidence.collateral, now) + .is_err()); + assert!(crate::ensure_report_data( + &verified.report.as_td10().unwrap().report_data, + &[0x24; 64] + ) + .is_err()); + } +} diff --git a/dstack/crates/mock-attestation/src/tpm.rs b/dstack/crates/mock-attestation/src/tpm.rs new file mode 100644 index 000000000..78b038dec --- /dev/null +++ b/dstack/crates/mock-attestation/src/tpm.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use dstack_types::Platform; +use p256::ecdsa::{signature::hazmat::PrehashSigner, Signature, SigningKey}; +use p256::pkcs8::DecodePrivateKey; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, CertificateRevocationListParams, + CrlDistributionPoint, CustomExtension, DnType, IsCa, KeyIdMethod, KeyPair, KeyUsagePurpose, + SerialNumber, +}; +use sha2::{Digest, Sha256}; +use time::{Duration, OffsetDateTime}; +use tpm_qvl::QuoteCollateral; +use tpm_types::{PcrValue, TpmQuote}; + +pub struct TpmGenerator { + root: Certificate, + root_key: KeyPair, + intermediate: Certificate, + ak: Certificate, + ak_key: KeyPair, + intermediate_crl: Vec, + root_crl: Vec, +} + +impl TpmGenerator { + pub fn new() -> Result { + Self::with_base_url("http://127.0.0.1:8088") + } + + pub fn with_base_url(base_url: &str) -> Result { + Self::from_seed(rand::random(), base_url) + } + + pub fn from_seed(seed: [u8; 32], base_url: &str) -> Result { + let root_key = crate::p256_key(&seed, "tpm-root")?; + let mut root_params = params("Mock TPM Root CA")?; + root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root_params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + root_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/root.crl")], + }); + let root = root_params.self_signed(&root_key)?; + + let intermediate_key = crate::p256_key(&seed, "tpm-intermediate")?; + let mut intermediate_params = params("Mock TPM Attestation Intermediate CA")?; + intermediate_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + intermediate_params.key_usages.extend([ + KeyUsagePurpose::DigitalSignature, + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + ]); + intermediate_params + .custom_extensions + .push(aia(&format!("{base_url}/tpm/aia/root.pem"))); + intermediate_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/root.crl")], + }); + let intermediate = intermediate_params.signed_by(&intermediate_key, &root, &root_key)?; + let ak_key = crate::p256_key(&seed, "tpm-ak")?; + let mut ak_params = params("Mock TPM Attestation Key")?; + ak_params.key_usages.push(KeyUsagePurpose::DigitalSignature); + ak_params + .custom_extensions + .push(aia(&format!("{base_url}/tpm/aia/intermediate.der"))); + ak_params + .crl_distribution_points + .push(CrlDistributionPoint { + uris: vec![format!("{base_url}/tpm/crl/intermediate.crl")], + }); + let ak = ak_params.signed_by(&ak_key, &intermediate, &intermediate_key)?; + let now = OffsetDateTime::now_utc(); + let crl = |issuer: &Certificate, key: &KeyPair, number| -> Result> { + Ok(CertificateRevocationListParams { + this_update: now - Duration::days(1), + next_update: now + Duration::days(30), + crl_number: SerialNumber::from(number), + issuing_distribution_point: None, + revoked_certs: Vec::new(), + key_identifier_method: KeyIdMethod::Sha256, + } + .signed_by(issuer, key)? + .der() + .to_vec()) + }; + let root_crl = crl(&root, &root_key, 1)?; + let intermediate_crl = crl(&intermediate, &intermediate_key, 2)?; + Ok(Self { + root, + root_key, + intermediate, + ak, + ak_key, + intermediate_crl, + root_crl, + }) + } + + pub fn root_ca_pem(&self) -> String { + self.root.pem() + } + + pub fn root_ca_der(&self) -> Vec { + self.root.der().to_vec() + } + + pub fn root_key_pem(&self) -> String { + self.root_key.serialize_pem() + } + + pub fn collateral(&self) -> QuoteCollateral { + QuoteCollateral { + cert_chain_pem: self.intermediate.pem(), + crls: vec![self.intermediate_crl.clone()], + root_ca_crl: Some(self.root_crl.clone()), + } + } + + pub fn intermediate_der(&self) -> Vec { + self.intermediate.der().to_vec() + } + pub fn intermediate_crl_der(&self) -> Vec { + self.intermediate_crl.clone() + } + pub fn root_crl_der(&self) -> Vec { + self.root_crl.clone() + } + + pub fn attest(&self, qualifying_data: &[u8]) -> Result { + let pcr = PcrValue { + index: 14, + algorithm: "sha256".into(), + value: vec![0x11; 32], + }; + let pcr_digest = Sha256::digest(&pcr.value); + let mut message = Vec::new(); + message.extend_from_slice(&0xff54_4347u32.to_be_bytes()); + message.extend_from_slice(&0x8018u16.to_be_bytes()); + message.extend_from_slice(&0u16.to_be_bytes()); // qualified signer + message.extend_from_slice(&(qualifying_data.len() as u16).to_be_bytes()); + message.extend_from_slice(qualifying_data); + message.extend_from_slice(&0u64.to_be_bytes()); // clock + message.extend_from_slice(&0u32.to_be_bytes()); // reset count + message.extend_from_slice(&0u32.to_be_bytes()); // restart count + message.push(1); // safe + message.extend_from_slice(&0u64.to_be_bytes()); // firmware + message.extend_from_slice(&1u32.to_be_bytes()); // selection count + message.extend_from_slice(&0x000bu16.to_be_bytes()); + message.push(3); + message.extend_from_slice(&[0, 0x40, 0]); // PCR14 + message.extend_from_slice(&(pcr_digest.len() as u16).to_be_bytes()); + message.extend_from_slice(&pcr_digest); + + let digest = Sha256::digest(&message); + let key = SigningKey::from_pkcs8_pem(&self.ak_key.serialize_pem())?; + let signature: Signature = key.sign_prehash(&digest)?; + let bytes = signature.to_bytes(); + let mut tpm_signature = Vec::with_capacity(72); + tpm_signature.extend_from_slice(&0x0018u16.to_be_bytes()); + tpm_signature.extend_from_slice(&0x000bu16.to_be_bytes()); + tpm_signature.extend_from_slice(&32u16.to_be_bytes()); + tpm_signature.extend_from_slice(&bytes[..32]); + tpm_signature.extend_from_slice(&32u16.to_be_bytes()); + tpm_signature.extend_from_slice(&bytes[32..]); + + Ok(TpmQuote { + message, + signature: tpm_signature, + pcr_values: vec![pcr], + ak_cert: self.ak.der().to_vec(), + platform: Platform::Gcp, + event_log: Vec::new(), + }) + } +} + +fn aia(url: &str) -> CustomExtension { + let der = yasna::construct_der(|writer| { + writer.write_sequence(|writer| { + writer.next().write_sequence(|writer| { + writer + .next() + .write_oid(&yasna::models::ObjectIdentifier::from_slice(&[ + 1, 3, 6, 1, 5, 5, 7, 48, 2, + ])); + writer + .next() + .write_tagged_implicit(yasna::Tag::context(6), |writer| { + writer.write_ia5_string(url) + }); + }); + }) + }); + CustomExtension::from_oid_content(&[1, 3, 6, 1, 5, 5, 7, 1, 1], der) +} + +fn params(name: &str) -> Result { + let mut params = CertificateParams::new(Vec::::new())?; + params.distinguished_name.push(DnType::CommonName, name); + let now = OffsetDateTime::now_utc(); + params.not_before = now - Duration::days(1); + params.not_after = now + Duration::days(30); + Ok(params) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_quote_passes_real_qvl_and_negative_cases_fail() { + let generator = TpmGenerator::new().unwrap(); + let qualifying_data = [0x42; 32]; + let quote = generator.attest(&qualifying_data).unwrap(); + let verifier = tpm_qvl::QuoteVerifier::new(generator.root_ca_pem()); + verifier.verify("e, &generator.collateral()).unwrap(); + + let wrong = TpmGenerator::new().unwrap(); + assert!(tpm_qvl::QuoteVerifier::new(wrong.root_ca_pem()) + .verify("e, &generator.collateral()) + .is_err()); + let mut tampered = quote.clone(); + tampered.message[10] ^= 1; + assert!(verifier.verify(&tampered, &generator.collateral()).is_err()); + let verified = verifier.verify("e, &generator.collateral()).unwrap(); + assert!(crate::ensure_report_data(&verified.attest.qualified_data, &[0x24; 32]).is_err()); + } +} diff --git a/dstack/ct_monitor/src/main.rs b/dstack/ct_monitor/src/main.rs index dd5d9550d..5ddc2e2c1 100644 --- a/dstack/ct_monitor/src/main.rs +++ b/dstack/ct_monitor/src/main.rs @@ -42,7 +42,6 @@ struct VerificationRequest { quote: String, event_log: String, vm_config: String, - pccs_url: Option, } /// Response from dstack-verifier @@ -88,7 +87,6 @@ struct AcmeInfoResponse { struct Monitor { gateway_uri: String, verifier_url: String, - pccs_url: Option, base_domain: String, known_keys: BTreeSet>, last_checked: Option, @@ -112,13 +110,12 @@ struct CTLog { impl Monitor { /// Create a new monitor /// `gateway` format: `base_domain[:port]`, e.g., `example.com` or `example.com:8443` - fn new(gateway: String, verifier_url: String, pccs_url: Option) -> Result { + fn new(gateway: String, verifier_url: String) -> Result { let (base_domain, gateway_uri) = Self::parse_gateway(&gateway)?; validate_domain(&base_domain)?; Ok(Self { gateway_uri, verifier_url, - pccs_url, base_domain, known_keys: BTreeSet::new(), last_checked: None, @@ -175,7 +172,6 @@ impl Monitor { quote: hex::encode("e_response.quote), event_log: quote_response.event_log, vm_config: quote_response.vm_config, - pccs_url: self.pccs_url.clone(), }; // Call verifier @@ -402,10 +398,6 @@ struct Args { /// The dstack-verifier URL #[arg(short, long, env = "VERIFIER_URL")] verifier_url: String, - - /// PCCS URL for TDX collateral fetching (optional) - #[arg(long, env = "PCCS_URL")] - pccs_url: Option, } #[tokio::main] @@ -416,7 +408,7 @@ async fn main() -> anyhow::Result<()> { fmt().with_env_filter(filter).with_ansi(false).init(); } let args = Args::parse(); - let mut monitor = Monitor::new(args.gateway, args.verifier_url, args.pccs_url)?; + let mut monitor = Monitor::new(args.gateway, args.verifier_url)?; monitor.run().await; Ok(()) } diff --git a/dstack/dstack-attest/Cargo.toml b/dstack/dstack-attest/Cargo.toml index 57c37a0c3..50fdc5d1d 100644 --- a/dstack/dstack-attest/Cargo.toml +++ b/dstack/dstack-attest/Cargo.toml @@ -20,6 +20,7 @@ fs-err.workspace = true hex.workspace = true hex_fmt.workspace = true or-panic.workspace = true +pem.workspace = true scale = { workspace = true, features = ["derive"] } sev-snp-attest.workspace = true sev-snp-qvl.workspace = true @@ -35,6 +36,7 @@ nsm-qvl.workspace = true tpm-qvl.workspace = true tpm-types.workspace = true tracing.workspace = true +x509-parser.workspace = true insta.workspace = true errify.workspace = true aws-nitro-enclaves-nsm-api = { version = "0.4", optional = true } diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index 1fb8813c9..1ba9a435b 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -18,6 +18,7 @@ use dcap_qvl::{ quote::{EnclaveReport, Quote, Report, TDReport10, TDReport15}, verify::VerifiedReport as TdxVerifiedReport, }; +pub use dstack_types::CollateralUrls; #[cfg(feature = "quote")] use dstack_types::SysConfig; use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo, Platform, VmConfig}; @@ -29,6 +30,203 @@ use serde_human_bytes as hex_bytes; use sha2::Digest as _; use tpm_qvl::verify::VerifiedReport as TpmVerifiedReport; +/// File paths for attestation trust anchors. Empty fields retain the vendor +/// production roots. Paths are read by the verifier, never by the attester. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct RootCaPaths { + pub tdx: Option, + pub gcp_tpm: Option, + pub aws_nitro_enclave: Option, + pub aws_nitro_tpm: Option, + pub sev_snp_milan: Option, + pub sev_snp_genoa: Option, + pub sev_snp_turin: Option, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AttestationVerifierConfig { + #[serde(default)] + pub insecure_allow_external_trust_anchors: bool, + #[serde(default)] + pub urls: CollateralUrls, + #[serde(default)] + pub root_ca: RootCaPaths, +} + +pub struct AttestationVerifier { + tdx: dcap_qvl::verify::QuoteVerifier, + tdx_collateral: CollateralClient, + gcp_tpm: tpm_qvl::QuoteVerifier, + aws_nitro_enclave: nsm_qvl::QuoteVerifier, + aws_nitro_tpm: nsm_qvl::QuoteVerifier, + sev_snp: sev_snp_qvl::QuoteVerifier, + amd_kds: AmdKdsClient, +} + +impl AttestationVerifier { + pub fn load(config: &AttestationVerifierConfig) -> Result { + let roots = &config.root_ca; + let RootCaPaths { + tdx, + gcp_tpm, + aws_nitro_enclave, + aws_nitro_tpm, + sev_snp_milan, + sev_snp_genoa, + sev_snp_turin, + } = roots; + let external_requested = [ + tdx, + gcp_tpm, + aws_nitro_enclave, + aws_nitro_tpm, + sev_snp_milan, + sev_snp_genoa, + sev_snp_turin, + ] + .into_iter() + .any(Option::is_some); + anyhow::ensure!( + !external_requested || config.insecure_allow_external_trust_anchors, + "external attestation trust anchors are configured but \ + insecure_allow_external_trust_anchors is false" + ); + let tdx = match read_root_file(tdx.as_deref(), "TDX")? { + Some(root) => dcap_qvl::verify::QuoteVerifier::new(tdx_root_der(root)?), + None => dcap_qvl::verify::QuoteVerifier::new_prod(), + }; + let gcp_tpm = match read_root_file(gcp_tpm.as_deref(), "GCP TPM")? { + Some(root) => tpm_qvl::QuoteVerifier::new(validated_pem_string(root, "GCP TPM")?), + None => tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?, + }; + let nsm = |path, name| -> Result { + Ok(match read_root_file(path, name)? { + Some(root) => nsm_qvl::QuoteVerifier::new(validated_pem_string(root, name)?), + None => nsm_qvl::QuoteVerifier::new_prod(), + }) + }; + let mut sev_snp = sev_snp_qvl::QuoteVerifier::new_prod(); + for (path, name, product) in [ + ( + sev_snp_milan.as_deref(), + "SEV-SNP Milan", + sev_snp_qvl::AmdSnpProduct::Milan, + ), + ( + sev_snp_genoa.as_deref(), + "SEV-SNP Genoa", + sev_snp_qvl::AmdSnpProduct::Genoa, + ), + ( + sev_snp_turin.as_deref(), + "SEV-SNP Turin", + sev_snp_qvl::AmdSnpProduct::Turin, + ), + ] { + if let Some(root) = read_root_file(path, name)? { + validate_x509_certificate(&root, name)?; + sev_snp = sev_snp.with_root(product, root); + } + } + let pccs = config + .urls + .pccs + .as_deref() + .filter(|v| !v.trim().is_empty()) + .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL); + let amd_kds = config + .urls + .amd_kds + .as_deref() + .filter(|v| !v.trim().is_empty()) + .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL); + Ok(Self { + tdx, + tdx_collateral: CollateralClient::with_default_http(pccs)?, + gcp_tpm, + aws_nitro_enclave: nsm(aws_nitro_enclave.as_deref(), "AWS Nitro Enclave")?, + aws_nitro_tpm: nsm(aws_nitro_tpm.as_deref(), "AWS NitroTPM")?, + sev_snp, + amd_kds: AmdKdsClient::with_base_url(amd_kds)?, + }) + } + + pub fn new_prod(collateral_urls: Option<&CollateralUrls>) -> Result { + let collateral_urls = collateral_urls.cloned().unwrap_or_default(); + Ok(Self { + tdx: dcap_qvl::verify::QuoteVerifier::new_prod(), + tdx_collateral: CollateralClient::with_default_http( + collateral_urls + .pccs + .as_deref() + .filter(|url| !url.trim().is_empty()) + .unwrap_or(dcap_qvl::collateral::PHALA_PCCS_URL), + )?, + gcp_tpm: tpm_qvl::QuoteVerifier::new_prod(Platform::Gcp)?, + aws_nitro_enclave: nsm_qvl::QuoteVerifier::new_prod(), + aws_nitro_tpm: nsm_qvl::QuoteVerifier::new_prod(), + sev_snp: sev_snp_qvl::QuoteVerifier::new_prod(), + amd_kds: AmdKdsClient::with_base_url( + collateral_urls + .amd_kds + .as_deref() + .filter(|url| !url.trim().is_empty()) + .unwrap_or(sev_snp_qvl::AMD_KDS_DEFAULT_BASE_URL), + )?, + }) + } + + async fn verify_tdx_quote(&self, quote: &[u8]) -> Result { + let collateral = self.tdx_collateral.fetch(quote).await?; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .context("system clock is before UNIX epoch")? + .as_secs(); + self.tdx.verify(quote, &collateral, now) + } +} + +fn read_root_file(path: Option<&std::path::Path>, platform: &str) -> Result>> { + let Some(path) = path else { + return Ok(None); + }; + fs_err::read(path) + .with_context(|| format!("failed to read {platform} root CA from {}", path.display())) + .map(Some) +} + +fn validated_pem_string(root: Vec, platform: &str) -> Result { + validate_x509_certificate(&root, platform)?; + String::from_utf8(root).with_context(|| format!("{platform} root CA is not UTF-8 PEM")) +} + +fn validate_x509_certificate(root: &[u8], platform: &str) -> Result<()> { + use x509_parser::prelude::{FromDer, X509Certificate}; + let der = if root.starts_with(b"-----BEGIN") { + pem::parse(root) + .with_context(|| format!("failed to parse {platform} root CA PEM"))? + .into_contents() + } else { + root.to_vec() + }; + let (remaining, certificate) = X509Certificate::from_der(&der) + .with_context(|| format!("failed to parse {platform} root CA certificate"))?; + anyhow::ensure!(remaining.is_empty(), "trailing data in {platform} root CA"); + anyhow::ensure!( + certificate.is_ca(), + "{platform} root certificate is not a CA" + ); + Ok(()) +} + +fn tdx_root_der(root: Vec) -> Result> { + validate_x509_certificate(&root, "TDX")?; + if root.starts_with(b"-----BEGIN") { + return Ok(pem::parse(root)?.into_contents()); + } + Ok(root) +} + // Re-export TpmQuote from tpm-types pub use tpm_types::TpmQuote; @@ -40,19 +238,6 @@ pub use crate::v1::{Attestation as AttestationV1, PlatformEvidence, StackEvidenc pub const SNP_REPORT_DATA_RANGE: std::ops::Range = 0x50..0x90; -const DSTACK_TDX: &str = "dstack-tdx"; -const DSTACK_AMD_SEV_SNP: &str = "dstack-amd-sev-snp"; -const DSTACK_GCP_TDX: &str = "dstack-gcp-tdx"; -const DSTACK_NITRO_ENCLAVE: &str = "dstack-nitro-enclave"; -const DSTACK_AWS_NITRO_TPM: &str = "dstack-aws-nitro-tpm"; - -fn dcap_collateral_client(pccs_url: Option<&str>) -> Result { - match pccs_url.map(str::trim).filter(|url| !url.is_empty()) { - Some(pccs_url) => CollateralClient::with_default_http(pccs_url), - None => CollateralClient::from_env(), - } -} - /// Path to sys-config.json in the host-shared dir. /// /// Honors `DSTACK_HOST_SHARED_DIR` (exported by `dstack-util setup` because the @@ -253,27 +438,7 @@ fn mr_config_document_from_config(config: &str) -> Result> { mr_config_document_from_value(&vm_config) } -/// Attestation mode -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Encode, Decode, Serialize, Deserialize)] -pub enum AttestationMode { - /// Intel TDX with DCAP quote only - #[default] - #[serde(rename = "dstack-tdx")] - DstackTdx, - /// GCP TDX with DCAP quote only - #[serde(rename = "dstack-gcp-tdx")] - DstackGcpTdx, - /// Dstack attestation SDK in AWS Nitro Enclave - #[serde(rename = "dstack-nitro-enclave")] - DstackNitroEnclave, - /// AMD SEV-SNP report generated by the dstack attestation SDK. - #[serde(rename = "dstack-amd-sev-snp")] - DstackAmdSevSnp, - /// AWS EC2 NitroTPM attestation document generated by the dstack attestation SDK. - /// Keep this last to preserve SCALE discriminants for existing variants. - #[serde(rename = "dstack-aws-nitro-tpm")] - DstackAwsNitroTpm, -} +pub use dstack_types::TeeVariant; #[cfg(feature = "quote")] fn has_sev_snp_tsm_provider() -> bool { @@ -285,77 +450,40 @@ fn has_sev_snp_tsm_provider() -> bool { false } -fn choose_dstack_attestation_mode(has_tdx: bool, has_sev_snp: bool) -> Result { +fn choose_dstack_attestation_mode(has_tdx: bool, has_sev_snp: bool) -> Result { if has_tdx { - return Ok(AttestationMode::DstackTdx); + return Ok(TeeVariant::DstackTdx); } if has_sev_snp { - return Ok(AttestationMode::DstackAmdSevSnp); + return Ok(TeeVariant::DstackAmdSevSnp); } bail!("Unsupported platform: Dstack(-tdx/-amd-sev-snp)"); } -impl AttestationMode { - /// Detect attestation mode from system - pub fn detect() -> Result { - let has_tdx = tdx_attest::is_tdx_available(); - let has_sev_snp = - std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); - - // First, try to detect platform from DMI product name - let platform = Platform::detect_or_dstack(); - match platform { - Platform::Dstack => choose_dstack_attestation_mode(has_tdx, has_sev_snp), - Platform::Gcp => { - // GCP platform: TDX + TPM dual mode - if has_tdx { - return Ok(Self::DstackGcpTdx); - } - bail!("Unsupported platform: GCP(-tdx)"); - } - Platform::NitroEnclave => Ok(Self::DstackNitroEnclave), - Platform::AwsEc2 => { - if std::path::Path::new("/dev/tpmrm0").exists() - || std::path::Path::new("/dev/tpm0").exists() - { - return Ok(Self::DstackAwsNitroTpm); - } - bail!("unsupported platform: AWS EC2 without NitroTPM"); - } - } - } - - /// Check if TDX quote should be included - pub fn has_tdx(&self) -> bool { - match self { - Self::DstackTdx => true, - Self::DstackAmdSevSnp => false, - Self::DstackGcpTdx => true, - Self::DstackNitroEnclave => false, - Self::DstackAwsNitroTpm => false, - } - } +/// Detect the attestation variant exposed by the current guest environment. +pub fn detect_tee_variant() -> Result { + let has_tdx = tdx_attest::is_tdx_available(); + let has_sev_snp = std::path::Path::new("/dev/sev-guest").exists() || has_sev_snp_tsm_provider(); - /// Single TPM PCR used for all dstack measured events (TDX RTMR3 analogue). - /// - /// AWS NitroTPM uses non-resettable SHA384 PCR14. GCP uses SHA256 PCR14. - /// There is no separate runtime PCR (PCR23 is not used). - pub fn tpm_event_pcr_and_bank(&self) -> Option<(u32, &'static str)> { - match self { - Self::DstackGcpTdx => Some((14, "sha256")), - Self::DstackAwsNitroTpm => Some((u32::from(AWS_NITRO_TPM_EVENT_PCR), "sha384")), - Self::DstackTdx | Self::DstackAmdSevSnp | Self::DstackNitroEnclave => None, + // First, try to detect platform from DMI product name + let platform = Platform::detect_or_dstack(); + match platform { + Platform::Dstack => choose_dstack_attestation_mode(has_tdx, has_sev_snp), + Platform::Gcp => { + // GCP platform: TDX + TPM dual mode + if has_tdx { + return Ok(TeeVariant::DstackGcpTdx); + } + bail!("Unsupported platform: GCP(-tdx)"); } - } - - /// As string for debug - pub fn as_str(&self) -> &'static str { - match self { - Self::DstackTdx => DSTACK_TDX, - Self::DstackAmdSevSnp => DSTACK_AMD_SEV_SNP, - Self::DstackGcpTdx => DSTACK_GCP_TDX, - Self::DstackNitroEnclave => DSTACK_NITRO_ENCLAVE, - Self::DstackAwsNitroTpm => DSTACK_AWS_NITRO_TPM, + Platform::NitroEnclave => Ok(TeeVariant::DstackNitroEnclave), + Platform::AwsEc2 => { + if std::path::Path::new("/dev/tpmrm0").exists() + || std::path::Path::new("/dev/tpm0").exists() + { + return Ok(TeeVariant::DstackAwsNitroTpm); + } + bail!("unsupported platform: AWS EC2 without NitroTPM"); } } } @@ -836,31 +964,14 @@ impl AttestationV1 { } } - /// Verify the quote with optional custom time (testing hook). - pub async fn verify_with_time( - self, - pccs_url: Option<&str>, - now: Option, - ) -> Result { - self.verify_with_time_with_amd_kds_client(pccs_url, now, None) - .await - } - - /// Verify the quote with a caller-owned AMD KDS client. - pub async fn verify_with_amd_kds_client( - self, - pccs_url: Option<&str>, - amd_kds_client: &AmdKdsClient, - ) -> Result { - self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) - .await + pub async fn verify(self, verifier: &AttestationVerifier) -> Result { + self.verify_with_time(verifier, None).await } - async fn verify_with_time_with_amd_kds_client( + pub async fn verify_with_time( self, - pccs_url: Option<&str>, + verifier: &AttestationVerifier, now: Option, - amd_kds_client: Option<&AmdKdsClient>, ) -> Result { let AttestationV1 { version: _, @@ -898,16 +1009,18 @@ impl AttestationV1 { }; let report = match &platform { PlatformEvidence::Tdx { quote, .. } => DstackVerifiedReport::DstackTdx( - verify_tdx_quote_with_events(pccs_url, quote, &runtime_events, &report_data) + verify_tdx_quote_with_events(verifier, quote, &runtime_events, &report_data) .await?, ), PlatformEvidence::GcpTdx { quote, tpm_quote, .. } => { let tdx_report = - verify_tdx_quote_with_events(pccs_url, quote, &runtime_events, &report_data) + verify_tdx_quote_with_events(verifier, quote, &runtime_events, &report_data) .await?; - let tpm_report = tpm_qvl::get_collateral_and_verify(tpm_quote) + let tpm_report = verifier + .gcp_tpm + .fetch_and_verify(tpm_quote) .await .context("failed to verify TPM quote")?; let qualifying_data = sha256(quote); @@ -935,13 +1048,10 @@ impl AttestationV1 { let nsm = DstackNitroQuote { nsm_quote: nsm_quote.clone(), }; - let verified_report = nsm_qvl::verify_attestation( - &nsm.nsm_quote, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("NSM attestation verification failed")?; + let verified_report = verifier + .aws_nitro_enclave + .verify(&nsm.nsm_quote, None, now) + .context("NSM attestation verification failed")?; let Some(user_data) = verified_report.user_data.clone() else { bail!("NSM attestation document does not contain user_data"); }; @@ -962,6 +1072,7 @@ impl AttestationV1 { } PlatformEvidence::AwsNitroTpm { attestation_doc } => { let verified_report = verify_aws_nitro_tpm_attestation_doc( + verifier, attestation_doc, &runtime_events, &report_data, @@ -975,16 +1086,9 @@ impl AttestationV1 { cert_chain, mr_config, } => { - let owned_kds_client; - let kds_client = match amd_kds_client { - Some(client) => client, - None => { - owned_kds_client = AmdKdsClient::new()?; - &owned_kds_client - } - }; - let verified = kds_client - .verify_evidence_with_kds_fallback(report, cert_chain, &report_data) + let verified = verifier + .sev_snp + .fetch_and_verify(&verifier.amd_kds, report, cert_chain, &report_data) .await?; verify_snp_mr_config_host_data(mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) @@ -1004,18 +1108,13 @@ impl AttestationV1 { pub async fn verify_with_ra_pubkey( self, ra_pubkey_der: &[u8], - pccs_url: Option<&str>, + verifier: &AttestationVerifier, ) -> Result { let expected_report_data = QuoteContentType::RaTlsCert.to_report_data(ra_pubkey_der); if self.report_data()? != expected_report_data { bail!("report data mismatch"); } - self.verify(pccs_url).await - } - - /// Verify the quote. - pub async fn verify(self, pccs_url: Option<&str>) -> Result { - self.verify_with_time(pccs_url, None).await + self.verify(verifier).await } } @@ -1187,13 +1286,13 @@ pub enum AttestationQuote { } impl AttestationQuote { - pub fn mode(&self) -> AttestationMode { + pub fn mode(&self) -> TeeVariant { match self { - AttestationQuote::DstackTdx(_) => AttestationMode::DstackTdx, - AttestationQuote::DstackAmdSevSnp(_) => AttestationMode::DstackAmdSevSnp, - AttestationQuote::DstackGcpTdx(_) => AttestationMode::DstackGcpTdx, - AttestationQuote::DstackNitroEnclave(_) => AttestationMode::DstackNitroEnclave, - AttestationQuote::DstackAwsNitroTpm(_) => AttestationMode::DstackAwsNitroTpm, + AttestationQuote::DstackTdx(_) => TeeVariant::DstackTdx, + AttestationQuote::DstackAmdSevSnp(_) => TeeVariant::DstackAmdSevSnp, + AttestationQuote::DstackGcpTdx(_) => TeeVariant::DstackGcpTdx, + AttestationQuote::DstackNitroEnclave(_) => TeeVariant::DstackNitroEnclave, + AttestationQuote::DstackAwsNitroTpm(_) => TeeVariant::DstackAwsNitroTpm, } } } @@ -1205,25 +1304,25 @@ mod compatibility_tests { #[test] fn attestation_mode_scale_discriminants_preserve_existing_wire_values() { - assert_eq!(AttestationMode::DstackTdx.encode(), vec![0]); - assert_eq!(AttestationMode::DstackGcpTdx.encode(), vec![1]); - assert_eq!(AttestationMode::DstackNitroEnclave.encode(), vec![2]); - assert_eq!(AttestationMode::DstackAmdSevSnp.encode(), vec![3]); - assert_eq!(AttestationMode::DstackAwsNitroTpm.encode(), vec![4]); + assert_eq!(TeeVariant::DstackTdx.encode(), vec![0]); + assert_eq!(TeeVariant::DstackGcpTdx.encode(), vec![1]); + assert_eq!(TeeVariant::DstackNitroEnclave.encode(), vec![2]); + assert_eq!(TeeVariant::DstackAmdSevSnp.encode(), vec![3]); + assert_eq!(TeeVariant::DstackAwsNitroTpm.encode(), vec![4]); } #[test] fn attestation_mode_deserializes_canonical_names() { - let parse = |value| serde_json::from_str::(value).unwrap(); - assert_eq!(parse(r#""dstack-tdx""#), AttestationMode::DstackTdx); - assert_eq!(parse(r#""dstack-gcp-tdx""#), AttestationMode::DstackGcpTdx); + let parse = |value| serde_json::from_str::(value).unwrap(); + assert_eq!(parse(r#""dstack-tdx""#), TeeVariant::DstackTdx); + assert_eq!(parse(r#""dstack-gcp-tdx""#), TeeVariant::DstackGcpTdx); assert_eq!( parse(r#""dstack-amd-sev-snp""#), - AttestationMode::DstackAmdSevSnp + TeeVariant::DstackAmdSevSnp ); assert_eq!( parse(r#""dstack-nitro-enclave""#), - AttestationMode::DstackNitroEnclave + TeeVariant::DstackNitroEnclave ); } @@ -1264,7 +1363,7 @@ mod compatibility_tests { fn dstack_attestation_mode_prefers_tdx_when_both_tdx_and_tsm_exist() { assert_eq!( choose_dstack_attestation_mode(true, true).unwrap(), - AttestationMode::DstackTdx + TeeVariant::DstackTdx ); } @@ -1272,7 +1371,7 @@ mod compatibility_tests { fn dstack_attestation_mode_uses_snp_when_only_snp_exists() { assert_eq!( choose_dstack_attestation_mode(false, true).unwrap(), - AttestationMode::DstackAmdSevSnp + TeeVariant::DstackAmdSevSnp ); } } @@ -1626,15 +1725,15 @@ fn decode_mr_tdx_from_quote( } async fn verify_tdx_quote_with_events( - pccs_url: Option<&str>, + verifier: &AttestationVerifier, quote: &[u8], runtime_events: &[RuntimeEvent], report_data: &[u8; 64], ) -> Result { - let tdx_report = dcap_collateral_client(pccs_url)? - .fetch_and_verify(quote) + let tdx_report = verifier + .verify_tdx_quote(quote) .await - .context("Failed to get collateral")?; + .context("failed to verify TDX quote")?; validate_tcb(&tdx_report)?; let td_report = tdx_report.report.as_td10().context("no td report")?; @@ -1654,18 +1753,16 @@ async fn verify_tdx_quote_with_events( } fn verify_aws_nitro_tpm_attestation_doc( + verifier: &AttestationVerifier, attestation_doc: &[u8], runtime_events: &[RuntimeEvent], report_data: &[u8; 64], now: Option, ) -> Result { - let verified_report = nsm_qvl::verify_attestation( - attestation_doc, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("COSE attestation document verification failed")?; + let verified_report = verifier + .aws_nitro_tpm + .verify(attestation_doc, None, now) + .context("COSE attestation document verification failed")?; let Some(user_data) = verified_report.user_data.clone() else { bail!("NitroTPM attestation document does not contain user_data"); @@ -2002,45 +2099,43 @@ impl Attestation { .lock() .map_err(|_| anyhow!("Quote lock poisoned"))?; - let mode = AttestationMode::detect()?; + let mode = detect_tee_variant()?; let config = match mode { - AttestationMode::DstackAmdSevSnp - | AttestationMode::DstackTdx - | AttestationMode::DstackGcpTdx + TeeVariant::DstackAmdSevSnp + | TeeVariant::DstackTdx + | TeeVariant::DstackGcpTdx // AWS: prefer host-shared sys-config vm_config (carries // aws_measurement + unified os_image_hash); validated below. - | AttestationMode::DstackAwsNitroTpm => { + | TeeVariant::DstackAwsNitroTpm => { read_vm_config().context("Failed to read vm config")? } // NitroEnclave derives config from the quote's image hash below. - AttestationMode::DstackNitroEnclave => String::new(), + TeeVariant::DstackNitroEnclave => String::new(), }; let runtime_events = match mode { - AttestationMode::DstackTdx - | AttestationMode::DstackGcpTdx - | AttestationMode::DstackAwsNitroTpm => { + TeeVariant::DstackTdx | TeeVariant::DstackGcpTdx | TeeVariant::DstackAwsNitroTpm => { RuntimeEvent::read_all().context("Failed to read runtime events")? } - AttestationMode::DstackAmdSevSnp => vec![], - AttestationMode::DstackNitroEnclave => match app_id { + TeeVariant::DstackAmdSevSnp => vec![], + TeeVariant::DstackNitroEnclave => match app_id { Some(app_id) => vec![RuntimeEvent::new("app-id".to_string(), app_id.to_vec())], None => vec![], }, }; let mut quote = match mode { - AttestationMode::DstackTdx => { + TeeVariant::DstackTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; AttestationQuote::DstackTdx(TdxQuote { quote, event_log }) } - AttestationMode::DstackAmdSevSnp => { + TeeVariant::DstackAmdSevSnp => { let quote = crate::sev_snp::get_report(*report_data) .context("Failed to get SEV-SNP report")?; AttestationQuote::DstackAmdSevSnp(quote) } - AttestationMode::DstackGcpTdx => { + TeeVariant::DstackGcpTdx => { let quote = tdx_attest::get_quote(report_data).context("Failed to get quote")?; let event_log = cc_eventlog::tdx::read_event_log().context("Failed to read event log")?; @@ -2056,12 +2151,12 @@ impl Attestation { tpm_quote, }) } - AttestationMode::DstackNitroEnclave => { + TeeVariant::DstackNitroEnclave => { let nsm_quote = nsm_attest::get_attestation(report_data) .context("Failed to get NSM attestation")?; AttestationQuote::DstackNitroEnclave(DstackNitroQuote { nsm_quote }) } - AttestationMode::DstackAwsNitroTpm => { + TeeVariant::DstackAwsNitroTpm => { // Challenge binding is report_data → NitroTPM user_data only // (same role as TDX/GCP report_data; no separate nonce/public_key). let attestation_doc = crate::aws_nitro_tpm::attestation_document(report_data) @@ -2136,56 +2231,37 @@ impl Attestation { self.into() } - /// Verify the quote with optional custom time (testing hook) - pub async fn verify_with_time( - self, - pccs_url: Option<&str>, - now: Option, - ) -> Result { - self.verify_with_time_with_amd_kds_client(pccs_url, now, None) - .await - } - - /// Verify the quote with a caller-owned AMD KDS client. - pub async fn verify_with_amd_kds_client( - self, - pccs_url: Option<&str>, - amd_kds_client: &AmdKdsClient, - ) -> Result { - self.verify_with_time_with_amd_kds_client(pccs_url, None, Some(amd_kds_client)) - .await + pub async fn verify(self, verifier: &AttestationVerifier) -> Result { + self.verify_with_time(verifier, None).await } - async fn verify_with_time_with_amd_kds_client( + pub async fn verify_with_time( self, - pccs_url: Option<&str>, + verifier: &AttestationVerifier, now: Option, - amd_kds_client: Option<&AmdKdsClient>, ) -> Result { let report = match &self.quote { AttestationQuote::DstackTdx(q) => { - let report = self.verify_tdx(pccs_url, &q.quote).await?; + let report = self.verify_tdx(verifier, &q.quote).await?; DstackVerifiedReport::DstackTdx(report) } AttestationQuote::DstackAmdSevSnp(q) => { - let owned_kds_client; - let kds_client = match amd_kds_client { - Some(client) => client, - None => { - owned_kds_client = AmdKdsClient::new()?; - &owned_kds_client - } - }; - let verified = kds_client - .verify_evidence_with_kds_fallback(&q.report, &q.cert_chain, &self.report_data) + let verified = verifier + .sev_snp + .fetch_and_verify( + &verifier.amd_kds, + &q.report, + &q.cert_chain, + &self.report_data, + ) .await?; verify_snp_mr_config_host_data(&q.mr_config, &verified.host_data)?; DstackVerifiedReport::DstackAmdSevSnp(verified) } AttestationQuote::DstackGcpTdx(q) => { - let tdx_report = self.verify_tdx(pccs_url, &q.tdx_quote.quote).await?; + let tdx_report = self.verify_tdx(verifier, &q.tdx_quote.quote).await?; let tpm_report = self - .verify_tpm(&q.tpm_quote, &sha256(&q.tdx_quote.quote)) + .verify_tpm(verifier, &q.tpm_quote, &sha256(&q.tdx_quote.quote)) .await .context("Failed to verify TPM quote")?; DstackVerifiedReport::DstackGcpTdx { @@ -2195,13 +2271,14 @@ impl Attestation { } AttestationQuote::DstackNitroEnclave(quote) => { let report = self - .verify_nitro_enclave_with_time(quote, now) + .verify_nitro_enclave_with_time(verifier, quote, now) .await .context("Failed to verify Nitro Enclave")?; DstackVerifiedReport::DstackNitroEnclave(report) } AttestationQuote::DstackAwsNitroTpm(quote) => { let report = verify_aws_nitro_tpm_attestation_doc( + verifier, "e.attestation_doc, &self.runtime_events, &self.report_data, @@ -2230,18 +2307,13 @@ impl Attestation { pub async fn verify_with_ra_pubkey( self, ra_pubkey_der: &[u8], - pccs_url: Option<&str>, + verifier: &AttestationVerifier, ) -> Result { let expected_report_data = QuoteContentType::RaTlsCert.to_report_data(ra_pubkey_der); if self.report_data != expected_report_data { bail!("report data mismatch"); } - self.verify(pccs_url).await - } - - /// Verify the quote - pub async fn verify(self, pccs_url: Option<&str>) -> Result { - self.verify_with_time(pccs_url, None).await + self.verify(verifier).await } /// Verify Nitro Enclave attestation with optional custom time (testing hook) @@ -2252,18 +2324,16 @@ impl Attestation { /// 3. Validates user_data matches expected report_data async fn verify_nitro_enclave_with_time( &self, + verifier: &AttestationVerifier, nsm_quote: &DstackNitroQuote, now: Option, ) -> Result { // Verify COSE signature and certificate chain using nsm-qvl // CRL fetch is unreliable (e.g. 403 from S3), so keep it disabled here by default. - let verified_report = nsm_qvl::verify_attestation( - &nsm_quote.nsm_quote, - nsm_qvl::AWS_NITRO_ENCLAVES_ROOT_G1, - None, - now, - ) - .context("NSM attestation verification failed")?; + let verified_report = verifier + .aws_nitro_enclave + .verify(&nsm_quote.nsm_quote, None, now) + .context("NSM attestation verification failed")?; // Verify user_data matches report_data let Some(user_data) = verified_report.user_data.clone() else { @@ -2288,10 +2358,11 @@ impl Attestation { async fn verify_tpm( &self, + verifier: &AttestationVerifier, quote: &TpmQuote, qualifying_data: &[u8], ) -> Result { - let report = tpm_qvl::get_collateral_and_verify(quote).await?; + let report = verifier.gcp_tpm.fetch_and_verify(quote).await?; let pcr_ind = self .quote .mode() @@ -2315,11 +2386,15 @@ impl Attestation { Ok(report) } - async fn verify_tdx(&self, pccs_url: Option<&str>, quote: &[u8]) -> Result { - let tdx_report = dcap_collateral_client(pccs_url)? - .fetch_and_verify(quote) + async fn verify_tdx( + &self, + verifier: &AttestationVerifier, + quote: &[u8], + ) -> Result { + let tdx_report = verifier + .verify_tdx_quote(quote) .await - .context("Failed to get collateral")?; + .context("failed to verify TDX quote")?; validate_tcb(&tdx_report)?; let td_report = tdx_report.report.as_td10().context("no td report")?; @@ -2404,6 +2479,110 @@ pub struct AppInfo { mod tests { use super::*; + #[test] + fn external_trust_anchor_requires_explicit_insecure_opt_in() { + let config = AttestationVerifierConfig { + root_ca: RootCaPaths { + tdx: Some("/tmp/mock-tdx-root.pem".into()), + ..Default::default() + }, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .unwrap() + .to_string(); + assert!(error.contains("insecure_allow_external_trust_anchors is false")); + } + + #[test] + fn production_attestation_verifier_loads_all_safe_defaults() { + AttestationVerifier::load(&AttestationVerifierConfig::default()) + .expect("production roots and URLs must load"); + } + + #[test] + fn opted_in_external_root_is_read_during_load() { + let config = AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + root_ca: RootCaPaths { + tdx: Some("/definitely/missing/tdx-root.pem".into()), + ..Default::default() + }, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .unwrap() + .to_string(); + assert!(error.contains("failed to read TDX root CA")); + } + + #[test] + fn root_file_is_loaded_without_environment_indirection() { + let path = std::env::temp_dir().join(format!("dstack-root-ca-test-{}", std::process::id())); + fs_err::write(&path, b"test root").unwrap(); + assert_eq!( + read_root_file(Some(&path), "test").unwrap(), + Some(b"test root".to_vec()) + ); + fs_err::remove_file(path).unwrap(); + } + + #[test] + fn every_external_root_is_parsed_during_load() { + let path = std::env::temp_dir().join(format!( + "dstack-invalid-root-ca-test-{}", + std::process::id() + )); + fs_err::write(&path, b"not a certificate").unwrap(); + let roots = [ + RootCaPaths { + tdx: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + gcp_tpm: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + aws_nitro_enclave: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + aws_nitro_tpm: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_milan: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_genoa: Some(path.clone()), + ..Default::default() + }, + RootCaPaths { + sev_snp_turin: Some(path.clone()), + ..Default::default() + }, + ]; + for root_ca in roots { + let config = AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + root_ca, + ..Default::default() + }; + let error = AttestationVerifier::load(&config) + .err() + .expect("invalid external root must fail during load"); + assert!( + format!("{error:#}").contains("root CA"), + "unexpected error: {error:#}" + ); + } + fs_err::remove_file(path).unwrap(); + } + fn patch_v1_report_data(attestation: AttestationV1, report_data: [u8; 64]) -> AttestationV1 { attestation.with_report_data(report_data) } diff --git a/dstack/dstack-attest/src/lib.rs b/dstack/dstack-attest/src/lib.rs index 2c5e31592..cc5b4d3cc 100644 --- a/dstack/dstack-attest/src/lib.rs +++ b/dstack/dstack-attest/src/lib.rs @@ -11,7 +11,7 @@ pub use cc_eventlog as ccel; pub use tdx_attest as tdx; pub use tpm_attest as tpm; -use crate::attestation::AttestationMode; +use crate::attestation::{detect_tee_variant, TeeVariant}; pub mod amd_sev_snp; pub mod attestation; @@ -42,7 +42,7 @@ static EMIT_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { let event = RuntimeEvent::new(event.to_string(), payload.to_vec()); - let mode = AttestationMode::detect()?; + let mode = detect_tee_variant()?; // Hold the lock across both the log append and the register extension so // that the on-disk log order always matches the RTMR extension order. @@ -82,8 +82,8 @@ pub fn emit_runtime_event(event: &str, payload: &[u8]) -> anyhow::Result<()> { /// of producing quotes that can never verify. Does **not** append to the /// dstack event log / PCR14 lane — config is a separate register. pub fn measure_aws_config_pcr(config_id: &[u8; 48]) -> anyhow::Result<()> { - let mode = AttestationMode::detect()?; - if mode != AttestationMode::DstackAwsNitroTpm { + let mode = detect_tee_variant()?; + if mode != TeeVariant::DstackAwsNitroTpm { return Ok(()); } let config_pcr = u32::from(crate::attestation::AWS_NITRO_TPM_CONFIG_PCR); diff --git a/dstack/dstack-attest/tests/nitro_verify.rs b/dstack/dstack-attest/tests/nitro_verify.rs index 77be34938..5becbb36e 100644 --- a/dstack/dstack-attest/tests/nitro_verify.rs +++ b/dstack/dstack-attest/tests/nitro_verify.rs @@ -4,7 +4,9 @@ //! Integration test: verify Nitro Enclave attestation end-to-end -use dstack_attest::attestation::{AttestationQuote, DstackVerifiedReport, VersionedAttestation}; +use dstack_attest::attestation::{ + AttestationQuote, AttestationVerifier, DstackVerifiedReport, VersionedAttestation, +}; use nsm_qvl::{AttestationDocument, CoseSign1}; use std::time::{Duration, SystemTime}; @@ -40,8 +42,9 @@ async fn verify_nitro_attestation_bin() { } _ => panic!("unexpected quote type"), }; + let verifier = AttestationVerifier::new_prod(None).unwrap(); let verified = attestation - .verify_with_time(None, Some(fixed_now)) + .verify_with_time(&verifier, Some(fixed_now)) .await .unwrap(); let DstackVerifiedReport::DstackNitroEnclave(report) = verified.report else { diff --git a/dstack/dstack-attest/tests/sev_snp_verify.rs b/dstack/dstack-attest/tests/sev_snp_verify.rs index 6fded651f..13a87759a 100644 --- a/dstack/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack/dstack-attest/tests/sev_snp_verify.rs @@ -271,6 +271,28 @@ fn forged_report_bytes_fail_signature_verification() { ); } +#[test] +fn tampered_real_amd_ask_fails_chain_verification() { + let report = fixture_report(); + let mut ask_der = pem::parse(SEV_ASK_PEM) + .expect("parse real AMD ASK") + .into_contents(); + let last = ask_der.last_mut().expect("ASK DER is non-empty"); + *last ^= 1; + let tampered_ask = pem::encode(&pem::Pem::new("CERTIFICATE", ask_der)); + + let error = verify_amd_snp_attestation(&AmdSnpAttestationInput { + report: &report, + ask_pem: tampered_ask.as_bytes(), + vcek_pem: SEV_VCEK_PEM, + }) + .expect_err("tampered AMD ASK must fail verification"); + assert!( + error.to_string().contains("cert chain verification"), + "unexpected error: {error:#}" + ); +} + #[test] fn wrong_collateral_is_rejected() { let report = fixture_report(); diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 841c5fdb4..d68d1ab5f 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -709,13 +709,22 @@ pub struct SysConfig { pub kms_urls: Vec, #[serde(default, alias = "tproxy_urls")] pub gateway_urls: Vec, - pub pccs_url: Option, + /// Backward-compatible input for sys-config files produced by older hosts. + #[serde(default, rename = "pccs_url", skip_serializing)] + legacy_pccs_url: Option, + /// Attestation collateral service endpoints. Platform defaults are used + /// for fields that are absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collateral_urls: Option, /// Optional NVIDIA attestation collateral proxy. When present, nvattest /// fetches both OCSP responses and RIM documents through this endpoint. #[serde(default, skip_serializing_if = "Option::is_none")] pub nvidia_attestation_proxy_url: Option, pub docker_registry: Option, pub host_api_url: Option, + /// Development-only TEE simulator selection. Production guests ignore it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tee_simulator: Option, /// MrConfigV3 document string for platform app/config binding. /// /// Hosts generate this in JCS form, but verifiers hash the supplied string @@ -727,6 +736,99 @@ pub struct SysConfig { pub vm_config: String, } +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq, Eq)] +pub struct CollateralUrls { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pccs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub amd_kds: Option, +} + +impl SysConfig { + pub fn collateral_urls(&self) -> CollateralUrls { + let mut urls = self.collateral_urls.clone().unwrap_or_default(); + if urls.pccs.is_none() { + urls.pccs.clone_from(&self.legacy_pccs_url); + } + urls + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct TeeSimulatorConfig { + /// Platform ABI exposed by dstack-tee-simulator. Defaults to `dstack-tdx`. + #[serde(default)] + pub platform: TeeVariant, + /// Hex-encoded 32-byte development PKI seed. The host collateral service + /// and guest simulator must receive the same seed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mock_attestation_seed: Option, + /// Base URL used in mock collateral certificates (AIA/CRL). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub collateral_base_url: Option, + /// Populated in memory by the simulator from the top-level sys-config. + #[serde(skip)] + pub mr_config: Option, + /// Populated in memory from the top-level sys-config. Mock reports use the + /// same image-measurement document that the guest sends to verifiers. + #[serde(skip)] + pub vm_config: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Copy, Default, PartialEq, Eq, Encode, Decode)] +pub enum TeeVariant { + #[default] + #[serde(rename = "dstack-tdx")] + DstackTdx, + #[serde(rename = "dstack-gcp-tdx")] + DstackGcpTdx, + #[serde(rename = "dstack-nitro-enclave")] + DstackNitroEnclave, + #[serde(rename = "dstack-amd-sev-snp")] + DstackAmdSevSnp, + #[serde(rename = "dstack-aws-nitro-tpm")] + DstackAwsNitroTpm, +} + +impl TeeVariant { + pub fn has_tdx(self) -> bool { + matches!(self, Self::DstackTdx | Self::DstackGcpTdx) + } + + pub fn tpm_event_pcr_and_bank(self) -> Option<(u32, &'static str)> { + match self { + Self::DstackGcpTdx => Some((14, "sha256")), + Self::DstackAwsNitroTpm => Some((14, "sha384")), + Self::DstackTdx | Self::DstackAmdSevSnp | Self::DstackNitroEnclave => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::DstackTdx => "dstack-tdx", + Self::DstackGcpTdx => "dstack-gcp-tdx", + Self::DstackAmdSevSnp => "dstack-amd-sev-snp", + Self::DstackNitroEnclave => "dstack-nitro-enclave", + Self::DstackAwsNitroTpm => "dstack-aws-nitro-tpm", + } + } +} + +impl std::str::FromStr for TeeVariant { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "dstack-tdx" => Ok(Self::DstackTdx), + "dstack-gcp-tdx" => Ok(Self::DstackGcpTdx), + "dstack-amd-sev-snp" => Ok(Self::DstackAmdSevSnp), + "dstack-nitro-enclave" => Ok(Self::DstackNitroEnclave), + "dstack-aws-nitro-tpm" => Ok(Self::DstackAwsNitroTpm), + _ => Err(format!("unsupported TEE variant: {value}")), + } + } +} + impl SysConfig { /// Canonical MrConfigV3 document for this VM, if any. /// @@ -1739,11 +1841,12 @@ impl Platform { /// Detect platform from system DMI information pub fn detect() -> Option { - // Nitro Enclave: NSM device exists only inside enclave + // `/dev/nsm` is the authoritative Nitro Enclave ABI. Check it before + // DMI because an enclave may inherit EC2-identifying DMI strings from + // its parent host, while a regular EC2 instance does not expose NSM. if Path::new("/dev/nsm").exists() { return Some(Self::NitroEnclave); } - let product_name = std::fs::read_to_string("/sys/class/dmi/id/product_name").ok(); let sys_vendor = std::fs::read_to_string("/sys/class/dmi/id/sys_vendor").ok(); Self::detect_from_dmi(product_name.as_deref(), sys_vendor.as_deref()) diff --git a/dstack/dstack-util/src/host_api.rs b/dstack/dstack-util/src/host_api.rs index 14715837e..f78291ac2 100644 --- a/dstack/dstack-util/src/host_api.rs +++ b/dstack/dstack-util/src/host_api.rs @@ -47,7 +47,8 @@ impl HostApi { None => { let local_config: SysConfig = deserialize_json_file(format!("{HOST_SHARED_DIR}/{SYS_CONFIG}"))?; - Self::new(local_config.host_api_url, local_config.pccs_url) + let pccs = local_config.collateral_urls().pccs; + Self::new(local_config.host_api_url, pccs) } }; Ok(api) diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index da5df9413..a61a0a1f6 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -1188,7 +1188,7 @@ async fn cmd_tpm_verify(args: TpmVerifyArgs) -> Result<()> { println!("[Step 1] Fetching quote collateral (certificates + CRLs)..."); let collateral = tpm_qvl::get_collateral(&tpm_quote, &root_ca_pem) .await - .context("Failed to get collateral")?; + .context("failed to get TPM collateral")?; let crl_count = collateral.crls.len() + if collateral.root_ca_crl.is_some() { 1 diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 7512ec29e..8c9049571 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::sync::Arc; use std::{ collections::{BTreeMap, BTreeSet}, fmt::Display, @@ -33,7 +34,7 @@ use ra_rpc::{ Attestation, }; use ra_tls::{ - attestation::{AttestationMode, QuoteContentType}, + attestation::{detect_tee_variant, AttestationVerifier, QuoteContentType, TeeVariant}, cert::{generate_ra_cert, CertConfigV2, CertSigningRequestV2, Csr}, }; use rand::Rng as _; @@ -443,7 +444,6 @@ impl<'a> GatewayContext<'a> { }; let client = RaClientConfig::builder() .remote_uri(url) - .maybe_pccs_url(self.shared.sys_config.pccs_url.clone()) .tls_client_cert(client_cert.to_string()) .tls_client_key(client_key.to_string()) .tls_ca_cert(ca_cert) @@ -539,9 +539,11 @@ impl<'a> GatewayContext<'a> { .map(|d| d.as_secs()) .unwrap_or(0); let cert_not_after = now + CERT_VALIDITY_SECS; + let collateral_urls = self.shared.sys_config.collateral_urls(); + let verifier = Arc::new(AttestationVerifier::new_prod(Some(&collateral_urls))?); let cert_client = CertRequestClient::create( self.keys, - self.shared.sys_config.pccs_url.as_deref(), + verifier, self.shared.sys_config.vm_config.clone(), ) .await @@ -809,12 +811,12 @@ fn verify_app_compose_policy(shared: &HostShared) -> Result<()> { bail!("Unsupported attestation platform: requirements.platforms is empty"); } let current_platform = - AttestationMode::detect().context("failed to detect current attestation platform")?; + detect_tee_variant().context("failed to detect current attestation platform")?; verify_platform_requirements(app_compose, current_platform)?; } if requirements.tdx_measure_acpi_tables.is_some() { let current_platform = - AttestationMode::detect().context("failed to detect current attestation platform")?; + detect_tee_variant().context("failed to detect current attestation platform")?; verify_tdx_measure_acpi_tables_requirement( app_compose, &sys_config.vm_config, @@ -867,7 +869,7 @@ fn verify_os_version_requirement(app_compose: &AppCompose, current_os_version: & fn verify_platform_requirements( app_compose: &AppCompose, - current_platform: AttestationMode, + current_platform: TeeVariant, ) -> Result<()> { let Some(requirements) = app_compose.requirements.as_ref() else { return Ok(()); @@ -895,7 +897,7 @@ fn verify_platform_requirements( ); } -fn parse_requirement_platform(platform: &str, index: usize) -> Result { +fn parse_requirement_platform(platform: &str, index: usize) -> Result { serde_json::from_value(serde_json::Value::String(platform.to_string())) .with_context(|| format!("Invalid requirements.platforms[{index}]: {platform}")) } @@ -907,7 +909,7 @@ fn format_requirement_platforms(platforms: &[String]) -> String { fn verify_tdx_measure_acpi_tables_requirement( app_compose: &AppCompose, vm_config: &str, - current_platform: AttestationMode, + current_platform: TeeVariant, ) -> Result<()> { let Some(measure_acpi_tables) = app_compose .requirements @@ -916,7 +918,7 @@ fn verify_tdx_measure_acpi_tables_requirement( else { return Ok(()); }; - if current_platform != AttestationMode::DstackTdx { + if current_platform != TeeVariant::DstackTdx { return Ok(()); } let vm_config: dstack_types::VmConfig = serde_json::from_str(vm_config) @@ -2037,7 +2039,7 @@ impl<'a> Stage0<'a> { fn host_api(&self) -> HostApi { HostApi::new( self.shared.sys_config.host_api_url.clone(), - self.shared.sys_config.pccs_url.clone(), + self.shared.sys_config.collateral_urls().pccs, ) } fn load(args: &'a SetupArgs) -> Result { @@ -2053,7 +2055,7 @@ impl<'a> Stage0<'a> { let host_shared = HostShared::copy("/tmp/.host-shared".as_ref(), &host_shared_copy_dir)?; let host_api = HostApi::new( host_shared.sys_config.host_api_url.clone(), - host_shared.sys_config.pccs_url.clone(), + host_shared.sys_config.collateral_urls().pccs, ); Ok(Self { args, @@ -2078,6 +2080,8 @@ impl<'a> Stage0<'a> { .context("Failed to get temp ca cert")? }; let cert_pair = generate_ra_cert(tmp_ca.temp_ca_cert.clone(), tmp_ca.temp_ca_key.clone())?; + let collateral_urls = self.shared.sys_config.collateral_urls(); + let attestation_verifier = Arc::new(AttestationVerifier::new_prod(Some(&collateral_urls))?); let ra_client = RaClientConfig::builder() .tls_no_check(false) .tls_built_in_root_certs(false) @@ -2085,7 +2089,7 @@ impl<'a> Stage0<'a> { .tls_client_cert(cert_pair.cert_pem) .tls_client_key(cert_pair.key_pem) .tls_ca_cert(tmp_ca.ca_cert.clone()) - .maybe_pccs_url(self.shared.sys_config.pccs_url.clone()) + .attestation_verifier(attestation_verifier) .cert_validator(Box::new(|cert| { let Some(cert) = cert else { bail!("Missing server cert"); @@ -2586,8 +2590,8 @@ impl<'a> Stage0<'a> { let truncated_compose_hash = truncate(&compose_hash, 20); let key_provider = self.shared.app_compose.key_provider(); let mut instance_info = self.shared.instance_info.clone(); - let is_snp = AttestationMode::detect() - .map(|mode| mode == AttestationMode::DstackAmdSevSnp) + let is_snp = detect_tee_variant() + .map(|mode| mode == TeeVariant::DstackAmdSevSnp) .unwrap_or(false); if instance_info.app_id.is_empty() { @@ -2855,7 +2859,6 @@ impl Stage1<'_> { let config = serde_json::json!({ "default": { "core": { - "pccs_url": self.shared.sys_config.pccs_url, "data_disks": data_disks, } } @@ -3200,15 +3203,14 @@ fn test_platform_requirements_accept_matching_platform() { None, Some(&["dstack-gcp-tdx", "dstack-tdx"]), ); - verify_platform_requirements(&app_compose, AttestationMode::DstackGcpTdx).unwrap(); - verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap(); + verify_platform_requirements(&app_compose, TeeVariant::DstackGcpTdx).unwrap(); + verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap(); } #[test] fn test_platform_requirements_reject_non_matching_platform() { let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&["dstack-gcp-tdx"])); - let err = - verify_platform_requirements(&app_compose, AttestationMode::DstackAmdSevSnp).unwrap_err(); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackAmdSevSnp).unwrap_err(); assert!(err.to_string().contains("Unsupported attestation platform")); } @@ -3235,13 +3237,13 @@ fn test_empty_requirements_require_v3_manifest() { #[test] fn test_platform_requirements_omitted_accepts_any_platform() { let app_compose = test_app_compose(serde_json::json!("3"), None, None); - verify_platform_requirements(&app_compose, AttestationMode::DstackAmdSevSnp).unwrap(); + verify_platform_requirements(&app_compose, TeeVariant::DstackAmdSevSnp).unwrap(); } #[test] fn test_platform_requirements_explicit_empty_rejects_all_platforms() { let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&[])); - let err = verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap_err(); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap_err(); assert!(err.to_string().contains("Unsupported attestation platform")); let app_compose = test_app_compose(serde_json::json!("2"), None, Some(&[])); @@ -3252,7 +3254,7 @@ fn test_platform_requirements_explicit_empty_rejects_all_platforms() { #[test] fn test_platform_requirements_reject_invalid_platform_value() { let app_compose = test_app_compose(serde_json::json!("3"), None, Some(&["gcptdx"])); - let err = verify_platform_requirements(&app_compose, AttestationMode::DstackTdx).unwrap_err(); + let err = verify_platform_requirements(&app_compose, TeeVariant::DstackTdx).unwrap_err(); assert!(err .to_string() .contains("Invalid requirements.platforms[0]")); @@ -3269,12 +3271,12 @@ fn test_tdx_measure_acpi_tables_requirement_matches_vm_config() { } })) .unwrap(); - verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, AttestationMode::DstackTdx) + verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, TeeVariant::DstackTdx) .unwrap(); let err = verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackTdx, + TeeVariant::DstackTdx, ) .unwrap_err(); assert!(err.to_string().contains("tdx_measure_acpi_tables=true")); @@ -3291,15 +3293,12 @@ fn test_tdx_measure_acpi_tables_requirement_matches_vm_config() { verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackTdx, + TeeVariant::DstackTdx, ) .unwrap(); - let err = verify_tdx_measure_acpi_tables_requirement( - &app_compose, - r#"{}"#, - AttestationMode::DstackTdx, - ) - .unwrap_err(); + let err = + verify_tdx_measure_acpi_tables_requirement(&app_compose, r#"{}"#, TeeVariant::DstackTdx) + .unwrap_err(); assert!(err.to_string().contains("tdx_measure_acpi_tables=false")); } @@ -3317,7 +3316,7 @@ fn test_tdx_measure_acpi_tables_requirement_ignored_on_non_tdx() { verify_tdx_measure_acpi_tables_requirement( &app_compose, r#"{"tdx_attestation_variant":"lite"}"#, - AttestationMode::DstackAmdSevSnp, + TeeVariant::DstackAmdSevSnp, ) .unwrap(); } diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 82b627846..72cf28755 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{bail, Context, Result}; -use dstack_attest::attestation::{Attestation, AttestationMode, AttestationQuote}; +use dstack_attest::attestation::{detect_tee_variant, Attestation, AttestationQuote, TeeVariant}; use dstack_types::{ mr_config::{MrConfig, MrConfigV3}, shared_filenames::{host_shared_dir, SYS_CONFIG}, @@ -72,7 +72,7 @@ pub fn verify_mr_config_id( key_provider: KeyProviderKind, key_provider_id: &[u8], ) -> Result<()> { - let mode = AttestationMode::detect().context("Failed to detect attestation mode")?; + let mode = detect_tee_variant().context("Failed to detect attestation mode")?; let local = LocalMrConfigValues { compose_hash, gpu_policy_hash, @@ -84,16 +84,13 @@ pub fn verify_mr_config_id( verify_mr_config_id_for_mode(mode, local) } -fn verify_mr_config_id_for_mode( - mode: AttestationMode, - local: LocalMrConfigValues<'_>, -) -> Result<()> { +fn verify_mr_config_id_for_mode(mode: TeeVariant, local: LocalMrConfigValues<'_>) -> Result<()> { match mode { - AttestationMode::DstackAmdSevSnp => verify_snp_mr_config(local), + TeeVariant::DstackAmdSevSnp => verify_snp_mr_config(local), // AWS PCR8 is computed by the guest from measured reality (MrConfig V2 // in measure_app_info); there is no host-supplied claim to cross-check. // The key_provider_id pin is enforced by verify_key_provider_id. - AttestationMode::DstackAwsNitroTpm => Ok(()), + TeeVariant::DstackAwsNitroTpm => Ok(()), _ => verify_tdx_mr_config_id(local), } } diff --git a/dstack/gateway/Cargo.toml b/dstack/gateway/Cargo.toml index d54db4b94..f81713bfe 100644 --- a/dstack/gateway/Cargo.toml +++ b/dstack/gateway/Cargo.toml @@ -10,6 +10,7 @@ edition.workspace = true license.workspace = true [dependencies] +dstack-attest.workspace = true arc-swap.workspace = true rocket = { workspace = true, features = ["mtls", "json"] } tracing.workspace = true diff --git a/dstack/gateway/gateway.toml b/dstack/gateway/gateway.toml index 74afe9c2b..1772a67e4 100644 --- a/dstack/gateway/gateway.toml +++ b/dstack/gateway/gateway.toml @@ -16,6 +16,24 @@ kms_url = "" set_ulimit = true rpc_domain = "" +# Required before any root_ca entry is accepted. Development/CI only; +# production should retain vendor roots. +[core.attestation] +insecure_allow_external_trust_anchors = false + +[core.attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[core.attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" + [core.auth] enabled = false url = "http://localhost/app-auth" diff --git a/dstack/gateway/src/config.rs b/dstack/gateway/src/config.rs index e69965aee..f3a8dcf58 100644 --- a/dstack/gateway/src/config.rs +++ b/dstack/gateway/src/config.rs @@ -4,6 +4,7 @@ use anyhow::{bail, Context, Result}; use cmd_lib::run_cmd as cmd; +use dstack_attest::attestation::AttestationVerifierConfig; use ipnet::Ipv4Net; use load_config::load_config; use rocket::figment::Figment; @@ -219,7 +220,8 @@ pub struct SyncConfig { pub struct Config { pub wg: WgConfig, pub proxy: ProxyConfig, - pub pccs_url: Option, + #[serde(default)] + pub attestation: AttestationVerifierConfig, pub recycle: RecycleConfig, pub set_ulimit: bool, pub rpc_domain: String, diff --git a/dstack/gateway/src/main.rs b/dstack/gateway/src/main.rs index 9c11d9a1f..f689d2d7f 100644 --- a/dstack/gateway/src/main.rs +++ b/dstack/gateway/src/main.rs @@ -10,13 +10,17 @@ use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, GetTlsKeyAr use dstack_kms_rpc::SignCertRequest; use http_client::prpc::PrpcClient; use ra_rpc::{client::RaClient, prpc_routes as prpc, rocket_helper::QuoteVerifier}; -use ra_tls::cert::{CertConfigV2, CertSigningRequestV2, Csr}; use ra_tls::rcgen::KeyPair; +use ra_tls::{ + attestation::AttestationVerifier, + cert::{CertConfigV2, CertSigningRequestV2, Csr}, +}; use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, }; use serde::{Deserialize, Serialize}; +use std::sync::Arc; use tracing::info; use admin_service::AdminRpcHandler; @@ -237,7 +241,6 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config = figment.focus("core").extract::()?; - // Validate node_id if config.sync.enabled && config.sync.node_id == 0 { anyhow::bail!("node_id must be greater than 0"); @@ -269,7 +272,10 @@ async fn main() -> Result<()> { Some(info.app_id) }; let proxy_config = config.proxy.clone(); - let pccs_url = config.pccs_url.clone(); + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); let admin_auth = if config.admin.enabled { Some(admin_auth::AdminAuthFairing::from_config(&config.admin)?) } else { @@ -315,7 +321,7 @@ async fn main() -> Result<()> { }) })) .manage(state.clone()); - let verifier = QuoteVerifier::new(pccs_url); + let verifier = QuoteVerifier::new(attestation_verifier); rocket = rocket.manage(verifier); let main_srv = rocket.launch(); let admin_state = state.clone(); diff --git a/dstack/guest-agent/src/config.rs b/dstack/guest-agent/src/config.rs index 7f94185d7..550502f98 100644 --- a/dstack/guest-agent/src/config.rs +++ b/dstack/guest-agent/src/config.rs @@ -48,8 +48,6 @@ pub struct Config { #[serde(deserialize_with = "deserialize_app_compose", flatten)] pub app_compose: AppComposeWrapper, pub sys_config_file: PathBuf, - #[serde(default)] - pub pccs_url: Option, pub data_disks: HashSet, } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e20126095..dbf548ea1 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -10,6 +10,7 @@ use std::{ use anyhow::{Context, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use cert_client::CertRequestClient; +use dstack_attest::attestation::AttestationVerifier; use dstack_guest_agent_rpc::{ dstack_guest_server::{DstackGuestRpc, DstackGuestServer}, tappd_server::{TappdRpc, TappdServer}, @@ -160,11 +161,12 @@ impl AppState { let sys_config: SysConfig = serde_json::from_str(&fs::read_to_string(&config.sys_config_file)?) .context("Failed to parse VM config")?; + let collateral_urls = sys_config.collateral_urls(); let vm_config = sys_config.vm_config; - let cert_client = - CertRequestClient::create(&keys, config.pccs_url.as_deref(), vm_config.clone()) - .await - .context("Failed to create cert signer")?; + let verifier = Arc::new(AttestationVerifier::new_prod(Some(&collateral_urls))?); + let cert_client = CertRequestClient::create(&keys, verifier, vm_config.clone()) + .await + .context("Failed to create cert signer")?; let me = Self { inner: Arc::new(AppStateInner { config, @@ -762,7 +764,6 @@ mod tests { keys_file: String::new(), app_compose: dummy_appcompose_wrapper, sys_config_file: String::new().into(), - pccs_url: None, data_disks: HashSet::new(), }; @@ -835,7 +836,8 @@ pNs85uhOZE8z2jr8Pg== }, }; - let dummy_cert_client = CertRequestClient::create(&dummy_keys, None, String::new()) + let verifier = Arc::new(AttestationVerifier::new_prod(None).unwrap()); + let dummy_cert_client = CertRequestClient::create(&dummy_keys, verifier, String::new()) .await .expect("Failed to create CertRequestClient"); diff --git a/dstack/kms/Cargo.toml b/dstack/kms/Cargo.toml index 6f6a27bb3..08a04bf4d 100644 --- a/dstack/kms/Cargo.toml +++ b/dstack/kms/Cargo.toml @@ -47,11 +47,11 @@ tempfile.workspace = true serde-duration.workspace = true dstack-verifier = { workspace = true, default-features = false } dstack-mr.workspace = true +dstack-attest.workspace = true dstack-api-auth.workspace = true [dev-dependencies] cc-eventlog.workspace = true -dstack-attest.workspace = true [features] default = [] diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index a8e213943..1ba51390b 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -31,10 +31,6 @@ site_name = "" # outside a TEE (e.g. local dev/testing) where the local guest agent socket # is unavailable. enforce_self_authorization = true -# Optional AMD KDS-compatible base URL for SEV-SNP collateral fetches. -# Leave empty to use the built-in default. -amd_kds_base_url = "" - # AMD SEV-SNP key/cert release remains disabled unless this local KMS gate is # explicitly enabled. External auth policy must still allow the verified # BootInfo before any sensitive material is returned. @@ -46,6 +42,23 @@ sev_snp_key_release = false # auth policy must still allow the verified BootInfo. aws_nitro_tpm_key_release = false +# Optional attestation trust-anchor files. Omitted entries use vendor roots. +[core.attestation] +insecure_allow_external_trust_anchors = false + +[core.attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[core.attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" + [core.image] verify = true cache_dir = "/usr/share/dstack/images" diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 621e5cb90..614c0e125 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use dstack_attest::attestation::AttestationVerifierConfig; use load_config::load_config; use rocket::figment::Figment; use serde::Deserialize; @@ -34,15 +35,8 @@ pub(crate) struct ImageConfig { #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, - pub pccs_url: Option, - /// Optional AMD KDS-compatible base URL used for SEV-SNP collateral requests. - /// - /// Empty by default. When set, the KMS process exports this base URL for - /// dstack-attest before any attestation verification happens. The base URL - /// must expose AMD KDS-compatible paths under `/vcek/v1`, e.g. - /// `https://kdsintf.amd.com/vcek/v1` or a trusted mirror/cache. #[serde(default)] - pub amd_kds_base_url: Option, + pub attestation: AttestationVerifierConfig, pub auth_api: AuthApi, pub onboard: OnboardConfig, pub image: ImageConfig, diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index 73420e18a..ba51ebb20 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -7,6 +7,7 @@ use clap::Parser; use config::KmsConfig; use main_service::{KmsState, RpcHandler}; use ra_rpc::rocket_helper::QuoteVerifier; +use ra_tls::attestation::AttestationVerifier; use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, @@ -49,11 +50,13 @@ async fn run_onboard_service(kms_config: KmsConfig, figment: Figment) -> Result< } if !kms_config.onboard.auto_bootstrap_domain.is_empty() { - onboard_service::bootstrap_keys(&kms_config).await?; + let verifier = AttestationVerifier::load(&kms_config.attestation) + .context("failed to load attestation verifier")?; + onboard_service::bootstrap_keys(&kms_config, &verifier).await?; return Ok(()); } - let state = OnboardState::new(kms_config); + let state = OnboardState::new(kms_config)?; let figment = figment .clone() .merge(Serialized::defaults(figment.find_value("core.onboard")?)); @@ -101,19 +104,6 @@ fn record_attestation_metrics(req: &rocket::Request<'_>, res: &rocket::Response< .record_attestation_request(res.status().code >= 400); } -fn configure_amd_kds_base_from_config(config: &KmsConfig) { - let Some(base_url) = config - .amd_kds_base_url - .as_deref() - .map(str::trim) - .filter(|base_url| !base_url.is_empty()) - else { - return; - }; - std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); - info!("AMD SEV-SNP KDS base URL configured"); -} - #[rocket::main] async fn main() -> Result<()> { { @@ -125,7 +115,6 @@ async fn main() -> Result<()> { let figment = config::load_config_figment(args.config.as_deref()); let config: KmsConfig = figment.focus("core").extract()?; - configure_amd_kds_base_from_config(&config); if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -146,8 +135,6 @@ async fn main() -> Result<()> { info!(" /prpc/{method}"); } - let pccs_url = config.pccs_url.clone(); - let amd_kds_base_url = config.amd_kds_base_url.clone(); let metrics_enabled = config.metrics.enabled; let admin_config = config.admin.clone(); // build the admin listener figment from `[core.admin]` before `config` is @@ -165,6 +152,7 @@ async fn main() -> Result<()> { None }; let state = main_service::KmsState::new(config).context("Failed to initialize KMS state")?; + let quote_verifier = QuoteVerifier::new(state.attestation_verifier()); let figment = figment .clone() .merge(Serialized::defaults(figment.find_value("rpc")?)); @@ -190,8 +178,7 @@ async fn main() -> Result<()> { .mount("/", rocket::routes![metrics]); } - let verifier = QuoteVerifier::new_with_amd_kds_base(pccs_url, amd_kds_base_url); - rocket = rocket.manage(verifier); + rocket = rocket.manage(quote_verifier); let main_srv = rocket.launch(); match admin_setup { diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 5ff8504a4..bf2618924 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -22,7 +22,7 @@ use fs_err as fs; use k256::ecdsa::SigningKey; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ - attestation::{AttestationMode, VerifiedAttestation}, + attestation::{AttestationVerifier, TeeVariant, VerifiedAttestation}, cert::{CaCert, CertRequest, CertSigningRequestV1, CertSigningRequestV2, Csr}, kdf, }; @@ -59,6 +59,7 @@ pub struct KmsStateInner { temp_ca_cert: String, temp_ca_key: String, verifier: CvmVerifier, + attestation_verifier: Arc, self_boot_info: OnceCell, metrics: KmsMetrics, } @@ -142,11 +143,15 @@ impl KmsState { fs::read_to_string(config.tmp_ca_key()).context("Faeild to read temp ca key")?; let temp_ca_cert = fs::read_to_string(config.tmp_ca_cert()).context("Faeild to read temp ca cert")?; + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); let verifier = CvmVerifier::new( config.image.cache_dir.display().to_string(), config.image.download_url.clone(), config.image.download_timeout, - config.pccs_url.clone(), + attestation_verifier.clone(), ); if !config.enforce_self_authorization { warn!( @@ -161,6 +166,7 @@ impl KmsState { temp_ca_cert, temp_ca_key, verifier, + attestation_verifier, self_boot_info: OnceCell::new(), metrics: KmsMetrics::default(), }), @@ -170,6 +176,10 @@ impl KmsState { pub(crate) fn metrics(&self) -> &KmsMetrics { &self.inner.metrics } + + pub(crate) fn attestation_verifier(&self) -> Arc { + self.inner.attestation_verifier.clone() + } } pub struct RpcHandler { @@ -207,10 +217,10 @@ fn ensure_key_release_allowed( aws_nitro_tpm_enabled: bool, ) -> Result<()> { match boot_info.attestation_mode { - AttestationMode::DstackAmdSevSnp if !snp_enabled => { + TeeVariant::DstackAmdSevSnp if !snp_enabled => { bail!("amd sev-snp key release is not enabled") } - AttestationMode::DstackAwsNitroTpm if !aws_nitro_tpm_enabled => { + TeeVariant::DstackAwsNitroTpm if !aws_nitro_tpm_enabled => { bail!("aws nitro-tpm key release is not enabled") } _ => Ok(()), @@ -236,7 +246,7 @@ impl RpcHandler { let boot_info = self .state .self_boot_info - .get_or_try_init(|| local_kms_boot_info(self.state.config.pccs_url.as_deref())) + .get_or_try_init(|| local_kms_boot_info(&self.state.attestation_verifier)) .await .context("Failed to load cached self boot info")?; let response = self @@ -310,7 +320,7 @@ impl RpcHandler { // SNP rootfs/app/config binding is handled by the SNP launch-measurement // helper above. The legacy OS-image verifier is TDX-oriented and still // rejects SNP quotes; keep SNP on the explicit fail-closed helper path. - if boot_info.attestation_mode != AttestationMode::DstackAmdSevSnp { + if boot_info.attestation_mode != TeeVariant::DstackAmdSevSnp { self.verify_os_image_hash(vm_config_str.into(), att) .await .context("Failed to verify os image hash")?; @@ -526,7 +536,7 @@ impl KmsRpc for RpcHandler { .attestation .clone() .into_v1() - .verify_with_ra_pubkey(&csr.pubkey, self.state.config.pccs_url.as_deref()) + .verify_with_ra_pubkey(&csr.pubkey, &self.state.attestation_verifier) .await .context("Quote verification failed")?; let app_info = self @@ -865,10 +875,7 @@ mod tests { .expect("aws nitrotpm attestation should produce KMS boot info"); let pcrs = verified_aws_nitro_tpm_pcrs(&attestation); - assert_eq!( - boot_info.attestation_mode, - AttestationMode::DstackAwsNitroTpm - ); + assert_eq!(boot_info.attestation_mode, TeeVariant::DstackAwsNitroTpm); assert_eq!(boot_info.tcb_status, "UpToDate"); assert!(boot_info.advisory_ids.is_empty()); assert_eq!(boot_info.app_id, vec![0x11; 20]); @@ -923,7 +930,7 @@ mod tests { ensure_key_release_allowed(&boot_info, false, true).unwrap(); // A TDX boot info is unaffected by the AWS gate even when it is disabled. - boot_info.attestation_mode = AttestationMode::DstackTdx; + boot_info.attestation_mode = TeeVariant::DstackTdx; ensure_key_release_allowed(&boot_info, false, false).unwrap(); } @@ -1016,7 +1023,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, &vm_config) .expect("snp attestation should build boot info through vm_config path"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.attestation_mode, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, vec![0xab; 64]); assert_eq!(boot_info.app_id, vec![0x11; 20]); @@ -1038,7 +1045,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, "") .expect("snp local KMS attestation should use embedded vm_config"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.attestation_mode, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.app_id, vec![0x11; 20]); } @@ -1053,7 +1060,7 @@ mod tests { let boot_info = build_boot_info_for_attestation(&attestation, false, &vm_config) .expect("self-contained SNP vm_config should not require KMS-local sev_snp config"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.attestation_mode, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.device_id, vec![0xab; 64]); } diff --git a/dstack/kms/src/main_service/amd_attest.rs b/dstack/kms/src/main_service/amd_attest.rs index 9e579984f..b629113bd 100644 --- a/dstack/kms/src/main_service/amd_attest.rs +++ b/dstack/kms/src/main_service/amd_attest.rs @@ -25,7 +25,7 @@ use anyhow::{bail, Context, Result}; use dstack_types::{mr_config::MrConfigV3, KeyProviderInfo}; -use ra_tls::attestation::{AttestationMode, VerifiedAttestation}; +use ra_tls::attestation::{TeeVariant, VerifiedAttestation}; use sha2::{Digest, Sha256}; use super::upgrade_authority::BootInfo; @@ -109,7 +109,7 @@ fn build_amd_snp_boot_info_with_tcb_status( let key_provider_info = mr_config_key_provider_info(&mr_config)?; Ok(BootInfo { - attestation_mode: AttestationMode::DstackAmdSevSnp, + attestation_mode: TeeVariant::DstackAmdSevSnp, mr_aggregated, os_image_hash: os_image_hash.to_vec(), mr_system, @@ -366,7 +366,7 @@ mod tests { let boot_info = build_amd_snp_boot_info(&verified, &chip_id, &input) .expect("matching measurement should build snp boot info"); - assert_eq!(boot_info.attestation_mode, AttestationMode::DstackAmdSevSnp); + assert_eq!(boot_info.attestation_mode, TeeVariant::DstackAmdSevSnp); assert_eq!(boot_info.mr_aggregated.len(), 32); assert_eq!(boot_info.device_id, chip_id.to_vec()); assert_eq!(boot_info.app_id, vec![0x11; 20]); diff --git a/dstack/kms/src/main_service/upgrade_authority.rs b/dstack/kms/src/main_service/upgrade_authority.rs index a577849ac..6c3bcd4c3 100644 --- a/dstack/kms/src/main_service/upgrade_authority.rs +++ b/dstack/kms/src/main_service/upgrade_authority.rs @@ -9,8 +9,7 @@ use dstack_guest_agent_rpc::{ dstack_guest_client::DstackGuestClient, AttestResponse, RawQuoteArgs, }; use http_client::prpc::PrpcClient; -use ra_tls::attestation::VerifiedAttestation; -use ra_tls::attestation::VersionedAttestation; +use ra_tls::attestation::{AttestationVerifier, VerifiedAttestation, VersionedAttestation}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -50,7 +49,7 @@ pub(crate) fn ensure_app_id_len(app_id: &[u8]) -> Result<()> { Ok(()) } -pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result { +pub(crate) async fn local_kms_boot_info(verifier: &AttestationVerifier) -> Result { let response = app_attest(pad64([0u8; 32])) .await .context("Failed to get local KMS attestation")?; @@ -58,7 +57,7 @@ pub(crate) async fn local_kms_boot_info(pccs_url: Option<&str>) -> Result Vec { padded } -pub(crate) async fn ensure_self_kms_allowed(cfg: &KmsConfig) -> Result<()> { +pub(crate) async fn ensure_self_kms_allowed( + cfg: &KmsConfig, + verifier: &AttestationVerifier, +) -> Result<()> { if !cfg.enforce_self_authorization { return Ok(()); } - let boot_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let boot_info = local_kms_boot_info(verifier) .await .context("failed to build local KMS boot info")?; let response = cfg @@ -214,6 +216,7 @@ pub(crate) async fn ensure_self_kms_allowed(cfg: &KmsConfig) -> Result<()> { pub(crate) async fn ensure_kms_allowed( cfg: &KmsConfig, attestation: &VerifiedAttestation, + verifier: &AttestationVerifier, ) -> Result<()> { let mut boot_info = build_boot_info_for_attestation(attestation, false, "") .context("failed to build KMS boot info from attestation")?; @@ -223,7 +226,7 @@ pub(crate) async fn ensure_kms_allowed( // validates OS image integrity transitively through the RTMR measurement chain. // TODO: remove once all source KMS instances use the unified PHALA_RATLS_ATTESTATION format. if boot_info.os_image_hash.is_empty() { - let local_info = local_kms_boot_info(cfg.pccs_url.as_deref()) + let local_info = local_kms_boot_info(verifier) .await .context("failed to get local KMS boot info for os_image_hash fallback")?; boot_info.os_image_hash = local_info.os_image_hash; diff --git a/dstack/kms/src/onboard_service.rs b/dstack/kms/src/onboard_service.rs index 84e01ef93..4fa65b761 100644 --- a/dstack/kms/src/onboard_service.rs +++ b/dstack/kms/src/onboard_service.rs @@ -19,7 +19,8 @@ use ra_rpc::{ }; use ra_tls::{ attestation::{ - GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, VersionedAttestation, + AttestationVerifier, GetDeviceId, PlatformEvidence, QuoteContentType, VerifiedAttestation, + VersionedAttestation, }, cert::{CaCert, CertRequest}, rcgen::{Certificate, KeyPair, PKCS_ECDSA_P256_SHA256}, @@ -40,11 +41,19 @@ use crate::{ #[derive(Clone)] pub struct OnboardState { config: KmsConfig, + attestation_verifier: Arc, } impl OnboardState { - pub fn new(config: KmsConfig) -> Self { - Self { config } + pub fn new(config: KmsConfig) -> Result { + let attestation_verifier = Arc::new( + AttestationVerifier::load(&config.attestation) + .context("failed to load attestation verifier")?, + ); + Ok(Self { + config, + attestation_verifier, + }) } } @@ -64,7 +73,7 @@ impl RpcCall for OnboardHandler { impl OnboardRpc for OnboardHandler { async fn bootstrap(self, request: BootstrapRequest) -> Result { - ensure_self_kms_allowed(&self.state.config) + ensure_self_kms_allowed(&self.state.config, &self.state.attestation_verifier) .await .context("KMS is not allowed to bootstrap")?; let keys = Keys::generate(&request.domain) @@ -98,7 +107,7 @@ impl OnboardRpc for OnboardHandler { &self.state.config, &source_url, &request.domain, - self.state.config.pccs_url.clone(), + self.state.attestation_verifier.clone(), ) .await .context("Failed to onboard")?; @@ -109,8 +118,6 @@ impl OnboardRpc for OnboardHandler { } async fn get_attestation_info(self) -> Result { - let pccs_url = self.state.config.pccs_url.clone(); - // Get attestation from guest agent let report_data = pad64([0u8; 32]); let response = app_attest(report_data) @@ -130,7 +137,7 @@ impl OnboardRpc for OnboardHandler { .to_string(); let verified = attestation .into_v1() - .verify(pccs_url.as_deref()) + .verify(&self.state.attestation_verifier) .await .context("Failed to verify attestation")?; @@ -412,7 +419,7 @@ impl Keys { cfg: &KmsConfig, other_kms_url: &str, domain: &str, - pccs_url: Option, + attestation_verifier: Arc, ) -> Result { let attestation_slot = Arc::new(Mutex::new(None::)); let attestation_slot_out = attestation_slot.clone(); @@ -432,22 +439,27 @@ impl Keys { *slot = Some(attestation); Ok(()) })) - .maybe_pccs_url(pccs_url.clone()) + .attestation_verifier(attestation_verifier.clone()) .build() .into_client()?; let mut kms_client = KmsClient::new(client); let tmp_ca = kms_client.get_temp_ca_cert().await?; let (ra_cert, ra_key) = gen_ra_cert(tmp_ca.temp_ca_cert, tmp_ca.temp_ca_key).await?; - let ra_client = RaClient::new_mtls(other_kms_url.into(), ra_cert, ra_key, pccs_url) - .context("Failed to create client")?; + let ra_client = RaClient::new_mtls( + other_kms_url.into(), + ra_cert, + ra_key, + attestation_verifier.clone(), + ) + .context("Failed to create client")?; kms_client = KmsClient::new(ra_client); let source_attestation = attestation_slot .lock() .map_err(|_| anyhow::anyhow!("source attestation mutex poisoned"))? .clone() .context("Missing source KMS attestation")?; - ensure_kms_allowed(cfg, &source_attestation) + ensure_kms_allowed(cfg, &source_attestation, &attestation_verifier) .await .context("Source KMS is not allowed for onboarding")?; @@ -525,8 +537,8 @@ pub(crate) async fn update_certs(cfg: &KmsConfig) -> Result<()> { Ok(()) } -pub(crate) async fn bootstrap_keys(cfg: &KmsConfig) -> Result<()> { - ensure_self_kms_allowed(cfg) +pub(crate) async fn bootstrap_keys(cfg: &KmsConfig, verifier: &AttestationVerifier) -> Result<()> { + ensure_self_kms_allowed(cfg, verifier) .await .context("KMS is not allowed to auto-bootstrap")?; let keys = Keys::generate(&cfg.onboard.auto_bootstrap_domain) diff --git a/dstack/nsm-qvl/src/lib.rs b/dstack/nsm-qvl/src/lib.rs index c8144dde1..e1afd8df2 100644 --- a/dstack/nsm-qvl/src/lib.rs +++ b/dstack/nsm-qvl/src/lib.rs @@ -48,6 +48,48 @@ pub struct NsmCollateral { /// Fingerprint: 64:1A:03:21:A3:E2:44:EF:E4:56:46:31:95:D6:06:31:7E:D7:CD:CC:3C:17:56:E0:98:93:F3:C6:8F:79:BB:5B pub const AWS_NITRO_ENCLAVES_ROOT_G1: &str = include_str!("../certs/AWS_NitroEnclaves_Root-G1.pem"); +/// Verifies NSM attestation documents against a caller-selected trust anchor. +/// +/// This mirrors `dcap_qvl::verify::QuoteVerifier`: use [`Self::new_prod`] for +/// AWS production or [`Self::new`] to inject a PEM encoded root certificate. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + root_ca_pem: String, +} + +impl QuoteVerifier { + pub fn new(root_ca_pem: impl Into) -> Self { + Self { + root_ca_pem: root_ca_pem.into(), + } + } + + pub fn new_prod() -> Self { + Self::new(AWS_NITRO_ENCLAVES_ROOT_G1) + } + + pub fn root_ca_pem(&self) -> &str { + &self.root_ca_pem + } + + pub fn verify( + &self, + cose_sign1_bytes: &[u8], + collateral: Option<&NsmCollateral>, + now: Option, + ) -> Result { + verify_attestation(cose_sign1_bytes, &self.root_ca_pem, collateral, now) + } + + pub async fn fetch_and_verify( + &self, + cose_sign1_bytes: &[u8], + now: Option, + ) -> Result { + collateral::get_collateral_and_verify(cose_sign1_bytes, &self.root_ca_pem, now).await + } +} + /// Parsed COSE Sign1 structure for NSM attestation #[derive(Debug)] pub struct CoseSign1 { diff --git a/dstack/ra-rpc/src/client.rs b/dstack/ra-rpc/src/client.rs index 07d875363..db6421e93 100644 --- a/dstack/ra-rpc/src/client.rs +++ b/dstack/ra-rpc/src/client.rs @@ -2,14 +2,17 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use anyhow::{bail, Context, Result}; use prpc::{ client::{Error, RequestClient}, Message, }; -use ra_tls::{attestation::VerifiedAttestation, traits::CertExt}; +use ra_tls::{ + attestation::{AttestationVerifier, VerifiedAttestation}, + traits::CertExt, +}; use reqwest::{tls::TlsInfo, Certificate, Client, Identity, Response}; use serde::{de::DeserializeOwned, Serialize}; @@ -38,7 +41,7 @@ pub struct RaClientConfig { tls_ca_cert: Option, #[builder(default = true)] tls_built_in_root_certs: bool, - pccs_url: Option, + attestation_verifier: Option>, cert_validator: Option, } @@ -75,9 +78,13 @@ impl RaClientConfig { builder = builder.tls_certs_only(ca_cert); } let client = builder.build().context("failed to create client")?; + let attestation_verifier = self + .attestation_verifier + .map(Ok) + .unwrap_or_else(|| AttestationVerifier::new_prod(None).map(Arc::new))?; Ok(RaClient { remote_uri: self.remote_uri, - pccs_url: self.pccs_url, + attestation_verifier, client, cert_validator: self.cert_validator, verify_server_attestation: self.verify_server_attestation, @@ -87,7 +94,7 @@ impl RaClientConfig { pub struct RaClient { remote_uri: String, - pccs_url: Option, + attestation_verifier: Arc, client: Client, cert_validator: Option, verify_server_attestation: bool, @@ -107,7 +114,7 @@ impl RaClient { remote_uri: String, cert_pem: String, key_pem: String, - pccs_url: Option, + attestation_verifier: Arc, ) -> Result { RaClientConfig::builder() .tls_no_check(true) @@ -115,7 +122,7 @@ impl RaClient { .remote_uri(remote_uri) .tls_client_cert(cert_pem) .tls_client_key(key_pem) - .maybe_pccs_url(pccs_url) + .attestation_verifier(attestation_verifier) .build() .into_client() .context("failed to create client") @@ -147,7 +154,7 @@ impl RaClient { Some(attestation) => { let verified_attestation = attestation .into_v1() - .verify_with_ra_pubkey(cert.public_key().raw, self.pccs_url.as_deref()) + .verify_with_ra_pubkey(cert.public_key().raw, &self.attestation_verifier) .await .context("Failed to verify the attestation report")?; Some(verified_attestation) diff --git a/dstack/ra-rpc/src/rocket_helper.rs b/dstack/ra-rpc/src/rocket_helper.rs index ede119569..2cc170630 100644 --- a/dstack/ra-rpc/src/rocket_helper.rs +++ b/dstack/ra-rpc/src/rocket_helper.rs @@ -14,10 +14,11 @@ use crate::openapi::{OpenApiDoc, RenderedDoc}; #[cfg(all(feature = "rocket", feature = "openapi"))] use rocket::response::content::{RawHtml, RawJson}; #[cfg(all(feature = "rocket", feature = "openapi"))] -use std::{borrow::Cow, sync::Arc}; +use std::borrow::Cow; +use std::sync::Arc; use anyhow::{Context, Result}; -use ra_tls::traits::CertExt; +use ra_tls::{attestation::AttestationVerifier, traits::CertExt}; use rocket::listener::unix::UnixStream; use rocket::listener::{Connection, Listener}; use rocket::tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; @@ -181,10 +182,9 @@ fn unix_peer_cred(stream: &UnixStream) -> Option { }) } -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct QuoteVerifier { - pccs_url: Option, - amd_kds_base_url: Option, + verifier: Arc, } pub mod deps { @@ -316,26 +316,8 @@ impl<'r> FromRequest<'r> for &'r QuoteVerifier { } impl QuoteVerifier { - pub fn new(pccs_url: Option) -> Self { - Self::new_with_amd_kds_base(pccs_url, None) - } - - pub fn new_with_amd_kds_base( - pccs_url: Option, - amd_kds_base_url: Option, - ) -> Self { - Self { - pccs_url, - amd_kds_base_url: amd_kds_base_url - .map(|url| url.trim().to_string()) - .filter(|url| !url.is_empty()), - } - } - - fn configure_amd_kds_base_for_request(&self) { - if let Some(base_url) = &self.amd_kds_base_url { - std::env::set_var("DSTACK_AMD_KDS_BASE_URL", base_url); - } + pub fn new(verifier: Arc) -> Self { + Self { verifier } } } @@ -459,21 +441,6 @@ mod tests { use rocket::tokio; use std::time::{SystemTime, UNIX_EPOCH}; - #[test] - fn quote_verifier_carries_trimmed_amd_kds_base_url() { - let verifier = QuoteVerifier::new_with_amd_kds_base( - None, - Some(" https://mirror.example.com/vcek/v1/ ".to_string()), - ); - assert_eq!( - verifier.amd_kds_base_url.as_deref(), - Some("https://mirror.example.com/vcek/v1/") - ); - - let verifier = QuoteVerifier::new_with_amd_kds_base(None, Some(" ".to_string())); - assert!(verifier.amd_kds_base_url.is_none()); - } - #[test] fn custom_unix_endpoint_maps_to_remote_endpoint() { let endpoint = Endpoint::new(UnixPeerEndpoint { @@ -567,7 +534,6 @@ pub async fn handle_prpc_impl>( .flatten(); let attestation = match (request.quote_verifier, attestation) { (Some(quote_verifier), Some(attestation)) => { - quote_verifier.configure_amd_kds_base_for_request(); let pubkey = request .certificate .context("certificate is missing")? @@ -576,7 +542,7 @@ pub async fn handle_prpc_impl>( .to_vec(); let verified = attestation .into_v1() - .verify_with_ra_pubkey(&pubkey, quote_verifier.pccs_url.as_deref()) + .verify_with_ra_pubkey(&pubkey, "e_verifier.verifier) .await .context("invalid quote")?; Some(verified) diff --git a/dstack/ra-tls/src/attestation.rs b/dstack/ra-tls/src/attestation.rs index d82ce4ff2..6d120020a 100644 --- a/dstack/ra-tls/src/attestation.rs +++ b/dstack/ra-tls/src/attestation.rs @@ -41,22 +41,22 @@ pub fn from_der(cert: &[u8]) -> Result> { /// DER SubjectPublicKeyInfo. That binding prevents an operator-controlled /// network endpoint from reusing valid attestation evidence with a different /// TLS key. -pub async fn verify_der(cert: &[u8], pccs_url: Option<&str>) -> Result { +pub async fn verify_der(cert: &[u8], verifier: &AttestationVerifier) -> Result { let (_, cert) = x509_parser::parse_x509_certificate(cert).context("failed to parse certificate")?; - verify_cert(&cert, pccs_url).await + verify_cert(&cert, verifier).await } /// Verify the RA-TLS attestation embedded in a PEM-encoded X.509 certificate. -pub async fn verify_pem(cert: &[u8], pccs_url: Option<&str>) -> Result { +pub async fn verify_pem(cert: &[u8], verifier: &AttestationVerifier) -> Result { let (_, pem) = x509_parser::pem::parse_x509_pem(cert).context("failed to parse PEM")?; - verify_der(&pem.contents, pccs_url).await + verify_der(&pem.contents, verifier).await } /// Verify the RA-TLS attestation embedded in a parsed X.509 certificate. async fn verify_cert( cert: &x509_parser::prelude::X509Certificate<'_>, - pccs_url: Option<&str>, + verifier: &AttestationVerifier, ) -> Result { let attestation = from_cert(cert)?.context("RA-TLS attestation extension missing")?; let public_key_der = cert.tbs_certificate.public_key().raw.to_vec(); @@ -68,10 +68,9 @@ async fn verify_cert( let special_usage = cert.get_special_usage()?; let attestation = attestation .into_v1() - .verify_with_ra_pubkey(&public_key_der, pccs_url) + .verify_with_ra_pubkey(&public_key_der, verifier) .await .context("RA-TLS attestation verification failed")?; - Ok(VerifiedRaTlsCert { public_key_der, attestation, @@ -139,7 +138,8 @@ mod tests { .self_signed() .unwrap(); - let result = verify_der(cert.der().as_ref(), None).await; + let verifier = AttestationVerifier::new_prod(None).unwrap(); + let result = verify_der(cert.der().as_ref(), &verifier).await; assert!(result.is_err()); let err = result.err().unwrap(); assert!(err.to_string().contains("attestation extension missing")); @@ -158,7 +158,8 @@ mod tests { .self_signed() .unwrap(); - let result = verify_der(cert.der().as_ref(), None).await; + let verifier = AttestationVerifier::new_prod(None).unwrap(); + let result = verify_der(cert.der().as_ref(), &verifier).await; assert!(result.is_err()); let err = result.err().unwrap(); assert!(format!("{err:#}").contains("report data mismatch")); diff --git a/dstack/sev-snp-attest/src/lib.rs b/dstack/sev-snp-attest/src/lib.rs index 04a2f9cca..1226070b4 100644 --- a/dstack/sev-snp-attest/src/lib.rs +++ b/dstack/sev-snp-attest/src/lib.rs @@ -127,12 +127,17 @@ fn get_report_configfs(report_data: [u8; 64]) -> Result { } fn write_first_existing(paths: &[std::path::PathBuf], binary: &[u8], hex: &[u8]) -> Result<()> { + use std::io::Write; let mut last_err = None; for path in paths { if !path.exists() { continue; } - match fs_err::write(path, binary).or_else(|_| fs_err::write(path, hex)) { + let write = |data: &[u8]| -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new().write(true).open(path)?; + file.write_all(data) + }; + match write(binary).or_else(|_| write(hex)) { Ok(()) => return Ok(()), Err(err) => last_err = Some(err), } diff --git a/dstack/sev-snp-qvl/Cargo.toml b/dstack/sev-snp-qvl/Cargo.toml index 58e4166d7..4d16e4500 100644 --- a/dstack/sev-snp-qvl/Cargo.toml +++ b/dstack/sev-snp-qvl/Cargo.toml @@ -17,3 +17,7 @@ moka.workspace = true reqwest.workspace = true sev.workspace = true tokio = { workspace = true, features = ["rt", "time"] } +x509-parser = { workspace = true, features = ["verify"] } +rustls-pki-types.workspace = true +rustls-webpki = { workspace = true, features = ["ring"] } +pem.workspace = true diff --git a/dstack/sev-snp-qvl/src/lib.rs b/dstack/sev-snp-qvl/src/lib.rs index 75add0834..1fa4da6f5 100644 --- a/dstack/sev-snp-qvl/src/lib.rs +++ b/dstack/sev-snp-qvl/src/lib.rs @@ -10,7 +10,7 @@ //! must still bind the verified measurement to app/config identity before //! production key release. -use std::{thread, time::Duration}; +use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use moka::sync::Cache; @@ -27,8 +27,7 @@ const VLEK_CERT_GUID: [u8; 16] = [ 0xa8, 0x07, 0x4b, 0xc2, 0xa2, 0x5a, 0x48, 0x3e, 0xaa, 0xe6, 0x39, 0xc0, 0x45, 0xa0, 0xb8, 0xa1, ]; const CERT_TABLE_ENTRY_SIZE: usize = 24; -const AMD_KDS_BASE_URL_ENV: &str = "DSTACK_AMD_KDS_BASE_URL"; -const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; +pub const AMD_KDS_DEFAULT_BASE_URL: &str = "https://kdsintf.amd.com/vcek/v1"; const AMD_KDS_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); const AMD_KDS_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); const AMD_KDS_CA_CACHE_CAPACITY: u64 = 16; @@ -41,6 +40,136 @@ pub enum AmdSnpProduct { Turin, } +/// AMD SEV-SNP quote verifier with caller-owned ARK trust anchors. +/// +/// One ARK is required per supported processor family because AMD operates a +/// separate hierarchy for each family. `new_prod` uses the ARKs shipped by the +/// `sev` crate; `new` accepts PEM encoded ARKs from the caller. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + milan_ark_pem: Vec, + genoa_ark_pem: Vec, + turin_ark_pem: Vec, + external_roots: [bool; 3], +} + +impl QuoteVerifier { + pub fn new(milan_ark_pem: Vec, genoa_ark_pem: Vec, turin_ark_pem: Vec) -> Self { + Self { + milan_ark_pem, + genoa_ark_pem, + turin_ark_pem, + external_roots: [true; 3], + } + } + + pub fn new_prod() -> Self { + Self { + milan_ark_pem: builtin::milan::ARK.to_vec(), + genoa_ark_pem: builtin::genoa::ARK.to_vec(), + turin_ark_pem: builtin::turin::ARK.to_vec(), + external_roots: [false; 3], + } + } + + /// Create a production verifier overriding one product-family ARK. + pub fn new_with_root(product: AmdSnpProduct, ark_pem: Vec) -> Self { + Self::new_prod().with_root(product, ark_pem) + } + + /// Override one product-family ARK. + pub fn with_root(mut self, product: AmdSnpProduct, ark_pem: Vec) -> Self { + match product { + AmdSnpProduct::Milan => { + self.milan_ark_pem = ark_pem; + self.external_roots[0] = true; + } + AmdSnpProduct::Genoa => { + self.genoa_ark_pem = ark_pem; + self.external_roots[1] = true; + } + AmdSnpProduct::Turin => { + self.turin_ark_pem = ark_pem; + self.external_roots[2] = true; + } + } + self + } + + fn ark(&self, product: AmdSnpProduct) -> CertBytes { + CertBytes { + bytes: match product { + AmdSnpProduct::Milan => self.milan_ark_pem.clone(), + AmdSnpProduct::Genoa => self.genoa_ark_pem.clone(), + AmdSnpProduct::Turin => self.turin_ark_pem.clone(), + }, + encoding: CertEncoding::Pem, + } + } + + fn uses_external_root(&self, product: AmdSnpProduct) -> bool { + self.external_roots[match product { + AmdSnpProduct::Milan => 0, + AmdSnpProduct::Genoa => 1, + AmdSnpProduct::Turin => 2, + }] + } + + pub fn verify( + &self, + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], + ) -> Result { + let (ask, vcek) = normalize_ask_vcek_certs(cert_chain)?; + let verified = + verify_amd_snp_attestation_with_certs_and_arks(report, ask, vcek, |product| { + (self.ark(product), self.uses_external_root(product)) + })?; + if &verified.report_data != expected_report_data { + bail!("amd sev-snp report_data mismatch"); + } + Ok(verified) + } + + pub async fn fetch_and_verify( + &self, + kds_client: &AmdKdsClient, + report: &[u8], + cert_chain: &[Vec], + expected_report_data: &[u8; 64], + ) -> Result { + if !cert_chain.is_empty() { + return self.verify(report, cert_chain, expected_report_data); + } + let report_obj = AttestationReport::from_bytes(report) + .map_err(|err| anyhow!("failed to parse amd sev-snp report: {err}"))?; + let mut errors = Vec::new(); + for product in amd_snp_product_candidates_for_report(&report_obj)? { + match kds_client + .fetch_collateral_for_product(product, &report_obj) + .await + { + Ok(collateral) => match verify_amd_snp_attestation_with_cert_chain( + report, + self.ark(product), + collateral.ask, + collateral.vcek, + self.uses_external_root(product), + ) { + Ok(verified) if &verified.report_data == expected_report_data => { + return Ok(verified); + } + Ok(_) => bail!("amd sev-snp report_data mismatch"), + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + }, + Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), + } + } + bail!("amd sev-snp verification failed: {}", errors.join("; ")) + } +} + impl AmdSnpProduct { fn kds_name(self) -> &'static str { match self { @@ -181,7 +310,7 @@ pub struct AmdKdsClient { impl AmdKdsClient { pub fn new() -> Result { - Self::with_base_url(amd_kds_base_url()) + Self::with_base_url(AMD_KDS_DEFAULT_BASE_URL) } pub fn with_base_url(base_url: impl AsRef) -> Result { @@ -199,56 +328,6 @@ impl AmdKdsClient { }) } - pub async fn verify_evidence_with_kds_fallback( - &self, - report: &[u8], - cert_chain: &[Vec], - expected_report_data: &[u8; 64], - ) -> Result { - if !cert_chain.is_empty() { - return verify_amd_snp_evidence(report, cert_chain, expected_report_data); - } - if report.len() != 1184 { - bail!( - "invalid amd sev-snp report length: expected 1184 bytes, got {}", - report.len() - ); - } - let report_obj = AttestationReport::from_bytes(report) - .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; - let collateral = self - .fetch_collateral_for_report(&report_obj) - .await - .context("failed to fetch amd sev-snp KDS collateral for empty cert_chain")?; - let verified = verify_amd_snp_attestation_with_cert_chain( - report, - collateral.ark, - collateral.ask, - collateral.vcek, - )?; - if &verified.report_data != expected_report_data { - bail!("amd sev-snp report_data mismatch"); - } - Ok(verified) - } - - async fn fetch_collateral_for_report( - &self, - report: &AttestationReport, - ) -> Result { - let mut errors = Vec::new(); - for product in amd_snp_product_candidates_for_report(report)? { - match self.fetch_collateral_for_product(product, report).await { - Ok(collateral) => return Ok(collateral), - Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), - } - } - bail!( - "amd sev-snp KDS collateral unavailable for supported products: {}", - errors.join("; ") - ) - } - async fn fetch_collateral_for_product( &self, product: AmdSnpProduct, @@ -400,6 +479,17 @@ fn verify_amd_snp_attestation_with_certs( report_bytes: &[u8], ask_bytes: CertBytes, vcek_bytes: CertBytes, +) -> Result { + verify_amd_snp_attestation_with_certs_and_arks(report_bytes, ask_bytes, vcek_bytes, |product| { + (product.builtin_ark(), false) + }) +} + +fn verify_amd_snp_attestation_with_certs_and_arks( + report_bytes: &[u8], + ask_bytes: CertBytes, + vcek_bytes: CertBytes, + ark_for_product: impl Fn(AmdSnpProduct) -> (CertBytes, bool), ) -> Result { if report_bytes.len() != 1184 { bail!( @@ -411,11 +501,13 @@ fn verify_amd_snp_attestation_with_certs( .map_err(|err| anyhow::anyhow!("failed to parse amd sev-snp report: {err}"))?; let mut errors = Vec::new(); for product in amd_snp_product_candidates_for_report(&report)? { + let (ark, external_root) = ark_for_product(product); match verify_amd_snp_attestation_with_cert_chain( report_bytes, - product.builtin_ark(), + ark, ask_bytes.clone(), vcek_bytes.clone(), + external_root, ) { Ok(verified) => return Ok(verified), Err(err) => errors.push(format!("{}: {err:#}", product.kds_name())), @@ -432,6 +524,7 @@ fn verify_amd_snp_attestation_with_cert_chain( ark_bytes: CertBytes, ask_bytes: CertBytes, vcek_bytes: CertBytes, + external_root: bool, ) -> Result { if report_bytes.len() != 1184 { bail!( @@ -445,14 +538,16 @@ fn verify_amd_snp_attestation_with_cert_chain( let ark = parse_certificate(&ark_bytes, "ark")?; let ask = parse_certificate(&ask_bytes, "ask")?; let vcek = parse_certificate(&vcek_bytes, "vcek")?; - - let chain = Chain { - ca: ca::Chain { ark, ask }, - vek: vcek.clone(), - }; - chain + if external_root { + verify_x509_chain(&ark_bytes, &ask_bytes, &vcek_bytes)?; + } else { + Chain { + ca: ca::Chain { ark, ask }, + vek: vcek.clone(), + } .verify() .map_err(|err| anyhow::anyhow!("amd cert chain verification failed: {err:?}"))?; + } (&vcek, &report).verify().map_err(|err| { anyhow::anyhow!("amd sev-snp report signature verification failed: {err:?}") })?; @@ -474,6 +569,97 @@ fn verify_amd_snp_attestation_with_cert_chain( }) } +fn verify_x509_chain(ark: &CertBytes, ask: &CertBytes, vcek: &CertBytes) -> Result<()> { + use x509_parser::prelude::{FromDer, X509Certificate}; + + let ark_der = certificate_der(ark)?; + let ask_der = certificate_der(ask)?; + let vcek_der = certificate_der(vcek)?; + let (_, ark) = X509Certificate::from_der(&ark_der).context("failed to parse ARK X.509")?; + let (_, ask) = X509Certificate::from_der(&ask_der).context("failed to parse ASK X.509")?; + let (_, vcek) = X509Certificate::from_der(&vcek_der).context("failed to parse VCEK X.509")?; + if !ark.is_ca() || !ask.is_ca() { + bail!("AMD ARK and ASK certificates must be certificate authorities"); + } + if ark.subject() != ark.issuer() + || ask.issuer() != ark.subject() + || vcek.issuer() != ask.subject() + { + bail!("AMD certificate issuer/subject chain mismatch"); + } + verify_certificate_signature(&ark_der, &ark) + .context("ARK self-signature verification failed")?; + verify_certificate_signature(&ark_der, &ask).context("ASK signature verification failed")?; + verify_certificate_signature(&ask_der, &vcek).context("VCEK signature verification failed")?; + Ok(()) +} + +fn verify_certificate_signature( + issuer_der: &[u8], + certificate: &x509_parser::certificate::X509Certificate<'_>, +) -> Result<()> { + use rustls_pki_types::CertificateDer; + const RSA_PSS_OID: &str = "1.2.840.113549.1.1.10"; + const ECDSA_SHA384_OID: &str = "1.2.840.10045.4.3.3"; + const SHA384_OID: &str = "2.16.840.1.101.3.4.2.2"; + const MGF1_OID: &str = "1.2.840.113549.1.1.8"; + + anyhow::ensure!( + certificate.signature_algorithm == certificate.tbs_certificate.signature, + "certificate signatureAlgorithm does not match TBSCertificate signature" + ); + let oid = certificate.signature_algorithm.algorithm.to_id_string(); + let algorithm = match oid.as_str() { + RSA_PSS_OID => { + use x509_parser::signature_algorithm::SignatureAlgorithm; + let SignatureAlgorithm::RSASSA_PSS(params) = + SignatureAlgorithm::try_from(&certificate.signature_algorithm) + .context("invalid RSA-PSS signature parameters")? + else { + bail!("invalid RSA-PSS signature parameters"); + }; + let mask = params + .mask_gen_algorithm() + .context("invalid RSA-PSS mask generation parameters")?; + anyhow::ensure!( + params.hash_algorithm_oid().to_id_string() == SHA384_OID + && mask.mgf.to_id_string() == MGF1_OID + && mask.hash.to_id_string() == SHA384_OID + && params.salt_length() == 48 + && params.trailer_field() == 1, + "RSA-PSS certificate signature must use SHA-384, MGF1-SHA384, and a 48-byte salt" + ); + webpki::ring::RSA_PSS_2048_8192_SHA384_LEGACY_KEY + } + ECDSA_SHA384_OID => { + anyhow::ensure!( + certificate.signature_algorithm.parameters.is_none(), + "ECDSA-SHA384 signature parameters must be absent" + ); + webpki::ring::ECDSA_P384_SHA384 + } + _ => bail!("unsupported certificate signature algorithm: {oid}"), + }; + let issuer_der = CertificateDer::from(issuer_der); + let issuer = webpki::EndEntityCert::try_from(&issuer_der) + .context("failed to parse certificate issuer")?; + let message = certificate.tbs_certificate.as_ref(); + let signature = certificate.signature_value.data.as_ref(); + issuer + .verify_signature(algorithm, message, signature) + .context("invalid certificate signature")?; + Ok(()) +} + +fn certificate_der(cert: &CertBytes) -> Result> { + match cert.encoding { + CertEncoding::Der => Ok(cert.bytes.clone()), + CertEncoding::Pem => Ok(::pem::parse(&cert.bytes) + .context("failed to parse certificate PEM")? + .into_contents()), + } +} + pub fn verify_amd_snp_evidence( report: &[u8], cert_chain: &[Vec], @@ -487,51 +673,6 @@ pub fn verify_amd_snp_evidence( Ok(verified) } -pub fn verify_amd_snp_evidence_with_kds_fallback( - report: &[u8], - cert_chain: &[Vec], - expected_report_data: &[u8; 64], -) -> Result { - if !cert_chain.is_empty() { - return verify_amd_snp_evidence(report, cert_chain, expected_report_data); - } - - let report = report.to_vec(); - let expected_report_data = *expected_report_data; - thread::spawn(move || -> Result { - let kds_client = AmdKdsClient::new()?; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("failed to create amd sev-snp KDS fallback runtime")?; - runtime.block_on(kds_client.verify_evidence_with_kds_fallback( - &report, - &[], - &expected_report_data, - )) - }) - .join() - .map_err(|_| anyhow!("amd sev-snp KDS fallback worker panicked"))? -} - -pub async fn verify_amd_snp_evidence_with_kds_fallback_async( - report: &[u8], - cert_chain: &[Vec], - expected_report_data: &[u8; 64], -) -> Result { - AmdKdsClient::new()? - .verify_evidence_with_kds_fallback(report, cert_chain, expected_report_data) - .await -} - -fn amd_kds_base_url() -> String { - std::env::var(AMD_KDS_BASE_URL_ENV) - .ok() - .map(|url| url.trim().trim_end_matches('/').to_string()) - .filter(|url| !url.is_empty()) - .unwrap_or_else(|| AMD_KDS_DEFAULT_BASE_URL.to_string()) -} - fn normalize_amd_kds_base_url(base_url: &str) -> Result { let base_url = base_url.trim().trim_end_matches('/').to_string(); if base_url.is_empty() { diff --git a/dstack/tee-simulator/Cargo.toml b/dstack/tee-simulator/Cargo.toml index 1a7ce39db..b235d5467 100644 --- a/dstack/tee-simulator/Cargo.toml +++ b/dstack/tee-simulator/Cargo.toml @@ -17,12 +17,29 @@ path = "src/main.rs" anyhow.workspace = true cc-eventlog.workspace = true clap.workspace = true +dstack-types.workspace = true +dstack-mr.workspace = true fuser.workspace = true libc.workspace = true sd-notify.workspace = true sha2.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +serde_json.workspace = true +fs-err.workspace = true +hex.workspace = true +mock-attestation = { path = "../crates/mock-attestation" } +tokio = { workspace = true, features = ["full"] } +libloading = "0.8" +pem.workspace = true +aws-nitro-enclaves-nsm-api = "0.4" +serde_cbor = "0.11" [dev-dependencies] dcap-qvl.workspace = true +tempfile.workspace = true +reqwest.workspace = true +sev-snp-qvl.workspace = true +tpm-qvl.workspace = true +tpm-types.workspace = true +nsm-qvl.workspace = true diff --git a/dstack/tee-simulator/src/main.rs b/dstack/tee-simulator/src/main.rs index 0021ca16b..9e89e2b5c 100644 --- a/dstack/tee-simulator/src/main.rs +++ b/dstack/tee-simulator/src/main.rs @@ -2,31 +2,45 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::path::{Path, PathBuf}; +use std::{ + ffi::CString, + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, +}; use anyhow::{bail, Context, Result}; -use clap::{Parser, ValueEnum}; +use clap::Parser; +use dstack_types::{SysConfig, TeeSimulatorConfig, TeeVariant}; use fuser::{Filesystem, MountOption, Session}; use tracing::info; +mod nsm; +mod sev_snp; mod tdx; - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] -enum TeePlatform { - #[default] - Tdx, -} +mod tpm; #[derive(Parser)] #[command(about = "Development-only simulator for Linux TEE guest ABIs")] struct Args { /// TEE platform ABI to simulate. - #[arg(long, value_enum, default_value = "tdx")] - platform: TeePlatform, + #[arg(long)] + platform: Option, + + /// sys-config used to select the simulated platform when --platform is omitted. + #[arg(long, default_value = "/dstack/.host-shared/.sys-config.json")] + sys_config: PathBuf, /// Override the platform backend's default mountpoint. #[arg(long)] mountpoint: Option, + + /// Runtime state directory (overridable by unprivileged E2E tests). + #[arg(long, default_value = "/run/dstack")] + runtime_dir: PathBuf, + + /// DMI sysfs directory. Tests may override this with a temporary directory. + #[arg(long, default_value = "/sys/class/dmi/id")] + dmi_root: PathBuf, } /// Platform-specific simulator backend. @@ -39,12 +53,15 @@ trait TeeBackend { const PLATFORM: &'static str; const DEFAULT_MOUNTPOINT: &'static str; - fn create_filesystem() -> Result; + fn create_filesystem(config: &TeeSimulatorConfig) -> Result; fn prepare_mountpoint(mountpoint: &Path) -> Result<()>; fn real_tee_available() -> bool; } -fn run_backend(mountpoint_override: Option) -> Result<()> { +fn run_backend( + mountpoint_override: Option, + config: &TeeSimulatorConfig, +) -> Result<()> { let default_mountpoint = Path::new(B::DEFAULT_MOUNTPOINT); let mountpoint = mountpoint_override.as_deref().unwrap_or(default_mountpoint); @@ -56,7 +73,7 @@ fn run_backend(mountpoint_override: Option) -> Result<() } B::prepare_mountpoint(mountpoint)?; - let fs = B::create_filesystem()?; + let fs = B::create_filesystem(config)?; let mut session = Session::new( fs, mountpoint, @@ -87,9 +104,99 @@ fn main() -> Result<()> { .init(); let args = Args::parse(); - match args.platform { - TeePlatform::Tdx => run_backend::(args.mountpoint), + let config = load_config(&args.sys_config)?; + let platform = args.platform.unwrap_or(config.platform); + fs_err::create_dir_all(&args.runtime_dir)?; + match platform { + TeeVariant::DstackTdx => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Dstack", "dstack")?; + run_backend::(args.mountpoint, &config) + } + TeeVariant::DstackGcpTdx => { + simulate_dmi( + &args.runtime_dir, + &args.dmi_root, + "Google", + "Google Compute Engine", + )?; + tpm::start_gcp_vtpm(&args.runtime_dir, &config)?; + run_backend::(args.mountpoint, &config) + } + TeeVariant::DstackNitroEnclave => { + simulate_dmi( + &args.runtime_dir, + &args.dmi_root, + "AWS Nitro Enclaves", + "Nitro Enclave", + )?; + nsm::run(&config) + } + TeeVariant::DstackAwsNitroTpm => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Amazon EC2", "t3.metal")?; + tpm::run_nitro_vtpm(&args.runtime_dir, &config) + } + TeeVariant::DstackAmdSevSnp => { + simulate_dmi(&args.runtime_dir, &args.dmi_root, "Dstack", "dstack")?; + run_backend::(args.mountpoint, &config) + } + } +} + +/// Override the DMI strings exported by SeaBIOS through sysfs. Platform +/// detection intentionally remains unchanged and observes the same values as +/// it does on a real cloud VM. +fn simulate_dmi( + runtime_dir: &Path, + dmi_root: &Path, + sys_vendor: &str, + product_name: &str, +) -> Result<()> { + if dmi_root != Path::new("/sys/class/dmi/id") { + fs_err::create_dir_all(dmi_root)?; + fs_err::write(dmi_root.join("sys_vendor"), format!("{sys_vendor}\n"))?; + fs_err::write(dmi_root.join("product_name"), format!("{product_name}\n"))?; + return Ok(()); + } + let dmi_dir = runtime_dir.join("dmi"); + fs_err::create_dir_all(&dmi_dir)?; + for (name, value) in [("sys_vendor", sys_vendor), ("product_name", product_name)] { + let source = dmi_dir.join(name); + let target = dmi_root.join(name); + fs_err::write(&source, format!("{value}\n"))?; + let source = CString::new(source.as_os_str().as_bytes())?; + let target_c = CString::new(target.as_os_str().as_bytes())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + std::ptr::null(), + libc::MS_BIND, + std::ptr::null(), + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error()).with_context(|| { + format!("failed to simulate SeaBIOS DMI file {}", target.display()) + }); + } + } + Ok(()) +} + +fn load_config(path: &Path) -> Result { + if !path.exists() { + return Ok(TeeSimulatorConfig::default()); } + let config: SysConfig = serde_json::from_slice( + &fs_err::read(path).with_context(|| format!("failed to read {}", path.display()))?, + ) + .with_context(|| format!("failed to parse {}", path.display()))?; + let mr_config = config.mr_config_document(); + let vm_config = (!config.vm_config.is_empty()).then_some(config.vm_config.clone()); + let mut tee_simulator = config.tee_simulator.unwrap_or_default(); + tee_simulator.mr_config = mr_config; + tee_simulator.vm_config = vm_config; + Ok(tee_simulator) } #[cfg(test)] @@ -99,7 +206,42 @@ mod tests { #[test] fn defaults_to_tdx() { let args = Args::try_parse_from(["dstack-tee-simulator"]).unwrap(); - assert_eq!(args.platform, TeePlatform::Tdx); + assert_eq!(args.platform, None); assert!(args.mountpoint.is_none()); } + + #[test] + fn missing_sys_config_defaults_to_tdx() { + assert_eq!( + load_config(Path::new("/definitely/missing/sys-config")) + .unwrap() + .platform, + TeeVariant::DstackTdx + ); + } + + #[test] + fn sys_config_selects_each_platform() { + for (name, expected) in [ + ("dstack-tdx", TeeVariant::DstackTdx), + ("dstack-gcp-tdx", TeeVariant::DstackGcpTdx), + ("dstack-amd-sev-snp", TeeVariant::DstackAmdSevSnp), + ("dstack-nitro-enclave", TeeVariant::DstackNitroEnclave), + ("dstack-aws-nitro-tpm", TeeVariant::DstackAwsNitroTpm), + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sys-config.json"); + fs_err::write( + &path, + serde_json::json!({ + "kms_urls": [], "gateway_urls": [], "pccs_url": null, + "docker_registry": null, "host_api_url": null, "vm_config": "{}", + "tee_simulator": {"platform": name} + }) + .to_string(), + ) + .unwrap(); + assert_eq!(load_config(&path).unwrap().platform, expected); + } + } } diff --git a/dstack/tee-simulator/src/nsm.rs b/dstack/tee-simulator/src/nsm.rs new file mode 100644 index 000000000..c0ef2de28 --- /dev/null +++ b/dstack/tee-simulator/src/nsm.rs @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! CUSE implementation of the AWS Nitro Security Module ioctl ABI. + +use std::{ + ffi::CString, + mem, + os::raw::{c_int, c_uint, c_void}, + ptr, + sync::{Arc, OnceLock}, +}; + +use anyhow::{Context, Result}; +use aws_nitro_enclaves_nsm_api::api::{Digest, ErrorCode, Request, Response}; +use dstack_types::TeeSimulatorConfig; +use libc::{iovec, size_t}; +use mock_attestation::{nsm::NsmGenerator, parse_seed}; + +static GENERATOR: OnceLock> = OnceLock::new(); +static CUSE: OnceLock = OnceLock::new(); + +type FuseReq = *mut c_void; + +#[repr(C)] +struct FuseFileInfo { + _opaque: [u8; 0], +} + +#[repr(C)] +struct CuseInfo { + dev_major: c_uint, + dev_minor: c_uint, + dev_info_argc: c_uint, + dev_info_argv: *mut *const libc::c_char, + flags: c_uint, +} + +#[repr(C)] +struct CuseOperations { + init: *const c_void, + init_done: *const c_void, + destroy: *const c_void, + open: Option, + read: *const c_void, + write: *const c_void, + flush: *const c_void, + release: Option, + fsync: *const c_void, + ioctl: Option< + unsafe extern "C" fn( + FuseReq, + c_int, + *mut c_void, + *mut FuseFileInfo, + c_uint, + *const c_void, + size_t, + size_t, + ), + >, + poll: *const c_void, +} + +struct CuseApi { + main: unsafe extern "C" fn( + c_int, + *mut *mut libc::c_char, + *const CuseInfo, + *const CuseOperations, + *mut c_void, + ) -> c_int, + reply_open: unsafe extern "C" fn(FuseReq, *const FuseFileInfo) -> c_int, + reply_err: unsafe extern "C" fn(FuseReq, c_int) -> c_int, + reply_ioctl_retry: + unsafe extern "C" fn(FuseReq, *const iovec, size_t, *const iovec, size_t) -> c_int, + reply_ioctl: unsafe extern "C" fn(FuseReq, c_int, *const c_void, size_t) -> c_int, +} + +fn cuse() -> &'static CuseApi { + let Some(cuse) = CUSE.get() else { + std::process::abort(); + }; + cuse +} + +unsafe fn load_cuse() -> Result { + let library = Box::leak(Box::new(libloading::Library::new("libfuse3.so.3")?)); + Ok(CuseApi { + main: *library.get(b"cuse_lowlevel_main\0")?, + reply_open: *library.get(b"fuse_reply_open\0")?, + reply_err: *library.get(b"fuse_reply_err\0")?, + reply_ioctl_retry: *library.get(b"fuse_reply_ioctl_retry\0")?, + reply_ioctl: *library.get(b"fuse_reply_ioctl\0")?, + }) +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct NsmMessage { + request: iovec, + response: iovec, +} + +unsafe extern "C" fn open(req: FuseReq, fi: *mut FuseFileInfo) { + (cuse().reply_open)(req, fi); +} + +unsafe extern "C" fn release(req: FuseReq, _fi: *mut FuseFileInfo) { + (cuse().reply_err)(req, 0); +} + +unsafe extern "C" fn ioctl( + req: FuseReq, + _cmd: c_int, + arg: *mut c_void, + _fi: *mut FuseFileInfo, + _flags: c_uint, + in_buf: *const c_void, + in_bufsz: size_t, + _out_bufsz: size_t, +) { + let message_size = mem::size_of::(); + if in_bufsz == 0 { + let message = iovec { + iov_base: arg, + iov_len: message_size, + }; + (cuse().reply_ioctl_retry)(req, &message, 1, &message, 1); + return; + } + if in_buf.is_null() || in_bufsz < message_size { + (cuse().reply_err)(req, libc::EINVAL); + return; + } + + let message = ptr::read_unaligned(in_buf.cast::()); + if in_bufsz == message_size { + if message.request.iov_len > 0x1000 || message.response.iov_len > 0x3000 { + (cuse().reply_err)(req, libc::EMSGSIZE); + return; + } + let input = [ + iovec { + iov_base: arg, + iov_len: message_size, + }, + message.request, + ]; + let output = [ + iovec { + iov_base: arg, + iov_len: message_size, + }, + message.response, + ]; + (cuse().reply_ioctl_retry)(req, input.as_ptr(), 2, output.as_ptr(), 2); + return; + } + + let request_bytes = std::slice::from_raw_parts( + in_buf.cast::().add(message_size), + in_bufsz - message_size, + ); + let response = match serde_cbor::from_slice::(request_bytes) { + Ok(request) => handle(request), + Err(_) => Response::Error(ErrorCode::InvalidOperation), + }; + let encoded = match serde_cbor::to_vec(&response) { + Ok(encoded) if encoded.len() <= message.response.iov_len => encoded, + _ => { + (cuse().reply_err)(req, libc::EMSGSIZE); + return; + } + }; + let mut updated = message; + updated.response.iov_len = encoded.len(); + let mut output = Vec::with_capacity(message_size + encoded.len()); + output.extend_from_slice(std::slice::from_raw_parts( + (&updated as *const NsmMessage).cast::(), + message_size, + )); + output.extend_from_slice(&encoded); + (cuse().reply_ioctl)(req, 0, output.as_ptr().cast(), output.len()); +} + +fn handle(request: Request) -> Response { + match request { + Request::Attestation { user_data, .. } => GENERATOR + .get() + .and_then(|generator| { + let user_data = user_data + .as_ref() + .map(|data| data.as_slice()) + .unwrap_or_default(); + generator.attest(user_data).ok() + }) + .map(|document| Response::Attestation { document }) + .unwrap_or(Response::Error(ErrorCode::InternalError)), + Request::DescribeNSM => Response::DescribeNSM { + version_major: 1, + version_minor: 0, + version_patch: 0, + module_id: "dstack-tee-simulator".into(), + max_pcrs: 32, + locked_pcrs: Default::default(), + digest: Digest::SHA384, + }, + Request::GetRandom => Response::GetRandom { + random: vec![0x42; 32], + }, + _ => Response::Error(ErrorCode::InvalidOperation), + } +} + +pub fn run(config: &TeeSimulatorConfig) -> Result<()> { + anyhow::ensure!( + !std::path::Path::new("/dev/nsm").exists(), + "refusing to replace a real NSM device" + ); + CUSE.set(unsafe { load_cuse()? }) + .map_err(|_| anyhow::anyhow!("CUSE API was already initialized"))?; + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + GENERATOR + .set(Arc::new(NsmGenerator::from_seed(parse_seed(seed)?)?)) + .map_err(|_| anyhow::anyhow!("NSM simulator was already initialized"))?; + + let device_name = CString::new("DEVNAME=nsm")?; + let mut device_args = [device_name.as_ptr(), ptr::null()]; + let info = CuseInfo { + dev_major: 0, + dev_minor: 0, + dev_info_argc: 1, + dev_info_argv: device_args.as_mut_ptr(), + flags: 1, // CUSE_UNRESTRICTED_IOCTL + }; + let operations = CuseOperations { + init: ptr::null(), + init_done: ptr::null(), + destroy: ptr::null(), + open: Some(open), + read: ptr::null(), + write: ptr::null(), + flush: ptr::null(), + release: Some(release), + fsync: ptr::null(), + ioctl: Some(ioctl), + poll: ptr::null(), + }; + sd_notify::notify(true, &[sd_notify::NotifyState::Ready])?; + let program = CString::new("dstack-tee-simulator")?; + let foreground = CString::new("-f")?; + let single_threaded = CString::new("-s")?; + let mut argv = [ + program.as_ptr().cast_mut(), + foreground.as_ptr().cast_mut(), + single_threaded.as_ptr().cast_mut(), + ]; + let result = unsafe { + (cuse().main)( + argv.len() as c_int, + argv.as_mut_ptr(), + &info, + &operations, + ptr::null_mut(), + ) + }; + anyhow::ensure!(result == 0, "CUSE NSM session failed with status {result}"); + Ok(()) +} diff --git a/dstack/tee-simulator/src/sev_snp.rs b/dstack/tee-simulator/src/sev_snp.rs new file mode 100644 index 000000000..c1409cfde --- /dev/null +++ b/dstack/tee-simulator/src/sev_snp.rs @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + ffi::OsStr, + path::Path, + sync::Arc, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context, Result}; +use dstack_types::TeeSimulatorConfig; +use fuser::{ + FileAttr, FileType, Filesystem, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, ReplyEmpty, + ReplyEntry, ReplyOpen, ReplyWrite, Request, +}; +use mock_attestation::sev_snp::SevSnpGenerator; + +use crate::TeeBackend; + +const ROOT: u64 = 1; +const ENTRY: u64 = 2; +const PROVIDER: u64 = 3; +const INBLOB: u64 = 4; +const OUTBLOB: u64 = 5; +const CERTS: u64 = 6; +const TTL: Duration = Duration::ZERO; + +pub(crate) struct SevSnpFs { + generator: Arc, + report: Vec, + certs: Vec, + uid: u32, + gid: u32, + host_data: [u8; 32], +} + +impl SevSnpFs { + fn new( + generator: Arc, + host_data: [u8; 32], + measurement: [u8; 48], + ) -> Result { + let evidence = generator.attest_with_measurement([0; 64], host_data, measurement)?; + Ok(Self { + generator, + report: evidence.report, + certs: encode_cert_table(&evidence.cert_chain)?, + uid: unsafe { libc::geteuid() }, + gid: unsafe { libc::getegid() }, + host_data, + }) + } + + fn attr(&self, ino: u64) -> Option { + let (kind, perm, size) = match ino { + ROOT | ENTRY => (FileType::Directory, 0o755, 0), + PROVIDER => (FileType::RegularFile, 0o444, 10), + INBLOB => (FileType::RegularFile, 0o200, 0), + OUTBLOB => (FileType::RegularFile, 0o400, self.report.len() as u64), + CERTS => (FileType::RegularFile, 0o400, self.certs.len() as u64), + _ => return None, + }; + Some(FileAttr { + ino, + size, + blocks: 0, + atime: SystemTime::UNIX_EPOCH, + mtime: SystemTime::UNIX_EPOCH, + ctime: SystemTime::UNIX_EPOCH, + crtime: SystemTime::UNIX_EPOCH, + kind, + perm, + nlink: if kind == FileType::Directory { 2 } else { 1 }, + uid: self.uid, + gid: self.gid, + rdev: 0, + blksize: 4096, + flags: 0, + }) + } + + fn child(parent: u64, name: &OsStr) -> Option { + match (parent, name.to_str()?) { + (ROOT, "provider") | (ENTRY, "provider") => Some(PROVIDER), + (ENTRY, "inblob" | "reportdata" | "report_data") => Some(INBLOB), + (ENTRY, "outblob" | "report") => Some(OUTBLOB), + (ENTRY, "certs" | "cert_chain" | "auxblob") => Some(CERTS), + (ROOT, name) if name.starts_with("dstack-") => Some(ENTRY), + _ => None, + } + } +} + +impl Filesystem for SevSnpFs { + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + match Self::child(parent, name).and_then(|ino| self.attr(ino).map(|attr| (ino, attr))) { + Some((_, attr)) => reply.entry(&TTL, &attr, 0), + None => reply.error(libc::ENOENT), + } + } + fn getattr(&mut self, _req: &Request<'_>, ino: u64, _fh: Option, reply: ReplyAttr) { + match self.attr(ino) { + Some(attr) => reply.attr(&TTL, &attr), + None => reply.error(libc::ENOENT), + } + } + fn mkdir( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + reply: ReplyEntry, + ) { + if parent == ROOT + && name + .to_str() + .is_some_and(|name| name.starts_with("dstack-")) + { + match self.attr(ENTRY) { + Some(attr) => reply.entry(&TTL, &attr, 0), + None => reply.error(libc::EIO), + } + } else { + reply.error(libc::EACCES); + } + } + fn rmdir(&mut self, _req: &Request<'_>, parent: u64, _name: &OsStr, reply: ReplyEmpty) { + if parent == ROOT { + reply.ok(); + } else { + reply.error(libc::EACCES); + } + } + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + let entries: &[(u64, FileType, &str)] = match ino { + ROOT => &[ + (ROOT, FileType::Directory, "."), + (ROOT, FileType::Directory, ".."), + (PROVIDER, FileType::RegularFile, "provider"), + ], + ENTRY => &[ + (ENTRY, FileType::Directory, "."), + (ROOT, FileType::Directory, ".."), + (PROVIDER, FileType::RegularFile, "provider"), + (INBLOB, FileType::RegularFile, "inblob"), + (OUTBLOB, FileType::RegularFile, "outblob"), + (CERTS, FileType::RegularFile, "certs"), + ], + _ => { + reply.error(libc::ENOENT); + return; + } + }; + for (index, entry) in entries.iter().enumerate().skip(offset.max(0) as usize) { + if reply.add(entry.0, (index + 1) as i64, entry.1, entry.2) { + break; + } + } + reply.ok(); + } + fn open(&mut self, _req: &Request<'_>, ino: u64, _flags: i32, reply: ReplyOpen) { + if matches!(ino, PROVIDER | INBLOB | OUTBLOB | CERTS) { + reply.opened(0, 1); + } else { + reply.error(libc::ENOENT); + } + } + fn create( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + _flags: i32, + reply: ReplyCreate, + ) { + match Self::child(parent, name).filter(|ino| *ino == INBLOB) { + Some(ino) => match self.attr(ino) { + Some(attr) => reply.created(&TTL, &attr, 0, 0, 1), + None => reply.error(libc::EIO), + }, + None => reply.error(libc::EACCES), + } + } + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock: Option, + reply: ReplyData, + ) { + let data: &[u8] = match ino { + PROVIDER => b"sev_guest\n", + OUTBLOB => &self.report, + CERTS => &self.certs, + _ => { + reply.error(libc::EACCES); + return; + } + }; + let start = offset.max(0) as usize; + reply.data( + data.get(start..start.saturating_add(size as usize).min(data.len())) + .unwrap_or_default(), + ); + } + fn write( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + data: &[u8], + _write_flags: u32, + _flags: i32, + _lock: Option, + reply: ReplyWrite, + ) { + if ino != INBLOB || offset != 0 || data.len() != 64 { + reply.error(libc::EINVAL); + return; + } + let Ok(report_data) = data.try_into() else { + reply.error(libc::EINVAL); + return; + }; + let Ok(measurement) = self.report[0x90..0xc0].try_into() else { + reply.error(libc::EIO); + return; + }; + match self + .generator + .attest_with_measurement(report_data, self.host_data, measurement) + { + Ok(evidence) => { + self.report = evidence.report; + self.certs = match encode_cert_table(&evidence.cert_chain) { + Ok(certs) => certs, + Err(_) => { + reply.error(libc::EIO); + return; + } + }; + reply.written(data.len() as u32); + } + Err(_) => reply.error(libc::EIO), + } + } +} + +fn encode_cert_table(cert_chain: &[Vec]) -> Result> { + const ASK_GUID: [u8; 16] = [ + 0x4a, 0xb7, 0xb3, 0x79, 0xbb, 0xac, 0x4f, 0xe4, 0xa0, 0x2f, 0x05, 0xae, 0xf3, 0x27, 0xc7, + 0x82, + ]; + const VCEK_GUID: [u8; 16] = [ + 0x63, 0xda, 0x75, 0x8d, 0xe6, 0x64, 0x45, 0x64, 0xad, 0xc5, 0xf4, 0xb9, 0x3b, 0xe8, 0xac, + 0xcd, + ]; + let [ask, vcek] = cert_chain else { + anyhow::bail!("mock SEV-SNP evidence must contain ASK and VCEK"); + }; + let ask = pem::parse(ask)?.into_contents(); + let vcek = pem::parse(vcek)?.into_contents(); + let header_len = 3 * 24; + let mut output = Vec::with_capacity(header_len + ask.len() + vcek.len()); + for (guid, offset, length) in [ + (ASK_GUID, header_len, ask.len()), + (VCEK_GUID, header_len + ask.len(), vcek.len()), + ] { + output.extend_from_slice(&guid); + output.extend_from_slice(&(offset as u32).to_le_bytes()); + output.extend_from_slice(&(length as u32).to_le_bytes()); + } + output.extend_from_slice(&[0u8; 24]); + output.extend_from_slice(&ask); + output.extend_from_slice(&vcek); + Ok(output) +} + +pub(crate) struct SevSnpBackend; +impl TeeBackend for SevSnpBackend { + type Fs = SevSnpFs; + const PLATFORM: &'static str = "amd-sev-snp"; + const DEFAULT_MOUNTPOINT: &'static str = "/sys/kernel/config/tsm/report"; + fn create_filesystem(config: &TeeSimulatorConfig) -> Result { + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let mr_config = config + .mr_config + .as_deref() + .context("mr_config is required for SEV-SNP simulation")?; + let host_data = dstack_types::mr_config::MrConfigV3::snp_host_data_from_document(mr_config); + let measurement = match config.vm_config.as_deref() { + Some(vm_config) + if serde_json::from_str::(vm_config)? + .get("sev_snp_measurement") + .is_some() => + { + let inputs = dstack_mr::sev::parse_snp_inputs_from_vm_config(vm_config)?; + dstack_mr::sev::compute_expected_measurement(&inputs.input)? + } + _ => [0x33; 48], + }; + SevSnpFs::new( + Arc::new(SevSnpGenerator::from_seed(mock_attestation::parse_seed( + seed, + )?)?), + host_data, + measurement, + ) + } + fn prepare_mountpoint(mountpoint: &Path) -> Result<()> { + crate::tdx::ensure_configfs_mount(mountpoint) + } + fn real_tee_available() -> bool { + Path::new("/dev/sev-guest").exists() + } +} diff --git a/dstack/tee-simulator/src/tdx.rs b/dstack/tee-simulator/src/tdx.rs index 60e0603dd..a3983c6ee 100644 --- a/dstack/tee-simulator/src/tdx.rs +++ b/dstack/tee-simulator/src/tdx.rs @@ -4,13 +4,16 @@ use std::ffi::{CString, OsStr}; use std::path::Path; +use std::sync::Arc; use std::time::{Duration, SystemTime}; use anyhow::{bail, Context, Result}; +use dstack_types::{TeeSimulatorConfig, VmConfig}; use fuser::{ FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, ReplyOpen, ReplyWrite, Request, TimeOrNow, }; +use mock_attestation::tdx::TdxGenerator; use sha2::{Digest, Sha384}; use crate::TeeBackend; @@ -31,17 +34,15 @@ const MEASUREMENTS_DIR_INO: u64 = 7; const RTMR0_INO: u64 = 8; const CCEL_INO: u64 = 12; -const TDX_QUOTE_MIN_SIZE: usize = 632; const MR_CONFIG_ID_RANGE: std::ops::Range = 232..280; const RTMR0_OFFSET: usize = 376; const REPORT_DATA_RANGE: std::ops::Range = 568..632; -const QUOTE_FIXTURE: &[u8] = include_bytes!("../../ra-tls/assets/tdx_quote"); const CCEL_FIXTURE: &[u8] = include_bytes!("../../cc-eventlog/samples/ccel.bin"); -#[derive(Clone)] struct SimulatorState { - base_quote: Vec, + generator: Arc, + mrtd: [u8; 48], ccel: Vec, rtmrs: [[u8; 48]; 4], outblob: Vec, @@ -49,38 +50,39 @@ struct SimulatorState { } impl SimulatorState { - fn new(base_quote: &[u8], ccel: &[u8]) -> Result { - if base_quote.len() < TDX_QUOTE_MIN_SIZE { - bail!("tdx quote fixture is too short: {}", base_quote.len()); - } - + fn new(generator: Arc, ccel: &[u8], vm_config: Option<&str>) -> Result { + let (mrtd, rtmrs) = measurements_for_config(ccel, vm_config)?; let mut state = Self { - base_quote: base_quote.to_vec(), + generator, + mrtd, ccel: ccel.to_vec(), - rtmrs: replay_boot_rtmrs(ccel)?, + rtmrs, outblob: Vec::new(), generation: 0, }; - state.outblob = state.make_quote([0u8; 64]); + state.outblob = state.make_quote([0u8; 64])?; Ok(state) } - fn make_quote(&self, report_data: [u8; 64]) -> Vec { - let mut quote = self.base_quote.clone(); + fn make_quote(&self, report_data: [u8; 64]) -> Result> { + let mut quote = self + .generator + .attest_with_measurements(report_data, self.mrtd, self.rtmrs)? + .quote; quote[MR_CONFIG_ID_RANGE].fill(0); for (index, rtmr) in self.rtmrs.iter().enumerate() { let start = RTMR0_OFFSET + index * 48; quote[start..start + 48].copy_from_slice(rtmr); } quote[REPORT_DATA_RANGE].copy_from_slice(&report_data); - quote + Ok(quote) } fn request_quote(&mut self, report_data: &[u8]) -> Result<()> { let report_data: [u8; 64] = report_data .try_into() .map_err(|_| anyhow::anyhow!("inblob must be exactly 64 bytes"))?; - self.outblob = self.make_quote(report_data); + self.outblob = self.make_quote(report_data)?; self.generation = self .generation .checked_add(1) @@ -115,6 +117,63 @@ impl SimulatorState { } } +fn measurements_for_config( + ccel: &[u8], + vm_config: Option<&str>, +) -> Result<([u8; 48], [[u8; 48]; 4])> { + let mut rtmrs = replay_boot_rtmrs(ccel)?; + let Some(vm_config) = vm_config else { + return Ok(([0x11; 48], rtmrs)); + }; + let value: serde_json::Value = serde_json::from_str(vm_config).context("invalid vm_config")?; + if value.get("tdx_measurement").is_none() { + return Ok(([0x11; 48], rtmrs)); + } + let vm_config: VmConfig = serde_json::from_value(value).context("invalid TDX vm_config")?; + let Some(document) = vm_config.tdx_measurement.as_ref() else { + return Ok(([0x11; 48], rtmrs)); + }; + let events = cc_eventlog::tdx::decode_ccel(ccel).context("failed to decode CCEL fixture")?; + let digest = |name: &str| -> Result> { + let event = events + .iter() + .find(|event| event.imr == 0 && event.event == name) + .with_context(|| format!("CCEL fixture is missing {name}"))?; + Ok(event.digest()) + }; + let acpi_hashes = dstack_mr::tdx::TdxRtmr0AcpiHashes { + loader: digest("acpi-loader")?, + rsdp: digest("acpi-rsdp")?, + tables: digest("acpi-tables")?, + }; + let measurements = dstack_mr::tdx::tdx_measurements_from_measurement_document( + document, + &vm_config, + &acpi_hashes, + )?; + let mrtd = measurements + .mrtd + .as_slice() + .try_into() + .context("invalid MRTD")?; + rtmrs[0] = measurements + .rtmr0 + .as_slice() + .try_into() + .context("invalid RTMR0")?; + rtmrs[1] = measurements + .rtmr1 + .as_slice() + .try_into() + .context("invalid RTMR1")?; + rtmrs[2] = measurements + .rtmr2 + .as_slice() + .try_into() + .context("invalid RTMR2")?; + Ok((mrtd, rtmrs)) +} + fn replay_boot_rtmrs(ccel: &[u8]) -> Result<[[u8; 48]; 4]> { let events = cc_eventlog::tdx::decode_ccel(ccel).context("failed to decode CCEL fixture")?; let mut rtmrs = [[0u8; 48]; 4]; @@ -145,9 +204,9 @@ pub(crate) struct TdxSimulatorFs { } impl TdxSimulatorFs { - fn new() -> Result { + fn new(generator: Arc) -> Result { Ok(Self { - state: SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE)?, + state: SimulatorState::new(generator, CCEL_FIXTURE, None)?, uid: unsafe { libc::geteuid() }, gid: unsafe { libc::getegid() }, }) @@ -381,7 +440,7 @@ impl Filesystem for TdxSimulatorFs { } } -fn ensure_configfs_mount(mountpoint: &Path) -> Result<()> { +pub(crate) fn ensure_configfs_mount(mountpoint: &Path) -> Result<()> { if mountpoint != Path::new(TDX_DEFAULT_MOUNTPOINT) { std::fs::create_dir_all(mountpoint) .with_context(|| format!("failed to create {}", mountpoint.display()))?; @@ -432,8 +491,20 @@ impl TeeBackend for TdxBackend { const PLATFORM: &'static str = "tdx"; const DEFAULT_MOUNTPOINT: &'static str = TDX_DEFAULT_MOUNTPOINT; - fn create_filesystem() -> Result { - TdxSimulatorFs::new() + fn create_filesystem(config: &TeeSimulatorConfig) -> Result { + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let generator = Arc::new(TdxGenerator::from_seed(mock_attestation::parse_seed( + seed, + )?)?); + let mut fs = TdxSimulatorFs::new(generator)?; + let (mrtd, rtmrs) = measurements_for_config(CCEL_FIXTURE, config.vm_config.as_deref())?; + fs.state.mrtd = mrtd; + fs.state.rtmrs = rtmrs; + fs.state.outblob = fs.state.make_quote([0; 64])?; + Ok(fs) } fn prepare_mountpoint(mountpoint: &Path) -> Result<()> { @@ -452,7 +523,13 @@ mod tests { #[test] fn quote_tracks_report_data_and_rtmr_extensions() { - let mut state = SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE).unwrap(); + let seed = [0x6b; 32]; + let mut state = SimulatorState::new( + Arc::new(TdxGenerator::from_seed(seed).unwrap()), + CCEL_FIXTURE, + None, + ) + .unwrap(); let original_rtmr3 = state.rtmrs[3]; let digest = [0x42; 48]; state.extend_rtmr(3, &digest).unwrap(); @@ -464,17 +541,30 @@ mod tests { let quote = Quote::parse(&state.outblob).unwrap(); let report = quote.report.as_td10().unwrap(); - let fixture = Quote::parse(QUOTE_FIXTURE).unwrap(); - let fixture_report = fixture.report.as_td10().unwrap(); assert_eq!(report.report_data, report_data); assert_eq!(report.mr_config_id, [0u8; 48]); assert_eq!(report.rt_mr3, state.rtmrs[3]); - assert_eq!(report.mr_owner, fixture_report.mr_owner); + assert_eq!(report.mr_owner, [0u8; 48]); + + let host = TdxGenerator::from_seed(seed).unwrap(); + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(host.root_ca_der()) + .verify(&state.outblob, &host.sample_collateral().unwrap(), now) + .unwrap(); + let wrong = TdxGenerator::from_seed([0x6c; 32]).unwrap(); + assert!(dcap_qvl::verify::QuoteVerifier::new(wrong.root_ca_der()) + .verify(&state.outblob, &host.sample_collateral().unwrap(), now) + .is_err()); } #[test] fn only_rtmr_two_and_three_are_extensible() { - let mut state = SimulatorState::new(QUOTE_FIXTURE, CCEL_FIXTURE).unwrap(); + let mut state = + SimulatorState::new(Arc::new(TdxGenerator::new().unwrap()), CCEL_FIXTURE, None) + .unwrap(); assert!(state.extend_rtmr(0, &[0u8; 48]).is_err()); assert!(state.extend_rtmr(2, &[0u8; 48]).is_ok()); assert!(state.extend_rtmr(3, &[0u8; 48]).is_ok()); diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs new file mode 100644 index 000000000..70b2ea00b --- /dev/null +++ b/dstack/tee-simulator/src/tpm.rs @@ -0,0 +1,518 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! swtpm-backed GCP vTPM environment, including the pre-provisioned AK +//! template and certificate NV indices consumed by `tpm-attest`. + +use std::{ + io::{Read, Write}, + os::{ + fd::{AsRawFd, FromRawFd}, + unix::net::UnixStream, + }, + path::Path, + process::{Command, Stdio}, + thread, + time::Duration, +}; + +use anyhow::{bail, Context, Result}; +use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; +use dstack_types::TeeSimulatorConfig; +use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; + +const AK_ECC_CERT: &str = "0x01c10002"; +const AK_ECC_TEMPLATE: &str = "0x01c10003"; +const TPM2_CC_NV_WRITE: u32 = 0x0000_0137; +const TPM2_CC_NV_DEFINE_SPACE: u32 = 0x0000_012a; +const TPM2_CC_NV_READ: u32 = 0x0000_014e; +const TPM2_CC_NV_READ_PUBLIC: u32 = 0x0000_0169; +const TPM2_CC_AWS_NSM_REQUEST: u32 = 0x2000_0001; +const VTPM_PROXY_IOC_NEW_DEV: libc::c_ulong = 0xc014_a100; +const VTPM_PROXY_FLAG_TPM2: u32 = 1; + +#[repr(C)] +#[derive(Default)] +struct VtpmProxyNewDev { + flags: u32, + tpm_num: u32, + fd: u32, + major: u32, + minor: u32, +} + +struct NvWriteTemplate { + request: Vec, +} + +fn read_be_u16(bytes: &[u8], field: &str) -> Result { + let bytes: [u8; 2] = bytes + .try_into() + .map_err(|_| anyhow::anyhow!("invalid {field} length"))?; + Ok(u16::from_be_bytes(bytes)) +} + +fn read_be_u32(bytes: &[u8], field: &str) -> Result { + let bytes: [u8; 4] = bytes + .try_into() + .map_err(|_| anyhow::anyhow!("invalid {field} length"))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn command(program: &str, args: &[&str]) -> Result<()> { + let status = Command::new(program) + .args(args) + .env("TPM2TOOLS_TCTI", "device:/dev/tpm0") + .status() + .with_context(|| format!("failed to execute {program}"))?; + if !status.success() { + bail!("{program} failed with status {status}"); + } + Ok(()) +} + +pub fn start_gcp_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result<()> { + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + bail!("refusing to start the GCP vTPM simulator when a real TPM is present"); + } + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let base_url = config + .collateral_base_url + .as_deref() + .unwrap_or("http://127.0.0.1:8088"); + let state = MockCollateralState::from_seed(parse_seed(seed)?, base_url)?; + // Keep swtpm state outside /run: swtpm drops privileges to `tss`, and some + // distributions reject its lock file when a parent runtime directory is + // owned by root even if the immediate state directory is writable. + let state_dir = std::env::temp_dir().join(format!("dstack-swtpm-{}", std::process::id())); + fs_err::create_dir_all(&state_dir)?; + let state_arg = format!("dir={}", state_dir.display()); + let pid_file = std::env::temp_dir().join(format!("dstack-swtpm-{}.pid", std::process::id())); + let pid_arg = format!("file={}", pid_file.display()); + command( + "swtpm", + &[ + "chardev", + "--tpm2", + "--tpmstate", + &state_arg, + "--vtpm-proxy", + "--pid", + &pid_arg, + "--daemon", + ], + )?; + fs_err::copy(&pid_file, runtime_dir.join("swtpm.pid"))?; + + for _ in 0..100 { + create_tpm_device_node()?; + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + break; + } + thread::sleep(Duration::from_millis(20)); + } + command("tpm2_startup", &["-c"])?; + replay_fixture_event_log()?; + + let template_with_size = state_dir.join("ak.tpm2b-public"); + let generated_public = state_dir.join("ak.public"); + let template = state_dir.join("ak.template"); + let public_pem = state_dir.join("ak.pem"); + let context = state_dir.join("ak.ctx"); + command( + "tpm2_createprimary", + &[ + "-C", + "e", + "-G", + "ecc256:ecdsa-sha256:null", + "-g", + "sha256", + "-a", + "fixedtpm|fixedparent|sensitivedataorigin|userwithauth|restricted|sign|noda", + "-c", + context.to_str().context("invalid AK context path")?, + "-o", + generated_public + .to_str() + .context("invalid generated AK public path")?, + "--template-data", + template_with_size + .to_str() + .context("invalid AK public path")?, + ], + )?; + command( + "tpm2_readpublic", + &[ + "-c", + context.to_str().context("invalid AK context path")?, + "-f", + "pem", + "-o", + public_pem.to_str().context("invalid AK PEM path")?, + ], + )?; + let bytes = fs_err::read(&template_with_size)?; + anyhow::ensure!(!bytes.is_empty(), "invalid TPMT_PUBLIC generated by swtpm"); + fs_err::write(&template, bytes)?; + + let root_cert = state_dir.join("root.pem"); + let root_key = state_dir.join("root-key.pem"); + let ak_cert = state_dir.join("ak-cert.pem"); + let extensions = state_dir.join("ak-cert.ext"); + fs_err::write(&root_cert, state.tpm.root_ca_pem())?; + fs_err::write(&root_key, state.tpm.root_key_pem())?; + fs_err::write( + &extensions, + format!( + "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nauthorityInfoAccess=caIssuers;URI:{base_url}/tpm/aia/root.pem\ncrlDistributionPoints=URI:{base_url}/tpm/crl/root.crl\n" + ), + )?; + command( + "openssl", + &[ + "x509", + "-new", + "-force_pubkey", + public_pem.to_str().context("invalid AK PEM path")?, + "-CAkey", + root_key.to_str().context("invalid root key path")?, + "-CA", + root_cert.to_str().context("invalid root cert path")?, + "-CAcreateserial", + "-days", + "30", + "-subj", + "/CN=Mock GCP vTPM AK", + "-extfile", + extensions.to_str().context("invalid extension path")?, + "-outform", + "DER", + "-out", + ak_cert.to_str().context("invalid AK certificate path")?, + ], + )?; + provision_nv(AK_ECC_TEMPLATE, &template)?; + provision_nv(AK_ECC_CERT, &ak_cert)?; + command("tpm2_flushcontext", &["-t"])?; + Ok(()) +} + +fn replay_fixture_event_log() -> Result<()> { + let bytes = include_bytes!("../../cc-eventlog/samples/tpm_eventlog.bin"); + let event_log = cc_eventlog::tpm::TpmEventLog::decode(&mut bytes.as_slice())?; + for event in event_log.events { + let extension = format!("{}:sha256={}", event.pcr_index, hex::encode(event.digest)); + command("tpm2_pcrextend", &[&extension])?; + } + Ok(()) +} + +fn create_tpm_device_node() -> Result<()> { + if Path::new("/dev/tpm0").exists() { + return Ok(()); + } + let sys_dev = Path::new("/sys/class/tpm/tpm0/dev"); + if !sys_dev.exists() { + return Ok(()); + } + let device = fs_err::read_to_string(sys_dev)?; + let (major, minor) = device + .trim() + .split_once(':') + .context("invalid /sys/class/tpm/tpm0/dev")?; + command("mknod", &["/dev/tpm0", "c", major, minor]) +} + +fn provision_nv(index: &str, contents: &Path) -> Result<()> { + let size = fs_err::metadata(contents)?.len().to_string(); + command( + "tpm2_nvdefine", + &[index, "-C", "o", "-s", &size, "-a", "ownerread|ownerwrite"], + )?; + command( + "tpm2_nvwrite", + &[ + index, + "-C", + "o", + "-i", + contents.to_str().context("invalid NV input path")?, + ], + ) +} + +/// Run a TPM2 device backed by swtpm while implementing the NitroTPM NSM +/// vendor command at the TPM wire-protocol boundary. All ordinary commands, +/// including EK creation, HMAC sessions, and NV access, are handled by swtpm. +pub fn run_nitro_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result<()> { + if Path::new("/dev/tpmrm0").exists() || Path::new("/dev/tpm0").exists() { + bail!("refusing to start the NitroTPM simulator when a real TPM is present"); + } + let seed = config + .mock_attestation_seed + .as_deref() + .context("tee_simulator.mock_attestation_seed is required")?; + let generator = NsmGenerator::from_seed(parse_seed(seed)?)?; + let (mut simulator, swtpm_stream) = UnixStream::pair()?; + let swtpm_fd = swtpm_stream.as_raw_fd(); + let flags = unsafe { libc::fcntl(swtpm_fd, libc::F_GETFD) }; + anyhow::ensure!(flags >= 0, "failed to get swtpm socket flags"); + anyhow::ensure!( + unsafe { libc::fcntl(swtpm_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) } >= 0, + "failed to make swtpm socket inheritable" + ); + + let state_dir = std::env::temp_dir().join(format!("dstack-nitro-swtpm-{}", std::process::id())); + fs_err::create_dir_all(&state_dir)?; + let state_arg = format!("dir={}", state_dir.display()); + let fd_arg = swtpm_fd.to_string(); + let mut child = Command::new("swtpm") + .args([ + "chardev", + "--tpm2", + "--tpmstate", + &state_arg, + "--fd", + &fd_arg, + "--flags", + "not-need-init,startup-clear", + "--locality", + "allow-set-locality", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .context("failed to start NitroTPM swtpm backend")?; + drop(swtpm_stream); + fs_err::write(runtime_dir.join("swtpm.pid"), child.id().to_string())?; + + let (control, mut proxy, tpm_num) = create_vtpm_proxy()?; + let proxy_thread = thread::spawn(move || { + let _control = control; + let result = proxy_tpm_commands(&mut proxy, &mut simulator, &generator); + if let Err(error) = &result { + tracing::error!(?error, "NitroTPM proxy failed"); + } + result + }); + let sys_dev = format!("/sys/class/tpm/tpm{tpm_num}/dev"); + for _ in 0..100 { + if Path::new(&sys_dev).exists() { + break; + } + thread::sleep(Duration::from_millis(10)); + } + let device = fs_err::read_to_string(&sys_dev).context("vTPM was not registered")?; + let (major, minor) = device + .trim() + .split_once(':') + .context("invalid vTPM device number")?; + command("mknod", &["/dev/tpm0", "c", major, minor])?; + sd_notify::notify(true, &[sd_notify::NotifyState::Ready])?; + let result = proxy_thread + .join() + .map_err(|_| anyhow::anyhow!("NitroTPM proxy thread panicked"))?; + let _ = child.kill(); + result +} + +fn create_vtpm_proxy() -> Result<(std::fs::File, std::fs::File, u32)> { + let control = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/vtpmx") + .context("failed to open /dev/vtpmx; load tpm_vtpm_proxy")?; + let mut new_dev = VtpmProxyNewDev { + flags: VTPM_PROXY_FLAG_TPM2, + ..Default::default() + }; + let rc = unsafe { libc::ioctl(control.as_raw_fd(), VTPM_PROXY_IOC_NEW_DEV, &mut new_dev) }; + anyhow::ensure!( + rc == 0, + "failed to create vTPM proxy device: {}", + std::io::Error::last_os_error() + ); + // SAFETY: the successful ioctl returned ownership of this new descriptor. + Ok(( + control, + unsafe { std::fs::File::from_raw_fd(new_dev.fd as i32) }, + new_dev.tpm_num, + )) +} + +fn proxy_tpm_commands( + proxy: &mut std::fs::File, + backend: &mut UnixStream, + generator: &NsmGenerator, +) -> Result<()> { + let mut nv_write = None; + let mut nsm_response = None; + loop { + let mut command = vec![0u8; 65_536]; + let size = loop { + match proxy.read(&mut command) { + Ok(size) => break size, + Err(error) if error.raw_os_error() == Some(libc::EPIPE) => { + thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(error).context("failed to read vTPM command"), + } + }; + if size == 0 { + return Ok(()); + } + command.truncate(size); + anyhow::ensure!(command.len() >= 10, "truncated TPM command"); + let code = read_be_u32(&command[6..10], "TPM command code")?; + let response = if code == TPM2_CC_AWS_NSM_REQUEST { + let template = nv_write + .as_ref() + .context("NitroTPM vendor command without an NV request")?; + nsm_response = Some(handle_nsm_vendor_command(generator, template)?); + tpm_success_response() + } else if code == TPM2_CC_NV_READ && nsm_response.is_some() { + nv_read_response( + &command, + nsm_response.as_deref().context("missing NSM response")?, + )? + } else if code == TPM2_CC_NV_READ_PUBLIC && nsm_response.is_some() { + let mut response = transact(backend, &command)?; + set_nv_public_size( + &mut response, + nsm_response + .as_deref() + .context("missing NSM response")? + .len(), + )?; + response + } else { + // NitroTPM permits an 8 KiB message NV space while libtpms caps a + // single NV index at 2 KiB. Define the backing index at that limit; + // the proxy virtualizes its public size and reads after the vendor + // command, while swtpm still handles its lifecycle and auth setup. + if code == TPM2_CC_NV_DEFINE_SPACE && command.ends_with(&8192u16.to_be_bytes()) { + let end = command.len(); + command[end - 2..].copy_from_slice(&2048u16.to_be_bytes()); + } + if code == TPM2_CC_NV_WRITE { + if let Some(template) = parse_nv_write(&command)? { + nv_write = Some(template); + } + } + transact(backend, &command)? + }; + proxy + .write_all(&response) + .context("failed to write vTPM response")?; + } +} + +fn parse_nv_write(command: &[u8]) -> Result> { + // sessions header + auth handle + NV index + authorizationSize + if command.len() < 24 { + return Ok(None); + } + let index = read_be_u32(&command[14..18], "NV index")?; + if !(0x0100_0000..=0x01ff_ffff).contains(&index) { + return Ok(None); + } + let auth_size = read_be_u32(&command[18..22], "NV authorization size")? as usize; + let data_size_pos = 22usize + .checked_add(auth_size) + .context("NV write size overflow")?; + anyhow::ensure!(command.len() >= data_size_pos + 4, "truncated NV write"); + let data_size = read_be_u16( + &command[data_size_pos..data_size_pos + 2], + "NV write data size", + )? as usize; + let data_start = data_size_pos + 2; + let data_end = data_start + data_size; + anyhow::ensure!(command.len() >= data_end + 2, "truncated NV write data"); + let request = command[data_start..data_end].to_vec(); + Ok(Some(NvWriteTemplate { request })) +} + +fn handle_nsm_vendor_command( + generator: &NsmGenerator, + template: &NvWriteTemplate, +) -> Result> { + let request: NsmRequest = serde_cbor::from_slice(&template.request)?; + let response = match request { + NsmRequest::Attestation { user_data, .. } => { + let pcrs = [4u16, 7, 8, 12, 14] + .into_iter() + .map(|i| (i, vec![0; 48])) + .collect(); + let document = generator.attest_with_pcrs( + user_data.as_ref().map(|v| v.as_slice()).unwrap_or_default(), + pcrs, + )?; + NsmResponse::Attestation { document } + } + _ => anyhow::bail!("unsupported NitroTPM NSM request"), + }; + Ok(serde_cbor::to_vec(&response)?) +} + +fn set_nv_public_size(response: &mut [u8], size: usize) -> Result<()> { + anyhow::ensure!(response.len() >= 26, "truncated NV_ReadPublic response"); + let policy_size_pos = 22; + let policy_size = read_be_u16( + &response[policy_size_pos..policy_size_pos + 2], + "NV policy size", + )? as usize; + let data_size_pos = policy_size_pos + 2 + policy_size; + anyhow::ensure!( + response.len() >= data_size_pos + 2, + "truncated NV public area" + ); + response[data_size_pos..data_size_pos + 2].copy_from_slice(&(size as u16).to_be_bytes()); + Ok(()) +} + +fn nv_read_response(command: &[u8], contents: &[u8]) -> Result> { + anyhow::ensure!(command.len() >= 4, "truncated NV_Read command"); + let size = read_be_u16( + &command[command.len() - 4..command.len() - 2], + "NV read size", + )? as usize; + let offset = read_be_u16(&command[command.len() - 2..], "NV read offset")? as usize; + let end = (offset + size).min(contents.len()); + anyhow::ensure!(offset <= end, "invalid NV_Read offset"); + let data = &contents[offset..end]; + let parameter_size = 2 + data.len(); + let total_size = 10 + 4 + parameter_size + 5; + let mut response = Vec::with_capacity(total_size); + response.extend_from_slice(&0x8002u16.to_be_bytes()); + response.extend_from_slice(&(total_size as u32).to_be_bytes()); + response.extend_from_slice(&0u32.to_be_bytes()); + response.extend_from_slice(&(parameter_size as u32).to_be_bytes()); + response.extend_from_slice(&(data.len() as u16).to_be_bytes()); + response.extend_from_slice(data); + response.extend_from_slice(&[0, 0, 0, 0, 0]); + Ok(response) +} + +fn transact(stream: &mut UnixStream, command: &[u8]) -> Result> { + stream.write_all(command)?; + let mut header = [0u8; 10]; + stream.read_exact(&mut header)?; + let size = read_be_u32(&header[2..6], "TPM response size")? as usize; + anyhow::ensure!(size >= header.len(), "invalid TPM response size"); + let mut response = Vec::with_capacity(size); + response.extend_from_slice(&header); + response.resize(size, 0); + stream.read_exact(&mut response[10..])?; + Ok(response) +} + +fn tpm_success_response() -> Vec { + [0x80, 0x01, 0, 0, 0, 10, 0, 0, 0, 0].to_vec() +} diff --git a/dstack/tee-simulator/tests/process_e2e.rs b/dstack/tee-simulator/tests/process_e2e.rs new file mode 100644 index 000000000..0dac8e90d --- /dev/null +++ b/dstack/tee-simulator/tests/process_e2e.rs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: © 2026 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + process::{Child, Command, Stdio}, + time::Duration, +}; + +use mock_attestation::server::MockCollateralState; + +struct ChildGuard(Child); +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +async fn start(platform: &str, seed: [u8; 32]) -> (tempfile::TempDir, ChildGuard) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sys-config.json"); + fs_err::write( + &path, + serde_json::json!({ + "kms_urls": [], "gateway_urls": [], "pccs_url": null, + "docker_registry": null, "host_api_url": null, "vm_config": "{}", + "mr_config": "{\"version\":3,\"app_id\":\"\",\"compose_hash\":\"\",\"key_provider\":\"none\"}", + "tee_simulator": { + "platform": platform, + "mock_attestation_seed": hex::encode(seed), + "collateral_base_url": "http://127.0.0.1:18088" + } + }) + .to_string(), + ) + .unwrap(); + let mountpoint = dir.path().join("tsm"); + let mut args = vec![ + "--sys-config".to_string(), + path.display().to_string(), + "--runtime-dir".to_string(), + dir.path().display().to_string(), + "--dmi-root".to_string(), + dir.path().join("dmi").display().to_string(), + ]; + if matches!(platform, "dstack-tdx" | "dstack-amd-sev-snp") { + args.extend(["--mountpoint".into(), mountpoint.display().to_string()]); + } + let child = ChildGuard( + Command::new(env!("CARGO_BIN_EXE_dstack-tee-simulator")) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(), + ); + for _ in 0..50 { + let ready = match platform { + "dstack-tdx" => mountpoint.join("com.intel.dcap/outblob").exists(), + "dstack-amd-sev-snp" => mountpoint.join("provider").exists(), + _ => false, + }; + if ready { + return (dir, child); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("{platform} simulator process did not become ready") +} + +#[tokio::test] +async fn separate_simulator_process_imports_sys_config_seed_for_tsm_platforms() { + let seed = [0x71; 32]; + let host = MockCollateralState::from_seed(seed, "http://127.0.0.1:18088").unwrap(); + + let (dir, child) = start("dstack-tdx", seed).await; + let rd = [0x21; 64]; + let provider = dir.path().join("tsm/com.intel.dcap"); + fs_err::write(provider.join("inblob"), rd).unwrap(); + let quote = fs_err::read(provider.join("outblob")).unwrap(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + dcap_qvl::verify::QuoteVerifier::new(host.tdx.root_ca_der()) + .verify("e, &host.tdx.sample_collateral().unwrap(), now) + .unwrap(); + drop(child); + + let (dir, child) = start("dstack-amd-sev-snp", seed).await; + let provider = dir.path().join("tsm"); + let entry = provider.join(format!("dstack-{}", std::process::id())); + use std::io::Write as _; + std::fs::OpenOptions::new() + .write(true) + .open(entry.join("inblob")) + .unwrap() + .write_all(&rd) + .unwrap(); + let report = fs_err::read(entry.join("outblob")).unwrap(); + let cert_chain = fs_err::read(entry.join("certs")).unwrap(); + sev_snp_qvl::QuoteVerifier::new_with_root( + sev_snp_qvl::AmdSnpProduct::Milan, + host.sev_snp.root_ca_pem().into_bytes(), + ) + .verify(&report, &[cert_chain], &rd) + .unwrap(); + drop(child); +} diff --git a/dstack/test-scripts/snp-e2e-smoke.sh b/dstack/test-scripts/snp-e2e-smoke.sh deleted file mode 100755 index 832ed2b44..000000000 --- a/dstack/test-scripts/snp-e2e-smoke.sh +++ /dev/null @@ -1,551 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: © 2026 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 -# -# Manual AMD SEV-SNP hardware smoke for dstack-managed KMS/app key release. -# -# This is intentionally not a CI script. It requires an SNP-capable host with the -# AMDSEV QEMU/OVMF build used by the PR smoke, sudo for QEMU/KVM, and locally -# built release binaries. -# -# Minimal setup used by the original smoke: -# cargo build --manifest-path dstack/Cargo.toml --release -p dstack-vmm -p supervisor -p dstack-kms -# export DSTACK_SNP_SMOKE_BIN_DIR=$PWD/dstack/target/release -# export DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB=1 # lab hosts only; auth API policy -# test-scripts/snp-e2e-smoke.sh -# -# Useful overrides: -# DSTACK_SNP_SMOKE_BASE=$HOME/dstack-snp-e2e -# DSTACK_SNP_SMOKE_REPO=$PWD/dstack -# DSTACK_SNP_SMOKE_QEMU=/opt/AMDSEV/usr/local/bin/qemu-system-x86_64 -# DSTACK_SNP_SMOKE_OVMF=/opt/AMDSEV/usr/local/share/qemu/OVMF.fd -# DSTACK_SNP_SMOKE_IMAGE_URL=https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz -# DSTACK_SNP_SMOKE_IMAGE_NAME=dstack-dev-0.5.11-snp-dnsfix -# DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 # bypasses the QEMU >= 10 preflight -# -# Host/image caveat: QEMU >= 10 is necessary but not sufficient. One local SNP -# host could boot a newer Lit SNP guest kernel but reset before Linux serial -# output with the stock meta-dstack v0.5.11 6.9.0-dstack kernel. If this smoke -# stops after `EFI stub: Loaded initrd ...` with `cpus are not resettable`, first -# validate the guest image/kernel on that host before debugging KMS or apps. -# -# Guest userspace caveat: rebuilding the host-side PR binaries is not enough for -# full app-key success if the downloaded meta-dstack image still embeds an older -# dstack-util/dstack-attest. On that skewed image the app guest can reach -# dstack-prepare.sh and fail at GetTempCaCert/GetAppKey with: -# amd sev-snp cert_chain must contain either ASK and VCEK certificates or one -# kernel certificate table auxblob -# For full SNP_APP_CONTAINER_STARTED / GetAppKey success, use a coherent -# meta-dstack guest image that includes the same PR cert-chain/KDS fallback code. -# If AMD KDS throttles VCEK/cert-chain retrieval (for example HTTP 429 from -# kdsintf.amd.com), keep verification fail-closed and set -# DSTACK_SNP_SMOKE_KDS_BASE_URL to a trusted AMD-KDS-compatible mirror/cache -# base, e.g. https://mirror.example.com/vcek/v1. For a path-prefix relay, set -# the full relayed base, e.g.: -# https://cors.litgateway.com/https://kdsintf.amd.com/vcek/v1 -# This is an external collateral-fetch boundary, not a guest boot or KMS startup -# failure. -# One reproducible way is to build os/yocto from the same monorepo revision, -# set the Yocto build MACHINE to `sev-snp` (not the -# default `tdx`, otherwise the guest kernel can miss AMD memory-encryption -# support and reset immediately after OVMF loads the kernel/initrd), then point -# DSTACK_SNP_SMOKE_IMAGE_NAME at the resulting dstack-dev image directory. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CORE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -BASE="${DSTACK_SNP_SMOKE_BASE:-$HOME/dstack-snp-e2e}" -REPO="${DSTACK_SNP_SMOKE_REPO:-$CORE_DIR}" -BIN="${DSTACK_SNP_SMOKE_BIN_DIR:-$REPO/target/release}" -ART="$BASE/artifacts" -LOG="$ART/snp-e2e-smoke.log" -IMAGE_NAME="${DSTACK_SNP_SMOKE_IMAGE_NAME:-dstack-dev-0.5.11-snp-dnsfix}" -IMAGE_URL="${DSTACK_SNP_SMOKE_IMAGE_URL:-https://github.com/Dstack-TEE/meta-dstack/releases/download/v0.5.11/dstack-dev-0.5.11.tar.gz}" -QEMU_PATH="${DSTACK_SNP_SMOKE_QEMU:-/opt/AMDSEV/usr/local/bin/qemu-system-x86_64}" -OVMF_PATH="${DSTACK_SNP_SMOKE_OVMF:-/opt/AMDSEV/usr/local/share/qemu/OVMF.fd}" -HOST_ART_PORT="${DSTACK_SNP_SMOKE_HOST_ART_PORT:-18080}" -AUTH_PORT="${DSTACK_SNP_SMOKE_AUTH_PORT:-18081}" -KMS_HOST_PORT="${DSTACK_SNP_SMOKE_KMS_HOST_PORT:-15443}" -STRICT_KMS_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_KMS_HOST_PORT:-15444}" -APP_HOST_PORT="${DSTACK_SNP_SMOKE_APP_HOST_PORT:-15543}" -STRICT_APP_HOST_PORT="${DSTACK_SNP_SMOKE_STRICT_APP_HOST_PORT:-15544}" -VMM_PORT="${DSTACK_SNP_SMOKE_VMM_PORT:-18082}" -VMM_URL="${DSTACK_SNP_SMOKE_VMM_URL:-http://127.0.0.1:$VMM_PORT}" -HOST_API_PORT="${DSTACK_SNP_SMOKE_HOST_API_PORT:-11100}" -ALLOW_OUT_OF_DATE_TCB="${DSTACK_SNP_SMOKE_ALLOW_OUT_OF_DATE_TCB:-0}" -RUN_STRICT_TCB_PROBE="${DSTACK_SNP_SMOKE_STRICT_TCB_PROBE:-1}" -ALLOW_OLD_QEMU="${DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU:-0}" - -need() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "missing required command: $1" >&2 - exit 1 - fi -} - -need curl -need jq -need openssl -need python3 -need sudo - -test -x "$BIN/dstack-vmm" || { echo "missing $BIN/dstack-vmm; run cargo build --release -p dstack-vmm" >&2; exit 1; } -test -x "$BIN/supervisor" || { echo "missing $BIN/supervisor; run cargo build --release -p supervisor" >&2; exit 1; } -test -x "$BIN/dstack-kms" || { echo "missing $BIN/dstack-kms; run cargo build --release -p dstack-kms" >&2; exit 1; } -test -x "$QEMU_PATH" || { echo "missing SNP QEMU: $QEMU_PATH" >&2; exit 1; } -test -r "$OVMF_PATH" || { echo "missing SNP OVMF: $OVMF_PATH" >&2; exit 1; } -test -f "$REPO/vmm/src/vmm-cli.py" || { echo "missing vmm-cli.py; set DSTACK_SNP_SMOKE_REPO" >&2; exit 1; } - -qemu_version_output=$("$QEMU_PATH" --version | head -1) -qemu_version=$(printf '%s\n' "$qemu_version_output" | sed -n 's/.*version \([0-9][0-9]*\)\.\([0-9][0-9]*\).*/\1.\2/p') -qemu_major=${qemu_version%%.*} -if [[ -z "$qemu_version" ]]; then - echo "Warning: could not parse QEMU version from: $qemu_version_output" >&2 -elif (( qemu_major < 10 )) && [[ "$ALLOW_OLD_QEMU" != "1" ]]; then - cat >&2 <= 10 build, or set DSTACK_SNP_SMOKE_ALLOW_OLD_QEMU=1 -if you intentionally want to reproduce/debug the older-QEMU failure. -EOF - exit 1 -fi - -mkdir -p "$ART" "$BASE/images" "$BASE/run" "$BASE/http-root" -exec > >(tee "$LOG") 2>&1 - -echo "== SNP E2E smoke start: $(date -Is) ==" -echo "repo=$REPO" -echo "repo_head=$(git -C "$REPO" rev-parse --short=16 HEAD 2>/dev/null || echo unknown)" -echo "qemu=$QEMU_PATH" -echo "qemu_version=$qemu_version_output" -echo "ovmf_sha256=$(sha256sum "$OVMF_PATH" | awk '{print $1}')" -echo "image=$IMAGE_NAME" -if [[ -n "${DSTACK_SNP_SMOKE_KDS_BASE_URL:-}" ]]; then - echo "amd_kds_base_url=${DSTACK_SNP_SMOKE_KDS_BASE_URL}" -fi - -cleanup() { - set +e - if [[ -f "$BASE/vmm.pid" ]]; then sudo kill "$(cat "$BASE/vmm.pid")" 2>/dev/null || true; fi - if [[ -f "$BASE/artifacts-http.pid" ]]; then kill "$(cat "$BASE/artifacts-http.pid")" 2>/dev/null || true; fi - if [[ -f "$BASE/auth.pid" ]]; then kill "$(cat "$BASE/auth.pid")" 2>/dev/null || true; fi - sudo pkill -f "$BIN/dstack-vmm" 2>/dev/null || true - sudo pkill -f "qemu-system-x86_64.*$BASE" 2>/dev/null || true - sudo pkill -f "$BASE/images" 2>/dev/null || true - if command -v fuser >/dev/null 2>&1; then - fuser -k "${HOST_ART_PORT}/tcp" "${AUTH_PORT}/tcp" "${KMS_HOST_PORT}/tcp" "${STRICT_KMS_HOST_PORT}/tcp" "${APP_HOST_PORT}/tcp" "${STRICT_APP_HOST_PORT}/tcp" "${VMM_PORT}/tcp" 2>/dev/null || true - fi -} -trap cleanup EXIT -cleanup -sudo pkill -f "$BIN/supervisor" 2>/dev/null || true -sudo rm -rf "$BASE/run"/* "$BASE/tmp"/* - -cp "$BIN/dstack-kms" "$BASE/http-root/dstack-kms" -chmod +x "$BASE/http-root/dstack-kms" - -if [[ ! -d "$BASE/images/$IMAGE_NAME" ]]; then - echo "== Downloading/extracting $IMAGE_NAME ==" - curl -L "$IMAGE_URL" -o "$BASE/$IMAGE_NAME.tar.gz" - mkdir -p "$BASE/images/$IMAGE_NAME" - tar -xzf "$BASE/$IMAGE_NAME.tar.gz" -C "$BASE/images/$IMAGE_NAME" --strip-components=1 -fi -cp "$OVMF_PATH" "$BASE/images/$IMAGE_NAME/ovmf.fd" -tmp_metadata="$(mktemp)" -jq '.bios = "ovmf.fd"' "$BASE/images/$IMAGE_NAME/metadata.json" >"$tmp_metadata" -mv "$tmp_metadata" "$BASE/images/$IMAGE_NAME/metadata.json" -jq . "$BASE/images/$IMAGE_NAME/metadata.json" | tee "$ART/image-metadata.json" - -cat >"$BASE/auth-server.py" <<'PY' -from http.server import BaseHTTPRequestHandler, HTTPServer -import json -import os -import time - - -ALLOW_OUT_OF_DATE_TCB = os.environ.get("ALLOW_OUT_OF_DATE_TCB") == "1" - - -class H(BaseHTTPRequestHandler): - def _send(self, obj, status=200): - body = json.dumps(obj).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, fmt, *args): - print(time.strftime("%Y-%m-%dT%H:%M:%S"), self.path, fmt % args, flush=True) - - def do_GET(self): - self._send({ - "status": "ok", - "kmsContractAddr": "0x0000000000000000000000000000000000000000", - "ethRpcUrl": "", - "gatewayAppId": "", - "chainId": 1, - "appImplementation": "0x0000000000000000000000000000000000000000", - }) - - def do_POST(self): - length = int(self.headers.get("Content-Length", "0") or 0) - body = self.rfile.read(length) - try: - data = json.loads(body or b"{}") - except Exception: - data = {} - summary = {k: data.get(k) for k in ["attestationMode", "tcbStatus", "advisoryIds"] if k in data} - for key in ["appId", "mrAggregated", "osImageHash", "composeHash", "instanceId"]: - if key in data: - summary[key] = str(data[key])[:96] - print(json.dumps({"path": self.path, "summary": summary}), flush=True) - - # TCB/advisory policy belongs to the auth API. The normal smoke can - # explicitly allow an OutOfDate lab host; the /strict auth namespace is - # used by the negative probe to prove denial comes from auth policy, not - # KMS-local config. - if self.path.endswith("/bootAuth/app"): - tcb_status = data.get("tcbStatus") or "" - advisory_ids = data.get("advisoryIds") or [] - strict_tcb = self.path.startswith("/strict/") - if tcb_status not in ("", "UpToDate"): - if strict_tcb or tcb_status != "OutOfDate" or not ALLOW_OUT_OF_DATE_TCB: - self._send({ - "isAllowed": False, - "gatewayAppId": "", - "reason": f"tcb_status is not allowed by auth api: {tcb_status}", - }) - return - if advisory_ids: - self._send({ - "isAllowed": False, - "gatewayAppId": "", - "reason": f"advisory_id is not allowed by auth api: {advisory_ids[0]}", - }) - return - - self._send({"isAllowed": True, "gatewayAppId": "", "reason": "snp smoke auth"}) - - -HTTPServer(("0.0.0.0", int(os.environ["AUTH_PORT"])), H).serve_forever() -PY - -(cd "$BASE/http-root" && python3 -m http.server "$HOST_ART_PORT" >"$ART/artifacts-http.log" 2>&1 & echo $! >"$BASE/artifacts-http.pid") -AUTH_PORT="$AUTH_PORT" ALLOW_OUT_OF_DATE_TCB="$ALLOW_OUT_OF_DATE_TCB" python3 "$BASE/auth-server.py" >"$ART/auth-server.log" 2>&1 & echo $! >"$BASE/auth.pid" -sleep 1 -curl -fsS "http://127.0.0.1:$HOST_ART_PORT/dstack-kms" -o /dev/null -curl -fsS "http://127.0.0.1:$AUTH_PORT/" | jq . | tee "$ART/auth-info.json" - -cat >"$BASE/vmm.toml" <"$ART/vmm.log" 2>&1 & echo $! >"$BASE/vmm.pid" -for i in $(seq 1 60); do - if python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" lsvm --json >/dev/null 2>&1; then break; fi - sleep 1 - if [[ $i -eq 60 ]]; then echo "VMM did not become ready"; tail -80 "$ART/vmm.log"; exit 1; fi -done -echo "== VMM ready ==" - -write_kms_config() { - local auth_prefix="$1" - local auth_url="http://10.0.2.2:$AUTH_PORT" - if [[ -n "$auth_prefix" ]]; then - auth_url="$auth_url/$auth_prefix" - fi - cat >"$BASE/http-root/kms.toml" </etc/docker/daemon.json <<'JSON' -{"dns":["10.0.2.3","1.1.1.1","8.8.8.8"]} -JSON -rm -f /etc/resolv.conf -printf 'nameserver 10.0.2.3\nnameserver 1.1.1.1\nnameserver 8.8.8.8\noptions timeout:2 attempts:3\n' >/etc/resolv.conf -if command -v systemctl >/dev/null 2>&1 && systemctl is-active docker >/dev/null 2>&1; then - systemctl restart docker -fi -SH -) - -KMS_BASH_SCRIPT=$(cat <<'SH' -set -eux -mkdir -p /dstack/kms-certs /dstack/kms-images -curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/dstack-kms -o /dstack/dstack-kms -curl -fsS http://10.0.2.2:__DSTACK_HOST_ART_PORT__/kms.toml -o /dstack/kms.toml -chmod +x /dstack/dstack-kms -echo SNP_KMS_CONTAINER_STARTED -RUST_LOG=info /dstack/dstack-kms -c /dstack/kms.toml -SH -) -KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT/__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} -KMS_BASH_SCRIPT=${KMS_BASH_SCRIPT//__DSTACK_HOST_ART_PORT__/$HOST_ART_PORT} - -deploy_kms() { - local name="$1" - local auth_prefix="$2" - local host_port="$3" - write_kms_config "$auth_prefix" - cat >"$BASE/kms-compose.yaml" <<'YAML' -services: - kms: - image: debian:bookworm-slim - command: sh -c 'echo unused-container-compose; sleep 300' -YAML - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/kms-compose.yaml" --name "$name" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 - jq --arg init_script "$DNS_INIT_SCRIPT" --arg bash_script "$KMS_BASH_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script | .runner="bash" | .bash_script=$bash_script | del(.docker_compose_file)' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" - mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --port "tcp:127.0.0.1:$host_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 - sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 -} - -wait_for_kms_metrics() { - local vm_id="$1" - local host_port="$2" - local label="$3" - for i in $(seq 1 240); do - if curl -kfsS "https://127.0.0.1:$host_port/metrics" >/dev/null 2>&1; then echo "$label KMS runtime ready after ${i}s"; break; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for $label KMS..."; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 30 || true; fi - if [[ $i -eq 240 ]]; then echo "$label KMS did not become ready"; python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$vm_id" -n 200 || true; exit 1; fi - done -} - -kms_key_provider_id() { - local kms_port="$1" - local label="${2:-$kms_port}" - local meta_file="$ART/kms-$label-meta.json" - local ca_file="$ART/kms-$label-ca.pem" - for i in $(seq 1 60); do - if curl -kfsS "https://127.0.0.1:$kms_port/prpc/KMS.GetMeta?json" -o "$meta_file"; then - jq -r '.ca_cert // .caCert // empty' "$meta_file" >"$ca_file" - if [[ -s "$ca_file" ]] && grep -q "BEGIN CERTIFICATE" "$ca_file"; then - # dstack-util treats KMS provider ID as the root CA - # SubjectPublicKeyInfo DER bytes (x509_parser public_key().raw). - openssl x509 -in "$ca_file" -pubkey -noout \ - | openssl pkey -pubin -outform DER \ - | od -An -tx1 -v \ - | tr -d ' \n' - echo - return 0 - fi - fi - sleep 2 - if [[ $((i % 10)) -eq 0 ]]; then echo "waiting for $label KMS GetMeta ca_cert..." >&2; fi - done - echo "failed to derive KMS key_provider_id from https://127.0.0.1:$kms_port/prpc/KMS.GetMeta" >&2 - exit 1 -} - -deploy_app() { - local name="$1" - local kms_port="$2" - local app_port="$3" - local key_provider_id - key_provider_id=$(kms_key_provider_id "$kms_port" "$name") - echo "$name key_provider_id=$key_provider_id" >&2 - cat >"$BASE/$name-compose.yaml" <<'YAML' -services: - smoke: - image: debian:bookworm-slim - command: sh -c 'echo SNP_APP_CONTAINER_STARTED; sleep 300' -YAML - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" compose --docker-compose "$BASE/$name-compose.yaml" --name "$name" --kms --key-provider-id "$key_provider_id" --public-logs --public-sysinfo --no-instance-id --output "$BASE/$name.app-compose.json" | tee "$ART/$name-compose-create.txt" >&2 - jq --arg init_script "$DNS_INIT_SCRIPT" '.storage_fs="ext4" | .init_script=$init_script' "$BASE/$name.app-compose.json" >"$BASE/$name.app-compose.json.tmp" - mv "$BASE/$name.app-compose.json.tmp" "$BASE/$name.app-compose.json" - python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" deploy --name "$name" --compose "$BASE/$name.app-compose.json" --image "$IMAGE_NAME" --kms-url "https://10.0.2.2:$kms_port" --port "tcp:127.0.0.1:$app_port:8000" --vcpu 2 --memory 4096 --disk 20G | tee "$ART/$name-deploy.txt" >&2 - sed -n 's/Created VM with ID: //p' "$ART/$name-deploy.txt" | tail -1 -} - -if [[ "$RUN_STRICT_TCB_PROBE" = "1" && "$ALLOW_OUT_OF_DATE_TCB" = "1" ]]; then - echo "== Strict TCB probe: expect app GetAppKey denial on lab OutOfDate host ==" - STRICT_KMS_VM_ID=$(deploy_kms snp-smoke-kms-strict strict "$STRICT_KMS_HOST_PORT") - echo "STRICT_KMS_VM_ID=$STRICT_KMS_VM_ID" - wait_for_kms_metrics "$STRICT_KMS_VM_ID" "$STRICT_KMS_HOST_PORT" strict - STRICT_APP_VM_ID=$(deploy_app snp-smoke-app-strict "$STRICT_KMS_HOST_PORT" "$STRICT_APP_HOST_PORT") - echo "STRICT_APP_VM_ID=$STRICT_APP_VM_ID" - for i in $(seq 1 240); do - logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_APP_VM_ID" -n 180 2>/dev/null || true) - kms_logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$STRICT_KMS_VM_ID" -n 220 2>/dev/null || true) - if { echo "$logs"; echo "$kms_logs"; } | grep -q "tcb_status is not allowed"; then - { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-denial-log.txt" - echo "strict_tcb_probe=denied_as_expected" - break - fi - if { echo "$logs"; echo "$kms_logs"; } | grep -q "KDS collateral unavailable\|HTTP status client error"; then - { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-kds-blocked-log.txt" - echo "strict_tcb_probe=blocked_by_kds_collateral" - break - fi - if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/strict-tcb-unexpected-success-log.txt"; echo "strict TCB probe unexpectedly reached app container"; exit 1; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for strict APP denial..."; echo "$logs" | tail -60; echo "$kms_logs" | tail -60; fi - if [[ $i -eq 240 ]]; then echo "strict TCB probe did not reach expected denial"; { echo "$logs"; echo "$kms_logs"; } | tee "$ART/strict-tcb-timeout-log.txt"; exit 1; fi - done -fi - -echo "== KMS success run ==" -KMS_VM_ID=$(deploy_kms snp-smoke-kms "" "$KMS_HOST_PORT") -echo "KMS_VM_ID=$KMS_VM_ID" -wait_for_kms_metrics "$KMS_VM_ID" "$KMS_HOST_PORT" success -curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-before-app.txt" - -APP_VM_ID=$(deploy_app snp-smoke-app "$KMS_HOST_PORT" "$APP_HOST_PORT") -echo "APP_VM_ID=$APP_VM_ID" - -for i in $(seq 1 240); do - logs=$(python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 160 2>/dev/null || true) - if echo "$logs" | grep -Eq "SNP_APP_CONTAINER_STARTED|Container dstack-smoke-1 Started"; then echo "$logs" | tee "$ART/app-ready-log.txt"; echo "APP ready after ${i}s"; break; fi - if echo "$logs" | grep -q "Failed to get app key\|Failed to verify app\|Invalid mr_config\|amd sev-snp key release\|measurement mismatch\|App not allowed\|KMS self authorization failed\|KDS collateral unavailable\|HTTP status client error"; then echo "$logs" | tee "$ART/app-failure-log.txt"; exit 2; fi - sleep 2 - if [[ $((i % 30)) -eq 0 ]]; then echo "waiting for APP..."; echo "$logs" | tail -60; fi - if [[ $i -eq 240 ]]; then echo "APP did not become ready"; echo "$logs" | tee "$ART/app-timeout-log.txt"; exit 1; fi -done - -curl -kfsS "https://127.0.0.1:$KMS_HOST_PORT/metrics" | tee "$ART/kms-metrics-after-app.txt" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$KMS_VM_ID" --json | tee "$ART/kms-info.json" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" info "$APP_VM_ID" --json | tee "$ART/app-info.json" -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$KMS_VM_ID" -n 200 | tee "$ART/kms-final-log.txt" || true -python3 "$REPO/vmm/src/vmm-cli.py" --url "$VMM_URL" logs "$APP_VM_ID" -n 200 | tee "$ART/app-final-log.txt" || true - -echo "== SNP E2E smoke success: $(date -Is) ==" -echo "Artifacts: $ART" diff --git a/dstack/tests/e2e/attestation/Dockerfile b/dstack/tests/e2e/attestation/Dockerfile new file mode 100644 index 000000000..2d6bb0413 --- /dev/null +++ b/dstack/tests/e2e/attestation/Dockerfile @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src +COPY . . +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/src/target \ + cargo build --locked \ + -p dstack-tee-simulator \ + -p mock-attestation \ + -p dstack-util \ + -p dstack-verifier \ + -p dstack-mr \ + && mkdir -p /out \ + && cp target/debug/dstack-tee-simulator target/debug/dstack-mock-attestation \ + target/debug/dstack-util target/debug/dstack-verifier target/debug/dstack-mr /out/ \ + && strip /out/* + +FROM debian:bookworm-slim +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl fuse3 jq kmod openssl swtpm tpm2-tools xxd \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /out/ /usr/local/bin/ +COPY cc-eventlog/samples/tpm_eventlog.bin /usr/local/share/dstack/tpm_eventlog.bin +COPY verifier/fixtures/tdx-lite-getquote.json /usr/local/share/dstack/tdx-lite-getquote.json +COPY verifier/fixtures/sev-snp-attestation.json /usr/local/share/dstack/sev-snp-attestation.json +ENTRYPOINT ["/usr/local/bin/run-platform"] diff --git a/dstack/tests/e2e/attestation/README.md b/dstack/tests/e2e/attestation/README.md new file mode 100644 index 000000000..aeafd81f2 --- /dev/null +++ b/dstack/tests/e2e/attestation/README.md @@ -0,0 +1,147 @@ +# Attestation Docker Compose E2E + +This test runs the complete attestation flow in privileged Docker containers, +without starting a VM: + +```text +dstack-tee-simulator -> dstack-util attest -> dstack-verifier +``` + +The simulator and verifier use development trust anchors derived from the same +test seed. Verification still follows the normal production verification paths; +there is no mock-certificate bypass in the verifier. + +## Covered platforms + +The Compose suite runs these six cases: + +- dstack TDX, legacy attestation selector +- dstack TDX, lite attestation selector +- GCP TDX with a simulated vTPM +- AMD SEV-SNP +- AWS Nitro Enclave +- AWS NitroTPM + +Every case must produce all of the following results: + +```text +is_valid=true +quote_verified=true +event_log_verified=true +os_image_hash_verified=true +``` + +## Requirements + +- Linux host +- Docker with the Compose plugin +- Permission to run privileged containers +- Host kernel modules available through `/lib/modules` +- A kernel providing: + - FUSE and CUSE support + - `tpm_vtpm_proxy` (`CONFIG_TCG_VTPM_PROXY`) for the GCP vTPM case + +The script loads `tpm_vtpm_proxy` on demand. The Docker daemon therefore needs +access to a `/lib/modules` tree matching the running host kernel. + +## Run the complete suite + +From the `dstack/` directory: + +```bash +./tests/e2e/attestation/run.sh +``` + +The wrapper builds the local image, runs all six services sequentially, and +removes Compose resources afterward. + +## Run one case + +From this directory: + +```bash +cd tests/e2e/attestation +docker compose build +docker compose run --rm dstack-tdx-legacy +docker compose run --rm dstack-tdx-lite +docker compose run --rm gcp-tdx +docker compose run --rm amd-sev-snp +docker compose run --rm aws-nitro-enclave +docker compose run --rm aws-nitro-tpm +``` + +To clean up manually: + +```bash +docker compose down --remove-orphans +``` + +## What the test configures + +`run-platform.sh` creates a temporary sys-config inside each container. It +selects the simulator platform and supplies the corresponding image measurement +material: + +| Case | Image binding material | +| --- | --- | +| TDX legacy/lite | `tdx_measurement`, VM parameters, and the CCEL ACPI event digests | +| GCP TDX | `gcp_measurement` bound to the UKI Authenticode event and vTPM PCRs | +| AMD SEV-SNP | `sev_snp_measurement` and `mr_config`, bound to report `MEASUREMENT` and `HOST_DATA` | +| Nitro Enclave | NSM-signed PCR0/1/2 | +| NitroTPM | `aws_measurement` bound to NSM-signed PCR4/7/12 | + +The TDX simulator exposes the repository CCEL fixture through the normal TSM +configfs ABI. The GCP case exposes a TPM event log through +`/sys/kernel/security/tpm0/binary_bios_measurements` and replays it into +`swtpm`, so the quoted PCRs and event log remain consistent. + +The NitroTPM case exposes a kernel vTPM device backed by `swtpm`. Its proxy +implements AWS's NSM vendor command on the TPM wire protocol; the attestation +client therefore follows the same EK, salted session, authenticated NV buffer, +and vendor-command path used on real NitroTPM hardware. It does not expose or +fall back to `/dev/nsm`. + +Development root certificates are written as files and configured in +`verifier.toml`. The test explicitly enables +`insecure_allow_external_trust_anchors`; production defaults remain unchanged. + +## Troubleshooting + +### `modprobe: FATAL: Module tpm_vtpm_proxy not found` + +Install modules matching the running kernel or enable +`CONFIG_TCG_VTPM_PROXY` in the host kernel. + +### `/dev/nsm` is not created + +Ensure the host kernel has CUSE support and that the container is privileged. +The test creates the device node from `/sys/class/cuse/nsm/dev` because no udev +daemon runs inside the container. + +### FUSE/configfs permission errors + +Confirm that privileged containers are allowed. Rootless Docker is not +supported by this test because it mounts configfs/securityfs and creates device +nodes. + +### Inspecting a failure + +The per-container working directory is `/run/attestation-e2e`. Before the +container exits, the useful files are: + +```text +simulator.log +collateral.log +verifier.log +request.json +request.json.verification.json +verifier.toml +``` + +For interactive inspection, override the entrypoint and run the platform script +manually: + +```bash +docker compose run --rm --entrypoint bash gcp-tdx +TEE_PLATFORM=gcp-tdx /usr/local/bin/run-platform +``` diff --git a/dstack/tests/e2e/attestation/compose.yaml b/dstack/tests/e2e/attestation/compose.yaml new file mode 100644 index 000000000..e2ac964a6 --- /dev/null +++ b/dstack/tests/e2e/attestation/compose.yaml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +x-attestation-test: &attestation-test + image: dstack-tests-attestation:local + build: + context: ../../.. + dockerfile: tests/e2e/attestation/Dockerfile + privileged: true + network_mode: bridge + volumes: + - /lib/modules:/lib/modules:ro + - ./run-platform.sh:/usr/local/bin/run-platform:ro + tmpfs: + - /run + - /dstack/.host-shared + - /tmp + +services: + dstack-tdx-legacy: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-tdx, TDX_ATTESTATION_VARIANT: legacy } + dstack-tdx-lite: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-tdx, TDX_ATTESTATION_VARIANT: lite } + gcp-tdx: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-gcp-tdx } + amd-sev-snp: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-amd-sev-snp } + aws-nitro-enclave: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-nitro-enclave } + aws-nitro-tpm: + <<: *attestation-test + environment: { TEE_PLATFORM: dstack-aws-nitro-tpm } diff --git a/dstack/tests/e2e/attestation/run-platform.sh b/dstack/tests/e2e/attestation/run-platform.sh new file mode 100755 index 000000000..535c64fec --- /dev/null +++ b/dstack/tests/e2e/attestation/run-platform.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +: "${TEE_PLATFORM:?TEE_PLATFORM is required}" +SEED=7171717171717171717171717171717171717171717171717171717171717171 +REPORT_DATA=42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +WORK=/run/attestation-e2e +SYS_CONFIG=/dstack/.host-shared/.sys-config.json +mkdir -p "$WORK" /run/log/dstack +mount -t tmpfs -o mode=0755 tmpfs /sys/kernel/config +mkdir -p /sys/kernel/config/tsm/report + +VM_CONFIG='{}' +MR_CONFIG='{"version":3,"app_id":"","compose_hash":"","key_provider":"none"}' +if [[ "$TEE_PLATFORM" == dstack-tdx ]]; then + VM_CONFIG=$(jq -c --arg variant "${TDX_ATTESTATION_VARIANT:?}" \ + '.vm_config | fromjson | .tdx_attestation_variant = $variant' \ + /usr/local/share/dstack/tdx-lite-getquote.json) +elif [[ "$TEE_PLATFORM" == dstack-gcp-tdx ]]; then + UKI_HASH=9ab14a46f858662a89adc102d2a57a13f52f75c1769d65a4c34edbbfc8855f0f + printf '%s\n' "$UKI_HASH" > "$WORK/gcp-uki-hash" + dstack-mr gcp-measurement-cbor "$WORK/gcp-uki-hash" > "$WORK/measurement.gcp.cbor" + MEASUREMENT_HASH=$(sha256sum "$WORK/measurement.gcp.cbor" | cut -d' ' -f1) + printf '%s measurement.gcp.cbor\n' "$MEASUREMENT_HASH" > "$WORK/sha256sum.txt" + OS_IMAGE_HASH=$(sha256sum "$WORK/sha256sum.txt" | cut -d' ' -f1) + VM_CONFIG=$(jq -cn \ + --arg os "$OS_IMAGE_HASH" \ + --arg checksum "$(base64 -w0 "$WORK/sha256sum.txt")" \ + --arg measurement "$(base64 -w0 "$WORK/measurement.gcp.cbor")" \ + '{os_image_hash:$os,gcp_measurement:{checksum_file:$checksum,measurement:$measurement}}') +elif [[ "$TEE_PLATFORM" == dstack-amd-sev-snp ]]; then + jq -r .attestation /usr/local/share/dstack/sev-snp-attestation.json | xxd -r -p > "$WORK/snp-fixture.bin" + dstack-util attest-json --input "$WORK/snp-fixture.bin" --output "$WORK/snp-fixture.json" + VM_CONFIG=$(jq -c '.config | fromjson' "$WORK/snp-fixture.json") + MR_CONFIG=$(jq -r .mr_config <<<"$VM_CONFIG") +elif [[ "$TEE_PLATFORM" == dstack-aws-nitro-tpm ]]; then + ZERO_PCR=$(printf '00%.0s' $(seq 1 48)) + printf '%s' "$ZERO_PCR" > "$WORK/pcr4" + cp "$WORK/pcr4" "$WORK/pcr7" + cp "$WORK/pcr4" "$WORK/pcr12" + dstack-mr aws-measurement-cbor "$ZERO_PCR" "$ZERO_PCR" "$ZERO_PCR" \ + > "$WORK/measurement.aws.cbor" + MEASUREMENT_HASH=$(sha256sum "$WORK/measurement.aws.cbor" | cut -d' ' -f1) + printf '%s measurement.aws.cbor\n' "$MEASUREMENT_HASH" > "$WORK/sha256sum.txt" + OS_IMAGE_HASH=$(sha256sum "$WORK/sha256sum.txt" | cut -d' ' -f1) + VM_CONFIG=$(jq -cn \ + --arg os "$OS_IMAGE_HASH" \ + --arg checksum "$(base64 -w0 "$WORK/sha256sum.txt")" \ + --arg measurement "$(base64 -w0 "$WORK/measurement.aws.cbor")" \ + '{os_image_hash:$os,aws_measurement:{checksum_file:$checksum,measurement:$measurement}}') +fi + +cleanup() { + set +e + [[ -n "${SIM_PID:-}" ]] && kill "$SIM_PID" 2>/dev/null + [[ -n "${COLLATERAL_PID:-}" ]] && kill "$COLLATERAL_PID" 2>/dev/null + [[ -s /run/dstack/swtpm.pid ]] && kill "$(cat /run/dstack/swtpm.pid)" 2>/dev/null + fusermount3 -uz /sys/kernel/config/tsm/report 2>/dev/null + umount /sys/class/dmi/id/sys_vendor 2>/dev/null + umount /sys/class/dmi/id/product_name 2>/dev/null +} +trap cleanup EXIT + +cat > "$SYS_CONFIG" <"$WORK/collateral.log" 2>&1 & +COLLATERAL_PID=$! +for _ in $(seq 1 100); do + curl -fsS http://127.0.0.1:18088/tpm/aia/root.pem >/dev/null 2>&1 && break + sleep .05 +done +curl -fsS http://127.0.0.1:18088/tpm/aia/root.pem >/dev/null + +if [[ "$TEE_PLATFORM" == dstack-gcp-tdx || "$TEE_PLATFORM" == dstack-aws-nitro-tpm ]]; then + mount -t tmpfs -o mode=0755 tmpfs /sys/kernel/security + mkdir -p /sys/kernel/security/tpm0 + cp /usr/local/share/dstack/tpm_eventlog.bin /sys/kernel/security/tpm0/binary_bios_measurements + modprobe tpm_vtpm_proxy + if [[ ! -e /dev/vtpmx && -r /sys/class/misc/vtpmx/dev ]]; then + IFS=: read -r major minor < /sys/class/misc/vtpmx/dev + mknod /dev/vtpmx c "$major" "$minor" + fi + chmod 0666 /dev/vtpmx +fi + +dstack-tee-simulator --sys-config "$SYS_CONFIG" >"$WORK/simulator.log" 2>&1 & +SIM_PID=$! +case "$TEE_PLATFORM" in + dstack-tdx|dstack-gcp-tdx|dstack-amd-sev-snp) + for _ in $(seq 1 200); do + [[ -e /sys/kernel/config/tsm/report/provider || -e /sys/kernel/config/tsm/report/com.intel.dcap/outblob ]] && break + sleep .05 + done + ;; + dstack-nitro-enclave) + for _ in $(seq 1 200); do + if [[ ! -e /dev/nsm && -r /sys/class/cuse/nsm/dev ]]; then + IFS=: read -r major minor < /sys/class/cuse/nsm/dev + mknod /dev/nsm c "$major" "$minor" + fi + [[ -e /dev/nsm ]] && break + sleep .05 + done + if [[ ! -e /dev/nsm ]]; then + echo "NSM device node was not created" >&2 + find /sys -path '*nsm*' -o -path '*cuse*' 2>/dev/null | head -100 >&2 + cat "$WORK/simulator.log" >&2 + exit 1 + fi + ;; + dstack-aws-nitro-tpm) + for _ in $(seq 1 200); do + [[ -e /dev/tpm0 || -e /dev/tpmrm0 ]] && break + sleep .05 + done + if [[ ! -e /dev/tpm0 && ! -e /dev/tpmrm0 ]]; then + echo "NitroTPM device node was not created" >&2 + cat "$WORK/simulator.log" >&2 + exit 1 + fi + ;; +esac +if ! kill -0 "$SIM_PID" 2>/dev/null; then + echo "[$TEE_PLATFORM] simulator failed:" >&2 + cat "$WORK/simulator.log" >&2 + exit 1 +fi +echo "platform=$TEE_PLATFORM dmi_vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || true) dmi_product=$(cat /sys/class/dmi/id/product_name 2>/dev/null || true) nsm=$(test -e /dev/nsm && echo yes || echo no)" + +export DCAP_TDX_RTMR_SYSFS_PATH=/sys/kernel/config/tsm/report/com.intel.dcap/measurements +export DSTACK_CCEL_FILE=/sys/kernel/config/tsm/report/com.intel.dcap/ccel +if ! ATTESTATION_HEX=$(dstack-util attest --report-data "$REPORT_DATA" --hex); then + cat "$WORK/simulator.log" >&2 + exit 1 +fi +jq -n --arg attestation "$ATTESTATION_HEX" \ + '{attestation: $attestation, debug: true}' > "$WORK/request.json" + +cat > "$WORK/verifier.toml" <"$WORK/verifier.log" 2>&1 +VERIFIER_RC=$? +set -e +if [[ ! -s "$WORK/request.json.verification.json" ]]; then + cat "$WORK/verifier.log" >&2 + exit 1 +fi +if (( VERIFIER_RC != 0 )); then + cat "$WORK/verifier.log" >&2 + exit "$VERIFIER_RC" +fi +cat "$WORK/request.json.verification.json" +jq -e '.details.quote_verified == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.is_valid == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.details.os_image_hash_verified == true' "$WORK/request.json.verification.json" >/dev/null +jq -e '.details.event_log_verified == true' "$WORK/request.json.verification.json" >/dev/null +echo "[$TEE_PLATFORM${TDX_ATTESTATION_VARIANT:+/$TDX_ATTESTATION_VARIANT}] dstack-util -> verifier full E2E passed" diff --git a/dstack/tests/e2e/attestation/run.sh b/dstack/tests/e2e/attestation/run.sh new file mode 100755 index 000000000..21ac55a22 --- /dev/null +++ b/dstack/tests/e2e/attestation/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +cd "$(dirname "$0")" +docker compose build +for platform in dstack-tdx-legacy dstack-tdx-lite gcp-tdx amd-sev-snp aws-nitro-enclave aws-nitro-tpm; do + echo "=== $platform ===" + docker compose run --rm "$platform" +done +docker compose down --remove-orphans diff --git a/dstack/test-scripts/get-app-key.sh b/dstack/tests/get-app-key.sh similarity index 76% rename from dstack/test-scripts/get-app-key.sh rename to dstack/tests/get-app-key.sh index 7285a4fbe..7fbff4fa4 100755 --- a/dstack/test-scripts/get-app-key.sh +++ b/dstack/tests/get-app-key.sh @@ -11,5 +11,5 @@ URL=https://localhost:8043/prpc/KMS.GetAppKey if [ -z "$CERT_FILE" ]; then curl -s --cacert ${CERT_DIR}/ca.cert ${URL} else - curl -vv --cacert ${CERT_DIR}/ca.cert --cert ${CERT_FILE} --key ${KEY_FILE} ${URL} + curl -vv --cacert "${CERT_DIR}/ca.cert" --cert "${CERT_FILE}" --key "${KEY_FILE}" "${URL}" fi diff --git a/dstack/test-scripts/inspect-cert.sh b/dstack/tests/inspect-cert.sh similarity index 77% rename from dstack/test-scripts/inspect-cert.sh rename to dstack/tests/inspect-cert.sh index 6ecb93389..da6fd62dc 100755 --- a/dstack/test-scripts/inspect-cert.sh +++ b/dstack/tests/inspect-cert.sh @@ -3,4 +3,4 @@ # SPDX-FileCopyrightText: © 2025 Phala Network # SPDX-License-Identifier: Apache-2.0 -openssl x509 -text -noout -in $1 +openssl x509 -text -noout -in "$1" diff --git a/dstack/tpm-qvl/src/lib.rs b/dstack/tpm-qvl/src/lib.rs index 6a260ab18..973b990a1 100644 --- a/dstack/tpm-qvl/src/lib.rs +++ b/dstack/tpm-qvl/src/lib.rs @@ -37,6 +37,48 @@ pub fn get_root_ca(platform: Platform) -> Result<&'static str> { } } +/// Verifies TPM quotes against a caller-selected trust anchor. +/// +/// This follows `dcap_qvl::verify::QuoteVerifier`: [`Self::new_prod`] selects +/// the bundled platform root and [`Self::new`] accepts a PEM root CA. +#[derive(Debug, Clone)] +pub struct QuoteVerifier { + root_ca_pem: String, +} + +impl QuoteVerifier { + pub fn new(root_ca_pem: impl Into) -> Self { + Self { + root_ca_pem: root_ca_pem.into(), + } + } + + pub fn new_prod(platform: Platform) -> Result { + Ok(Self::new(get_root_ca(platform)?)) + } + + pub fn root_ca_pem(&self) -> &str { + &self.root_ca_pem + } + + pub fn verify( + &self, + quote: &tpm_types::TpmQuote, + collateral: &QuoteCollateral, + ) -> Result { + verify::verify_quote_with_ca(quote, collateral, &self.root_ca_pem) + } + + #[cfg(feature = "crl-download")] + pub async fn fetch_and_verify( + &self, + quote: &tpm_types::TpmQuote, + ) -> Result { + let collateral = collateral::get_collateral(quote, &self.root_ca_pem).await?; + self.verify(quote, &collateral).map_err(Into::into) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QuoteCollateral { /// Intermediate certificate chain (PEM format) from device diff --git a/dstack/verifier/README.md b/dstack/verifier/README.md index 0dada3cfe..dff564d66 100644 --- a/dstack/verifier/README.md +++ b/dstack/verifier/README.md @@ -100,7 +100,8 @@ You usually don't need to edit the config file. Just using the default is fine, - `image_cache_dir`: Directory for cached OS images (default: "/tmp/dstack-verifier/cache") - `image_download_url`: URL template for downloading OS images (default: dstack official releases URL) - `image_download_timeout_secs`: Download timeout in seconds (default: 300) -- `pccs_url`: PCCS URL for quote verification (default: uses Intel's public PCCS) +- `attestation.urls.pccs`: PCCS URL (default: production PCCS) +- `attestation.urls.amd_kds`: AMD KDS URL (default: AMD production KDS) ### Example Configuration File @@ -110,7 +111,9 @@ port = 8080 image_cache_dir = "/tmp/dstack-verifier/cache" image_download_url = "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.tar.gz" image_download_timeout_secs = 300 -# pccs_url = "https://pccs.phala.network" +[attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" ``` ## Usage @@ -260,7 +263,7 @@ Beyond pass/fail, the result carries a few descriptive fields so a relying party - **`os_image_is_dev`** — `true` for a development OS image, `false` for production. Dev images are built for local testing and are not hardened for production use, so a relying party generally wants to reject them. - **`os_image_version`** — the dstack OS version (e.g. `0.5.10`), useful for enforcing a minimum version. -- **`attestation_mode`** — the attestation mode that produced the verified quote, serialized as `AttestationMode`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. +- **`attestation_mode`** — the TEE variant that produced the verified quote, serialized as `TeeVariant`: `dstack-tdx`, `dstack-gcp-tdx`, `dstack-nitro-enclave`, `dstack-amd-sev-snp`, or `dstack-aws-nitro-tpm`. - **`acpi_tables_verified`** — whether TDX ACPI table contents were verified. This is useful for relying parties that require `requirements.tdx_measure_acpi_tables = true`. - **`key_provider`** — the decoded `app_info.key_provider_info` (`{name, id}`); `name` is e.g. `kms` or `local`. A `local` key provider means the CVM is not KMS-backed, which is itself a dev/insecure posture signal. The raw bytes remain in `app_info.key_provider_info`. - **`boot_info`** — the policy object a relying party should feed to its auth/governance layer. For AWS EC2 NitroTPM this includes `attestationMode = dstack-aws-nitro-tpm`, PCR4/7/12-derived `osImageHash`, PCR14-bound `mrAggregated`, app identity, instance/device identity, and a `tcbStatus` normalized to `UpToDate`. diff --git a/dstack/verifier/dstack-verifier.toml b/dstack/verifier/dstack-verifier.toml index b098c533a..bdb09fe92 100644 --- a/dstack/verifier/dstack-verifier.toml +++ b/dstack/verifier/dstack-verifier.toml @@ -15,5 +15,18 @@ image_download_url = "https://download.dstack.org/os-images/mr_{OS_IMAGE_HASH}.t # Image download timeout in seconds image_download_timeout_secs = 300 -# Optional PCCS URL for quote verification -# pccs_url = "https://pccs.phala.network" +[attestation] +insecure_allow_external_trust_anchors = false + +[attestation.urls] +# pccs = "https://pccs.phala.network" +# amd_kds = "https://kdsintf.amd.com/vcek/v1" + +[attestation.root_ca] +# tdx = "/etc/dstack/roots/intel-sgx-root-ca.der" +# gcp_tpm = "/etc/dstack/roots/gcp-tpm-root-ca.pem" +# aws_nitro_enclave = "/etc/dstack/roots/aws-nitro-enclave-root-ca.pem" +# aws_nitro_tpm = "/etc/dstack/roots/aws-nitro-tpm-root-ca.pem" +# sev_snp_milan = "/etc/dstack/roots/amd-milan-ark.pem" +# sev_snp_genoa = "/etc/dstack/roots/amd-genoa-ark.pem" +# sev_snp_turin = "/etc/dstack/roots/amd-turin-ark.pem" diff --git a/dstack/verifier/src/main.rs b/dstack/verifier/src/main.rs index c3a171c85..90d5f603a 100644 --- a/dstack/verifier/src/main.rs +++ b/dstack/verifier/src/main.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use anyhow::{Context, Result}; use clap::Parser; +use dstack_attest::attestation::AttestationVerifierConfig; use dstack_verifier::{ CvmVerifier, VerificationDetails, VerificationRequest, VerificationResponse, }; @@ -13,6 +14,7 @@ use figment::{ providers::{Env, Format, Toml}, Figment, }; +use ra_tls::attestation::AttestationVerifier; use rocket::{fairing::AdHoc, get, post, serde::json::Json, State}; use serde::{Deserialize, Serialize}; use tracing::{error, info}; @@ -38,7 +40,8 @@ pub struct Config { pub address: String, pub port: u16, pub image_cache_dir: String, - pub pccs_url: Option, + #[serde(default)] + pub attestation: AttestationVerifierConfig, pub image_download_url: String, pub image_download_timeout_secs: u64, } @@ -87,7 +90,7 @@ async fn run_oneshot(file_path: &str, config: &Config) -> anyhow::Result<()> { config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + Arc::new(AttestationVerifier::load(&config.attestation)?), ); // Run verification @@ -166,10 +169,11 @@ async fn run_cert_oneshot(file_path: &str, config: &Config) -> anyhow::Result<() let cert = fs::read(file_path) .map_err(|e| anyhow::anyhow!("failed to read certificate {}: {}", file_path, e))?; + let attestation_verifier = Arc::new(AttestationVerifier::load(&config.attestation)?); let verified = if cert.starts_with(b"-----BEGIN") { - ra_tls::attestation::verify_pem(&cert, config.pccs_url.as_deref()).await + ra_tls::attestation::verify_pem(&cert, attestation_verifier.as_ref()).await } else { - ra_tls::attestation::verify_der(&cert, config.pccs_url.as_deref()).await + ra_tls::attestation::verify_der(&cert, attestation_verifier.as_ref()).await } .map_err(|e| anyhow::anyhow!("failed to verify RA-TLS certificate: {:#}", e))?; @@ -178,7 +182,8 @@ async fn run_cert_oneshot(file_path: &str, config: &Config) -> anyhow::Result<() // every platform except TDX legacy this is a self-contained check (no image // download); relying parties should only trust `os_image_hash` when // `os_image_hash_verified` is true. - let os_image_hash_verified = verify_cert_os_image_hash(&verified.attestation, config).await; + let os_image_hash_verified = + verify_cert_os_image_hash(&verified.attestation, config, &attestation_verifier).await; let output = serde_json::json!({ "is_valid": true, "details": { @@ -226,6 +231,7 @@ async fn run_cert_oneshot(file_path: &str, config: &Config) -> anyhow::Result<() async fn verify_cert_os_image_hash( attestation: &ra_tls::attestation::VerifiedAttestation, config: &Config, + attestation_verifier: &Arc, ) -> bool { use ra_tls::attestation::AttestationQuote; // Only TDX legacy verification downloads the image; skip it here and report @@ -242,7 +248,7 @@ async fn verify_cert_os_image_hash( config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + attestation_verifier.clone(), ); let mut details = VerificationDetails::default(); verifier @@ -269,7 +275,6 @@ async fn main() -> Result<()> { .merge(Env::prefixed("DSTACK_VERIFIER_")); let config: Config = figment.extract().context("Failed to load configuration")?; - // Check for oneshot modes if let Some(file_path) = cli.verify { if let Err(e) = run_oneshot(&file_path, &config).await { @@ -290,7 +295,7 @@ async fn main() -> Result<()> { config.image_cache_dir.clone(), config.image_download_url.clone(), std::time::Duration::from_secs(config.image_download_timeout_secs), - config.pccs_url.clone(), + Arc::new(AttestationVerifier::load(&config.attestation)?), )); rocket::custom(figment) diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index 8b886ae2a..84cfbdecb 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 use dstack_types::KeyProviderInfo; -use ra_tls::attestation::{AppInfo, AttestationMode}; +use ra_tls::attestation::{AppInfo, TeeVariant}; use serde::{Deserialize, Serialize}; use serde_human_bytes as serde_bytes; @@ -32,7 +32,7 @@ pub struct VerificationResponse { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct PolicyBootInfo { - pub attestation_mode: AttestationMode, + pub attestation_mode: TeeVariant, #[serde(with = "serde_bytes")] pub mr_aggregated: Vec, #[serde(with = "serde_bytes")] @@ -55,7 +55,7 @@ pub struct PolicyBootInfo { impl PolicyBootInfo { pub fn from_app_info( - attestation_mode: AttestationMode, + attestation_mode: TeeVariant, app_info: &AppInfo, tcb_status: String, advisory_ids: Vec, @@ -100,7 +100,7 @@ pub struct VerificationDetails { /// dstack OS version, from the same metadata.json. pub os_image_version: Option, /// Attestation mode that produced the verified quote. - pub attestation_mode: Option, + pub attestation_mode: Option, pub report_data: Option, pub tcb_status: Option, pub advisory_ids: Vec, @@ -157,7 +157,7 @@ pub enum RtmrEventStatus { #[cfg(test)] mod tests { use super::*; - use ra_tls::attestation::AttestationMode; + use ra_tls::attestation::TeeVariant; // the README documents sending either `attestation` or // (`quote` + `event_log` + `vm_config`); every field is optional, so any @@ -206,7 +206,7 @@ mod tests { }; let boot_info = PolicyBootInfo::from_app_info( - AttestationMode::DstackAwsNitroTpm, + TeeVariant::DstackAwsNitroTpm, &app_info, String::new(), Vec::new(), diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 2404ec163..7a665839b 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -5,7 +5,7 @@ use std::{ ffi::OsStr, path::{Path, PathBuf}, - sync::OnceLock, + sync::Arc, time::Duration, }; @@ -17,14 +17,13 @@ use cc_eventlog::{ }, TdxEvent, }; -use dstack_attest::amd_sev_snp::AmdKdsClient; use dstack_mr::{ tdx::TdxRtmr0AcpiHashes, RtmrLog, RtmrLogs, TdxMeasurementDetails, TdxMeasurements, }; use dstack_types::VmConfig; use hex_literal::hex; use ra_tls::attestation::{ - AppInfo, Attestation, AttestationQuote, DstackVerifiedReport, NitroPcrs, PlatformEvidence, + AppInfo, Attestation, AttestationQuote, AttestationVerifier, DstackVerifiedReport, NitroPcrs, TpmQuote, VerifiedAttestation, VersionedAttestation, }; use serde::{Deserialize, Serialize}; @@ -190,8 +189,7 @@ pub struct CvmVerifier { pub image_cache_dir: String, pub download_url: String, pub download_timeout: Duration, - pub pccs_url: Option, - amd_kds_client: OnceLock>, + pub attestation_verifier: Arc, } impl CvmVerifier { @@ -199,24 +197,13 @@ impl CvmVerifier { image_cache_dir: String, download_url: String, download_timeout: Duration, - pccs_url: Option, + attestation_verifier: Arc, ) -> Self { Self { image_cache_dir, download_url, download_timeout, - pccs_url, - amd_kds_client: OnceLock::new(), - } - } - - fn amd_kds_client(&self) -> Result<&AmdKdsClient> { - match self - .amd_kds_client - .get_or_init(|| AmdKdsClient::new().map_err(|err| format!("{err:#}"))) - { - Ok(client) => Ok(client), - Err(err) => bail!("failed to create amd sev-snp KDS client: {err}"), + attestation_verifier, } } @@ -421,6 +408,31 @@ impl CvmVerifier { .is_some_and(|digest| digest == expected)) } + fn prune_unlisted_image_files(extracted_dir: &Path, files_doc: &str) -> Result<()> { + let listed_files: Vec<&OsStr> = files_doc + .lines() + .flat_map(|line| line.split_whitespace().nth(1)) + .map(|s| s.as_ref()) + .collect(); + let files = fs_err::read_dir(extracted_dir).context("Failed to read directory")?; + for file in files { + let file = file.context("Failed to read directory entry")?; + let filename = file.file_name(); + // sha256sum.txt is the content-addressed OS identity and is needed + // again when a legacy TDX quote is verified from the cache. + if filename != OsStr::new("sha256sum.txt") + && !listed_files.contains(&filename.as_os_str()) + { + if file.path().is_dir() { + fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; + } else { + fs_err::remove_file(file.path()).context("Failed to remove file")?; + } + } + } + Ok(()) + } + fn tdx_acpi_hashes_from_event_log(event_log: &[TdxEvent]) -> Result { let rtmr0_events = event_log .iter() @@ -570,13 +582,7 @@ impl CvmVerifier { let debug = request.debug.unwrap_or(false); let attestation = attestation.into_v1(); - let verified = if matches!(&attestation.platform, PlatformEvidence::SevSnp { .. }) { - attestation - .verify_with_amd_kds_client(self.pccs_url.as_deref(), self.amd_kds_client()?) - .await - } else { - attestation.verify(self.pccs_url.as_deref()).await - }; + let verified = attestation.verify(&self.attestation_verifier).await; let verified_attestation = match verified { Ok(att) => { details.quote_verified = true; @@ -668,7 +674,13 @@ impl CvmVerifier { self.verify_os_image_hash_for_gcp_tdx(&vm_config, "e.tpm_quote)?; } AttestationQuote::DstackTdx(_) => { - if vm_config.tdx_attestation_variant.is_lite() { + // New images carry a self-contained measurement document even + // when the boot kept the legacy attestation selector. Prefer + // that signed-MR-bound material; retain image download only + // for old legacy images which do not provide it. + if vm_config.tdx_attestation_variant.is_lite() + || vm_config.tdx_measurement.is_some() + { self.verify_os_image_hash_for_dstack_tdx_lite( &vm_config, attestation, @@ -1180,23 +1192,7 @@ impl CvmVerifier { let sha256sum_path = extracted_dir.join("sha256sum.txt"); let files_doc = fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; - let listed_files: Vec<&OsStr> = files_doc - .lines() - .flat_map(|line| line.split_whitespace().nth(1)) - .map(|s| s.as_ref()) - .collect(); - let files = fs_err::read_dir(&extracted_dir).context("Failed to read directory")?; - for file in files { - let file = file.context("Failed to read directory entry")?; - let filename = file.file_name(); - if !listed_files.contains(&filename.as_os_str()) { - if file.path().is_dir() { - fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; - } else { - fs_err::remove_file(file.path()).context("Failed to remove file")?; - } - } - } + Self::prune_unlisted_image_files(&extracted_dir, &files_doc)?; // All image modes are addressed by sha256(sha256sum.txt). Extra // measurement CBOR files are ordinary sha256sum.txt entries and do not @@ -1302,7 +1298,16 @@ mod tests { } fn test_verifier() -> CvmVerifier { - CvmVerifier::new(String::new(), String::new(), Duration::from_secs(1), None) + CvmVerifier::new( + String::new(), + String::new(), + Duration::from_secs(1), + test_attestation_verifier(), + ) + } + + fn test_attestation_verifier() -> Arc { + Arc::new(AttestationVerifier::new_prod(None).unwrap()) } #[test] @@ -1385,6 +1390,21 @@ mod tests { assert!(decode_key_provider_info(b"not json").is_none()); } + #[test] + fn image_cache_pruning_keeps_checksum_identity() { + let dir = tempfile::tempdir().expect("temp image directory"); + let files_doc = "00 metadata.json\n"; + fs_err::write(dir.path().join("sha256sum.txt"), files_doc).unwrap(); + fs_err::write(dir.path().join("metadata.json"), "{}").unwrap(); + fs_err::write(dir.path().join("unmeasured"), "remove me").unwrap(); + + CvmVerifier::prune_unlisted_image_files(dir.path(), files_doc).unwrap(); + + assert!(dir.path().join("sha256sum.txt").exists()); + assert!(dir.path().join("metadata.json").exists()); + assert!(!dir.path().join("unmeasured").exists()); + } + #[tokio::test] async fn verifies_sev_snp_attestation_fixture_without_image_download() { let request: VerificationRequest = @@ -1396,7 +1416,7 @@ mod tests { image_cache_dir.display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); @@ -1407,7 +1427,7 @@ mod tests { assert!(!response.details.acpi_tables_verified); assert_eq!( response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackAmdSevSnp) + Some(ra_tls::attestation::TeeVariant::DstackAmdSevSnp) ); assert!( !image_cache_dir.exists(), @@ -1428,14 +1448,14 @@ mod tests { cache.path().join("cache").display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); assert!(response.is_valid, "{:?}", response.reason); assert_eq!( response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackAmdSevSnp) + Some(ra_tls::attestation::TeeVariant::DstackAmdSevSnp) ); } @@ -1450,7 +1470,7 @@ mod tests { image_cache_dir.display().to_string(), "http://127.0.0.1:9/should-not-download/{OS_IMAGE_HASH}.tar.gz".to_string(), Duration::from_secs(1), - None, + test_attestation_verifier(), ); let response = verifier.verify(request).await.expect("verifier runs"); @@ -1461,7 +1481,7 @@ mod tests { assert!(!response.details.acpi_tables_verified); assert_eq!( response.details.attestation_mode, - Some(ra_tls::attestation::AttestationMode::DstackTdx) + Some(ra_tls::attestation::TeeVariant::DstackTdx) ); assert!( !image_cache_dir.exists(), diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 637e9883c..797185622 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1073,8 +1073,8 @@ impl App { .app_compose() .context("Failed to get app compose")?; let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::TeePlatform::AmdSevSnp - || (platform == crate::config::TeePlatform::Tdx + && (platform == crate::config::CvmPlatform::AmdSevSnp + || (platform == crate::config::CvmPlatform::Tdx && cfg.cvm.use_mrconfigid && !app_compose.key_provider_id.is_empty())); let mr_config = if use_mr_config_v3 { @@ -1269,6 +1269,8 @@ pub(crate) fn make_sys_config( "kms_urls": kms_urls, "gateway_urls": gateway_urls, "pccs_url": cfg.cvm.pccs_url, + "collateral_urls": { "pccs": cfg.cvm.pccs_url }, + "tee_simulator": cfg.cvm.tee_simulator, "nvidia_attestation_proxy_url": cfg.cvm.nvidia_attestation_proxy_url, "docker_registry": cfg.cvm.docker_registry, "host_api_url": format!("vsock://2:{}/api", cfg.host_api.port), @@ -1338,8 +1340,8 @@ fn make_vm_config( requirements: Option<&dstack_types::Requirements>, ) -> Result { let platform = cfg.cvm.resolved_platform(); - let is_amd_sev_snp = platform == crate::config::TeePlatform::AmdSevSnp && !manifest.no_tee; - let is_tdx = platform == crate::config::TeePlatform::Tdx && !manifest.no_tee; + let is_amd_sev_snp = platform == crate::config::CvmPlatform::AmdSevSnp && !manifest.no_tee; + let is_tdx = platform == crate::config::CvmPlatform::Tdx && !manifest.no_tee; let tdx_attestation_variant = if is_tdx { tdx_attestation_variant_from_requirements(requirements).unwrap_or_else(|| { cfg.cvm @@ -1438,7 +1440,7 @@ fn make_vm_config( mod tests { use super::*; use crate::config::{ - load_config_figment, Networking, NetworkingMode, TdxAttestationVariantConfig, TeePlatform, + load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -1674,7 +1676,7 @@ mod tests { fn test_tdx_config() -> Result { let mut config: Config = Figment::from(load_config_figment(None)).extract()?; - config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.platform = Some(CvmPlatform::Tdx); config.cvm.tdx_attestation_variant = TdxAttestationVariantConfig::Auto; Ok(config) } @@ -1855,7 +1857,7 @@ mod tests { let mut config: Config = Figment::from(load_config_figment(None)).extract()?; config.image.path = image_root; - config.cvm.platform = Some(TeePlatform::AmdSevSnp); + config.cvm.platform = Some(CvmPlatform::AmdSevSnp); config.cvm.nvidia_attestation_proxy_url = Some("http://10.0.2.2:8090".to_string()); let compose_hash = hex_of(0x22, 32); let manifest = Manifest { @@ -1915,6 +1917,8 @@ mod tests { let sys_config_document = make_sys_config(&config, &manifest, &compose_hash, Some(mr_config), None)?; let sys_config: serde_json::Value = serde_json::from_str(&sys_config_document)?; + assert_eq!(sys_config["pccs_url"], config.cvm.pccs_url); + assert_eq!(sys_config["collateral_urls"]["pccs"], config.cvm.pccs_url); let vm_config: serde_json::Value = serde_json::from_str( sys_config["vm_config"] .as_str() diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 270c62c73..dd45cbf7f 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -3,20 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 //! QEMU launch preparation and command construction. -use crate::{ - app::Manifest, - config::{CvmConfig, Networking, NetworkingMode, ProcessAnnotation, TeePlatform}, - vm_launcher::{ChildCommand, LaunchSpec}, -}; -use std::collections::HashMap; -use std::os::unix::fs::PermissionsExt; -use std::{ - fs::Permissions, - io::Write, - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - use super::{ effective_vcpu_count, host_share::create_shared_disk, @@ -26,12 +12,27 @@ use super::{ network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, pci_numa_node, round_up, GpuConfig, VmWorkDir, }; +use crate::{ + app::Manifest, + config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, + vm_launcher::{ChildCommand, LaunchSpec}, +}; use anyhow::{bail, Context, Result}; use bon::Builder; -use dstack_types::{shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind}; +use dstack_types::{ + shared_filenames::HOST_SHARED_DISK_LABEL, KeyProviderKind, TeeSimulatorConfig, TeeVariant, +}; use fs_err as fs; use nix::unistd::User; use serde::Serialize; +use std::collections::HashMap; +use std::os::unix::fs::PermissionsExt; +use std::{ + fs::Permissions, + io::Write, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; use supervisor_client::supervisor::ProcessConfig; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -174,9 +175,19 @@ fn virtio_pci_device(device: &str, snp: bool) -> String { } } +fn needs_qemu_swtpm(key_provider: KeyProviderKind, simulator: Option<&TeeSimulatorConfig>) -> bool { + if !matches!(key_provider, KeyProviderKind::Tpm) { + return false; + } + !matches!( + simulator.map(|config| config.platform), + Some(TeeVariant::DstackGcpTdx | TeeVariant::DstackAwsNitroTpm) + ) +} + struct PreparedQemuLaunch { workdir: VmWorkDir, - platform: TeePlatform, + platform: CvmPlatform, networks: Vec, hugepage_numa_nodes: Option>, gpu_numa_nodes: HashMap, @@ -223,7 +234,7 @@ impl PreparedQemuLaunch { None }; let (swtpm_socket, swtpm_path) = - if matches!(app_compose.key_provider(), KeyProviderKind::Tpm) { + if needs_qemu_swtpm(app_compose.key_provider(), cfg.tee_simulator.as_ref()) { let swtpm_path = which::which("swtpm") .context("tpm key provider requested but swtpm is not installed")?; let state_dir = workdir.swtpm_state_dir(); @@ -240,7 +251,7 @@ impl PreparedQemuLaunch { let tee_enabled = !vm.manifest.no_tee; let tdx_mr_config_id = if tee_enabled - && platform == TeePlatform::Tdx + && platform == CvmPlatform::Tdx && cfg.use_mrconfigid && vm.image.info.version_tuple().unwrap_or_default() >= (0, 5, 2) { @@ -249,7 +260,7 @@ impl PreparedQemuLaunch { None }; let (snp_host_data, snp_launch_params) = - if tee_enabled && platform == TeePlatform::AmdSevSnp { + if tee_enabled && platform == CvmPlatform::AmdSevSnp { ( Some(snp_host_data(&workdir)?), Some( @@ -430,7 +441,7 @@ impl QemuCommandBuilder<'_> { } fn is_amd_sev_snp(&self) -> bool { - self.prepared.platform == TeePlatform::AmdSevSnp && !self.vm.manifest.no_tee + self.prepared.platform == CvmPlatform::AmdSevSnp && !self.vm.manifest.no_tee } fn base_command(&self) -> Command { @@ -775,13 +786,13 @@ impl VmConfig { } match prepared.platform { - TeePlatform::Tdx => { + CvmPlatform::Tdx => { command .arg("-machine") .arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off"); self.configure_tdx_guest(command, cfg, prepared.tdx_mr_config_id.as_deref())?; } - TeePlatform::AmdSevSnp => { + CvmPlatform::AmdSevSnp => { let host_data = prepared .snp_host_data .as_deref() @@ -938,12 +949,29 @@ mod tests { }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, - PreparedQemuLaunch, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, needs_qemu_swtpm, parse_amd_sev_snp_qmp_capabilities, + virtio_pci_device, PreparedQemuLaunch, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; use crate::app::{GpuConfig, Manifest, PortMapping, VmWorkDir}; - use crate::config::{Config, Protocol, TeePlatform, DEFAULT_CONFIG}; + use crate::config::{Config, CvmPlatform, Protocol, DEFAULT_CONFIG}; + use dstack_types::{KeyProviderKind, TeeSimulatorConfig, TeeVariant}; + + #[test] + fn qemu_swtpm_is_omitted_when_simulator_provides_the_tpm() { + for platform in [TeeVariant::DstackGcpTdx, TeeVariant::DstackAwsNitroTpm] { + let simulator = TeeSimulatorConfig { + platform, + ..Default::default() + }; + assert!(!needs_qemu_swtpm(KeyProviderKind::Tpm, Some(&simulator))); + } + + let tdx = TeeSimulatorConfig::default(); + assert!(needs_qemu_swtpm(KeyProviderKind::Tpm, Some(&tdx))); + assert!(needs_qemu_swtpm(KeyProviderKind::Tpm, None)); + assert!(!needs_qemu_swtpm(KeyProviderKind::Kms, None)); + } #[test] fn amd_sev_snp_memory_backend_arg_uses_passed_final_memory_size() { @@ -982,7 +1010,7 @@ mod tests { let mut config: Config = Figment::from(Toml::string(DEFAULT_CONFIG)) .extract() .unwrap(); - config.cvm.platform = Some(TeePlatform::Tdx); + config.cvm.platform = Some(CvmPlatform::Tdx); config.cvm.qemu_path = PathBuf::from("/not-installed/qemu-system-x86_64"); config.cvm.qgs_port = None; @@ -1041,7 +1069,7 @@ mod tests { }; let mut prepared = PreparedQemuLaunch { workdir: VmWorkDir::new("/does-not-exist/vm-1"), - platform: TeePlatform::Tdx, + platform: CvmPlatform::Tdx, networks: vec![config.cvm.networking.clone(), config.cvm.networking.clone()], hugepage_numa_nodes: None, gpu_numa_nodes: HashMap::new(), diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 38e9a7bd2..658074648 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -109,12 +109,12 @@ impl Protocol { #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] -pub enum TeePlatform { +pub enum CvmPlatform { Tdx, AmdSevSnp, } -impl TeePlatform { +impl CvmPlatform { /// Detect the host TEE platform from /proc/cpuinfo. Used when the operator /// did not pin a platform in the config (`platform` omitted, or `auto`). pub fn detect() -> Self { @@ -181,7 +181,7 @@ impl PortMappingConfig { /// Deserialize the optional `platform` config field. `None` (field omitted, or /// the legacy literal `auto`) means "detect the host TEE"; `tdx` / `amd-sev-snp` /// pin a platform. Keeping `auto` accepted preserves existing vmm.toml configs. -fn deserialize_platform<'de, D>(deserializer: D) -> Result, D::Error> +fn deserialize_platform<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, { @@ -195,8 +195,8 @@ where Ok( match Option::::deserialize(deserializer)? { None | Some(PlatformSetting::Auto) => None, - Some(PlatformSetting::Tdx) => Some(TeePlatform::Tdx), - Some(PlatformSetting::AmdSevSnp) => Some(TeePlatform::AmdSevSnp), + Some(PlatformSetting::Tdx) => Some(CvmPlatform::Tdx), + Some(PlatformSetting::AmdSevSnp) => Some(CvmPlatform::AmdSevSnp), }, ) } @@ -204,8 +204,8 @@ where impl CvmConfig { /// The effective TEE platform: the configured one, or host auto-detection /// when left unset (`platform` omitted / `auto`). - pub fn resolved_platform(&self) -> TeePlatform { - self.platform.unwrap_or_else(TeePlatform::detect) + pub fn resolved_platform(&self) -> CvmPlatform { + self.platform.unwrap_or_else(CvmPlatform::detect) } } @@ -250,7 +250,7 @@ pub struct CvmConfig { /// the host TEE from /proc/cpuinfo (AMD SEV-SNP vs Intel TDX); set `tdx` or /// `amd-sev-snp` to force a platform. #[serde(default, deserialize_with = "deserialize_platform")] - pub platform: Option, + pub platform: Option, pub qemu_path: PathBuf, /// The URL of the KMS server pub kms_urls: Vec, @@ -260,6 +260,9 @@ pub struct CvmConfig { /// The URL of the PCCS server #[serde(default)] pub pccs_url: String, + /// Development-image simulator configuration copied into guest sys-config. + #[serde(default)] + pub tee_simulator: Option, /// Optional NVIDIA OCSP/RIM cache passed to guests in sys-config. #[serde(default)] pub nvidia_attestation_proxy_url: Option, @@ -726,8 +729,8 @@ mod tests { #[test] fn tee_platform_deserializes_amd_sev_snp() { - let platform: TeePlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); - assert_eq!(platform, TeePlatform::AmdSevSnp); + let platform: CvmPlatform = serde_json::from_str("\"amd-sev-snp\"").unwrap(); + assert_eq!(platform, CvmPlatform::AmdSevSnp); } #[test] @@ -735,17 +738,17 @@ mod tests { #[derive(Deserialize)] struct Wrap { #[serde(default, deserialize_with = "deserialize_platform")] - platform: Option, + platform: Option, } let parse = |s: &str| serde_json::from_str::(s).unwrap().platform; // Omitted and the legacy `auto` literal both mean "auto-detect" (None). assert_eq!(parse("{}"), None); assert_eq!(parse(r#"{"platform":"auto"}"#), None); // Explicit platforms are pinned. - assert_eq!(parse(r#"{"platform":"tdx"}"#), Some(TeePlatform::Tdx)); + assert_eq!(parse(r#"{"platform":"tdx"}"#), Some(CvmPlatform::Tdx)); assert_eq!( parse(r#"{"platform":"amd-sev-snp"}"#), - Some(TeePlatform::AmdSevSnp) + Some(CvmPlatform::AmdSevSnp) ); } @@ -788,20 +791,20 @@ mod tests { fn tee_platform_auto_detects_amd_sev_snp_from_flag() { let cpuinfo = "flags : fpu svm sev sev_es sev_snp debug_swap"; assert_eq!( - TeePlatform::resolve_from_cpuinfo(cpuinfo), - TeePlatform::AmdSevSnp + CvmPlatform::resolve_from_cpuinfo(cpuinfo), + CvmPlatform::AmdSevSnp ); } #[test] fn tee_platform_auto_detects_tdx_host() { let cpuinfo = "flags : fpu vmx tdx_host_platform"; - assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); } #[test] fn tee_platform_auto_falls_back_to_tdx_without_tee_flag() { let cpuinfo = "flags : fpu vmx"; - assert_eq!(TeePlatform::resolve_from_cpuinfo(cpuinfo), TeePlatform::Tdx); + assert_eq!(CvmPlatform::resolve_from_cpuinfo(cpuinfo), CvmPlatform::Tdx); } } diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index abcb36ba4..c02b2ce44 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -240,8 +240,8 @@ Compose file content (first 200 chars): .context("Failed to get app compose")?; let platform = config.cvm.resolved_platform(); let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::TeePlatform::AmdSevSnp - || (platform == crate::config::TeePlatform::Tdx + && (platform == crate::config::CvmPlatform::AmdSevSnp + || (platform == crate::config::CvmPlatform::Tdx && config.cvm.use_mrconfigid && !app_compose.key_provider_id.is_empty())); let mr_config = if use_mr_config_v3 { diff --git a/dstack/vmm/src/tests/test_vmm_cli.py b/dstack/vmm/src/tests/test_vmm_cli.py new file mode 100644 index 000000000..f3c33c371 --- /dev/null +++ b/dstack/vmm/src/tests/test_vmm_cli.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Focused regression tests for vmm-cli update request construction.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + + +def load_vmm_cli(): + """Load the executable vmm-cli.py as a normal Python module.""" + path = Path(__file__).resolve().parents[1] / "vmm-cli.py" + spec = importlib.util.spec_from_file_location("dstack_vmm_cli", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +VMM_CLI = load_vmm_cli() + + +class UpdateVmTests(unittest.TestCase): + """Verify update flags are not silently discarded by the CLI.""" + + def test_kms_urls_are_sent_to_upgrade_app(self) -> None: + """Send both the update flag and requested KMS URL list.""" + cli = VMM_CLI.VmmCLI("http://127.0.0.1:8080") + calls: list[tuple[str, dict]] = [] + cli.rpc_call = lambda method, params=None: calls.append((method, params)) or {} + + with contextlib.redirect_stdout(io.StringIO()): + cli.update_vm("vm-id", kms_urls=["https://kms.example:8000"]) + + self.assertEqual( + calls, + [ + ( + "UpgradeApp", + { + "id": "vm-id", + "update_kms_urls": True, + "kms_urls": ["https://kms.example:8000"], + }, + ) + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 9671a025e..b86fc8981 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -1150,6 +1150,11 @@ def update_vm( upgrade_params["user_config"] = user_config updates.append("user config") + if kms_urls is not None: + upgrade_params["update_kms_urls"] = True + upgrade_params["kms_urls"] = kms_urls + updates.append(f"KMS URLs ({len(kms_urls)})") + # handle port updates - only update if --port or --no-ports is specified if no_ports or ports is not None: if no_ports: diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb index 032c12490..5805f4024 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb @@ -13,7 +13,7 @@ DSTACK_CORE_SRC ?= "${DSTACK_MONOREPO_ROOT}/dstack" S = "${UNPACKDIR}/repo/dstack" DEPENDS += "rsync-native" -RDEPENDS:${PN} += "fuse3-utils kernel-module-fuse" +RDEPENDS:${PN} += "fuse3-utils kernel-module-fuse swtpm tpm2-tools openssl" do_unpack[depends] += "rsync-native:do_populate_sysroot" SYSTEMD_SERVICE:${PN} = "dstack-tee-simulator.service" diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/dstack-tee-simulator.service b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/dstack-tee-simulator.service index 10777a77a..113759ec0 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/dstack-tee-simulator.service +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/dstack-tee-simulator.service @@ -11,7 +11,11 @@ ExecCondition=/bin/sh -c '! grep -qE "^(tdx_guest|sev_guest)" /sys/kernel/config ExecStart=/usr/bin/dstack-tee-simulator ExecStartPost=/bin/systemctl set-environment DCAP_TDX_RTMR_SYSFS_PATH=/sys/kernel/config/tsm/report/com.intel.dcap/measurements DSTACK_CCEL_FILE=/sys/kernel/config/tsm/report/com.intel.dcap/ccel ExecStop=-/bin/fusermount3 -u /sys/kernel/config/tsm/report +ExecStopPost=-/bin/sh -c 'test ! -s /run/dstack/swtpm.pid || kill "$(cat /run/dstack/swtpm.pid)"' ExecStopPost=-/bin/fusermount3 -uz /sys/kernel/config/tsm/report +ExecStopPost=-/bin/umount /sys/class/dmi/id/sys_vendor +ExecStopPost=-/bin/umount /sys/class/dmi/id/product_name +ExecStopPost=-/bin/sh -c 'test ! -e /run/dstack/created-tpm-marker || rm -f /dev/tpmrm0 /run/dstack/created-tpm-marker' ExecStopPost=/bin/systemctl unset-environment DCAP_TDX_RTMR_SYSFS_PATH DSTACK_CCEL_FILE Restart=on-failure RestartSec=1s diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg index 93f355d3e..489c09841 100644 --- a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -75,3 +75,9 @@ CONFIG_HARDENED_USERCOPY=y CONFIG_FORTIFY_SOURCE=y CONFIG_MAGIC_SYSRQ=n CONFIG_LEGACY_TIOCSTI=n + +# Userspace character device and virtual TPM support. +CONFIG_FUSE_FS=y +CONFIG_CUSE=y +CONFIG_TCG_TPM=y +CONFIG_TCG_VTPM_PROXY=y diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example new file mode 100644 index 000000000..ad86f18ec --- /dev/null +++ b/test-suites/full-stack-compose/.env.example @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Directory containing unpacked dstack guest images: +# //{bzImage,initramfs...,rootfs...,digest.txt,sha256sum.txt} +DSTACK_E2E_IMAGE_STORE=../../../meta-dstack/build/images +DSTACK_E2E_IMAGE_NAME=dstack-0.6.0 +DSTACK_E2E_PLATFORM=tdx + +# Exact released upgrade sources. Both are digest-pinned on Docker Hub and +# checked by their in-image --version output before any CVM is launched. +DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d +DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411 +DSTACK_E2E_APP_IMAGE=nginx:alpine + +# Mock ACME/DNS + externally visible test domain for Gateway SNI. +DSTACK_E2E_BASE_DOMAIN=e2e.test +DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest + +# Host ports. Defaults are chosen to avoid common dstackup ports. +DSTACK_E2E_VMM_PORT=29080 +DSTACK_E2E_AUTH_PORT=28011 +DSTACK_E2E_ARTIFACT_PORT=38081 +DSTACK_E2E_KMS_OLD_HOST_PORT=28082 +DSTACK_E2E_KMS_LATEST_HOST_PORT=28083 +# KMS 0.5.8 encodes this value as a DNS SAN. It must resolve to the test host +# from BOTH the host-side deployment client and each CVM. If omitted, the +# renderer derives .nip.io (for example below). +# DSTACK_E2E_KMS_RPC_DOMAIN=192.168.1.90.nip.io +DSTACK_E2E_GATEWAY1_RPC_HOST_PORT=28000 +DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT=28001 +DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT=28443 +DSTACK_E2E_GATEWAY1_WG_HOST_PORT=28120 +DSTACK_E2E_GATEWAY2_RPC_HOST_PORT=28100 +DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT=28101 +DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT=28543 +DSTACK_E2E_GATEWAY2_WG_HOST_PORT=28121 +DSTACK_E2E_KEY_PROVIDER_PORT=13443 +DSTACK_E2E_HOST_API_PORT=20011 +DSTACK_E2E_MOCK_CF_HTTP_PORT=38080 +DSTACK_E2E_PEBBLE_HTTP_PORT=34000 +DSTACK_E2E_PEBBLE_MGMT_PORT=35000 + +# Optional exact 20-byte Gateway application ID. If omitted, the renderer +# deterministically derives one. "any" is intentionally rejected. +# DSTACK_E2E_GATEWAY_APP_ID=0123456789abcdef0123456789abcdef01234567 + +# VMM/QEMU knobs. +DSTACK_E2E_CID_START=15000 +DSTACK_E2E_QGS_PORT=4050 +DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64 + +# SGX QCNL config used by the Local-Key-Provider that seals each KMS CVM disk. +DSTACK_E2E_QCNL_CONF=../../dstack/local-key-provider/build/sgx_default_qcnl.conf + +# Runner behaviour. +DSTACK_E2E_CLEAN_START=true +DSTACK_E2E_CLEANUP_AFTER=false +DSTACK_E2E_UPGRADE_CLEAN_STATE=true +DSTACK_E2E_KEEP_STACK=true + +# Set only when the current x86_64-musl KMS/Gateway binaries were already built. +DSTACK_E2E_SKIP_CURRENT_BUILD=false diff --git a/test-suites/full-stack-compose/.gitignore b/test-suites/full-stack-compose/.gitignore new file mode 100644 index 000000000..89e702f8d --- /dev/null +++ b/test-suites/full-stack-compose/.gitignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.env +/state/* +!/state/.gitkeep +__pycache__/ +*.pyc diff --git a/test-suites/full-stack-compose/README.md b/test-suites/full-stack-compose/README.md new file mode 100644 index 000000000..6329cb042 --- /dev/null +++ b/test-suites/full-stack-compose/README.md @@ -0,0 +1,140 @@ + + +# Production-compatible KMS/Gateway upgrade E2E + +This suite runs the stateful services and applications in **real TDX CVMs**. +Docker Compose is used only for host infrastructure (VMM, authorization, +Local-Key-Provider, image server, and mock ACME/DNS): + +```text +host Docker Compose + ├─ dstack-vmm + dstack-auth + ├─ AESMD + SGX Local-Key-Provider + ├─ verified OS/image artifact server + └─ Pebble + mock Cloudflare DNS + +real TDX CVMs + ├─ KMS 0.5.8 (Local-Key-Provider, quote-enabled onboarding) + ├─ current KMS (new CVM, quote-attested replication from 0.5.8) + ├─ two-node Gateway 0.5.8 cluster, rolled one node at a time to current + ├─ forced-legacy nginx app CVM + └─ forced-lite nginx app CVM +``` + +The upgrade scenario intentionally rejects the development trust settings that +would make a passing result irrelevant to production: + +- KMS 0.5.8 uses `quote_enabled = true`. +- Both KMS manifests explicitly set `enforce_self_authorization = true`. +- KMS OS image verification is enabled and downloads a checksum-verified image + archive from an initially empty cache. +- KMS root keys and Gateway TLS identity are created inside TDX CVMs; the suite + never generates or injects them on the host. +- The authorization webhook pins one OS digest, exact KMS aggregate + measurements, the quote-derived physical TDX device ID, exact app compose + hashes, and a concrete 20-byte Gateway app ID. `allowAnyDevice = true` and + `gatewayAppId = "any"` are rejected. +- Container references in measured app manifests are content-addressed Docker + image IDs. Tags are used only as the Docker Hub/build inputs that are pulled, + inspected, saved, and imported before boot. + +## Prerequisites + +1. Build the host-side control binaries: + + ```bash + cargo build --release --manifest-path ../../dstack/Cargo.toml \ + -p dstack-cli -p dstack-auth -p dstack-vmm -p supervisor + ``` + + The driver builds current KMS and Gateway binaries for + `x86_64-unknown-linux-musl` and places them in test-only container images. + The static binaries are layered on the released 0.5.8 production runtime; + the KMS production builder still uses the same Debian/QEMU revision, and the + current Gateway entrypoint is copied from this checkout. The built binaries + and in-CVM version endpoints must report the current workspace version and + Git revision, so Docker cache or an old binary cannot silently turn the + upgrade into a no-op. + +2. Provide a TDX/SGX host with: + + - `/dev/kvm`, `/dev/vhost-vsock`, Intel TDX, and QGS (default port `4050`) + - `/dev/sgx_enclave` and `/dev/sgx_provision` + - a working SGX QCNL/PCCS configuration + - an IPv4/DNS name for the host that is reachable from both the host-side + deployment client and QEMU user-networked CVMs; the suite derives + `.nip.io`, or accepts `DSTACK_E2E_KMS_RPC_DOMAIN` + - Docker and WireGuard kernel support + +3. Provide an unpacked dstack image directory containing `digest.txt` and + `sha256sum.txt`. Copy `.env.example` to `.env` when paths or ports differ. + +## Run + +```bash +DOCKER_BUILDKIT=0 ./run-upgrade-e2e.sh +``` + +The defaults pull these released images from Docker Hub: + +- `dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d` +- `dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411` + +The driver checks each released binary's `--version` output before deploying +anything. Set `DSTACK_E2E_SKIP_CURRENT_BUILD=true` only when the current musl +binaries were already built. + +## Upgrade sequence and assertions + +### KMS 0.5.8 to current + +1. Deploy KMS 0.5.8 as a Local-Key-Provider TDX CVM with no pre-created keys. +2. Call `Onboard.GetAttestationInfo`, authorize its exact `mrAggregated`, then + bootstrap it. The bootstrap response must contain non-empty attestation. +3. Boot the forced-legacy app through KMS 0.5.8 and record the KMS CA SPKI, + root k256 public key, and the app's derived environment-encryption public + key. (The expected onboarding path reissues the self-signed CA certificate, + so its DER bytes/expiry are not a stable identity; its private key/SPKI is.) +4. Deploy current KMS in a new Local-Key-Provider TDX CVM, authorize its exact + measurement, and call `Onboard.Onboard` against KMS 0.5.8. This exercises the + production RA-TLS root-key replication path; it is not a directory copy. +5. Require all recorded identities/derived keys to be byte-for-byte equal, + stop KMS 0.5.8, and force the legacy app to boot against only current KMS. +6. Boot a fresh forced-lite app against current KMS. Both modes must reach + `boot_progress=done`, which occurs after boot-time `GetAppKey` succeeds. + +The artifact server journal must contain two successful GETs for the exact +measured OS archive (one from each fresh KMS data disk). VM logs are also +rejected if they say image verification or self-authorization is disabled. + +### Gateway 0.5.8 to current + +1. Deploy two Gateway 0.5.8 TDX CVMs under one pinned Gateway app ID. Both get + app keys and RA-TLS certificates from KMS; their environment is encrypted by + VMM/KMS. +2. Form a WaveKV cluster, configure Pebble/Cloudflare mock DNS once, and require + the second node to receive the configuration and wildcard certificate. +3. Verify the legacy and lite app SNI routes through **both** Gateway nodes. +4. Snapshot each node's UUID, CVM registrations/IPs, certbot configuration, DNS + credential metadata, ZT domains, and wildcard-certificate fingerprint. +5. Run a rapid HA probe that alternates the preferred physical Gateway for + every legacy/lite request and falls back to the other node in the same cycle. +6. Stop, update the measured app compose, and restart node 1 on current KMS; + require it to carry both apps and retain all durable state. Repeat for node 2. +7. Require zero failed HA cycles and unchanged durable state/certificates. +8. Force certificate issuance through each upgraded node against a mock DNS API + that rejects anything except the original bearer token. This proves the DNS + credential value survived even though the current admin API redacts it. + +This is a production-style **rolling two-node** zero-downtime assertion. The +suite does not claim that rebooting a single Gateway CVM can preserve that +node's listener; availability is maintained by the peer, as it must be in a +production rollout. + +Artifacts, manifests, version headers, canonical state snapshots, probe +results, attestation information, and VM logs are written to `state/work/`. +The stack is retained by default for inspection; set +`DSTACK_E2E_KEEP_STACK=false` and/or `DSTACK_E2E_CLEANUP_AFTER=true` to clean it. diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml new file mode 100644 index 000000000..f2fb877ad --- /dev/null +++ b/test-suites/full-stack-compose/compose.yml @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: dstack-full-e2e + +x-suite-env: &suite-env + DSTACK_E2E_STATE_DIR: /suite-state + DSTACK_E2E_CONFIG_DIR: /suite-state/config + DSTACK_E2E_RUN_DIR: /suite-state/run + DSTACK_E2E_VM_DIR: /suite-state/vm + DSTACK_E2E_VOLUMES_DIR: /suite-state/volumes + DSTACK_E2E_IMAGE_ROOT: /images + DSTACK_E2E_IMAGE_NAME: ${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} + DSTACK_E2E_PLATFORM: ${DSTACK_E2E_PLATFORM:-tdx} + DSTACK_E2E_VMM_PORT: ${DSTACK_E2E_VMM_PORT:-29080} + DSTACK_E2E_AUTH_PORT: ${DSTACK_E2E_AUTH_PORT:-28011} + DSTACK_E2E_ARTIFACT_PORT: ${DSTACK_E2E_ARTIFACT_PORT:-38081} + DSTACK_E2E_KMS_OLD_HOST_PORT: ${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} + DSTACK_E2E_KMS_LATEST_HOST_PORT: ${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} + DSTACK_E2E_KMS_RPC_DOMAIN: ${DSTACK_E2E_KMS_RPC_DOMAIN:-} + DSTACK_E2E_GATEWAY1_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} + DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} + DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} + DSTACK_E2E_GATEWAY1_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} + DSTACK_E2E_GATEWAY2_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} + DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} + DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} + DSTACK_E2E_GATEWAY2_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} + DSTACK_E2E_GATEWAY_APP_ID: ${DSTACK_E2E_GATEWAY_APP_ID:-} + DSTACK_E2E_KEY_PROVIDER_PORT: ${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} + DSTACK_E2E_HOST_API_PORT: ${DSTACK_E2E_HOST_API_PORT:-20011} + DSTACK_E2E_CID_START: ${DSTACK_E2E_CID_START:-15000} + DSTACK_E2E_QGS_PORT: ${DSTACK_E2E_QGS_PORT:-4050} + DSTACK_E2E_QEMU_PATH: ${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} + DSTACK_E2E_BASE_DOMAIN: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test} + DSTACK_E2E_MOCK_CF_HTTP_PORT: ${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} + DSTACK_E2E_PEBBLE_HTTP_PORT: ${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} + DSTACK_E2E_APP_NAME: ${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} + DSTACK_E2E_NAME_PREFIX: ${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + DSTACK_E2E_PHASE: ${DSTACK_E2E_PHASE:-upgrade} + DSTACK_E2E_CURRENT_VERSION: ${DSTACK_E2E_CURRENT_VERSION:-0.6.0} + +services: + init-config: + build: + context: ./runtime + image: dstack-e2e-runtime:local + network_mode: host + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro + environment: + <<: *suite-env + command: ["/suite/scripts/render-config.sh"] + + mock-cf-dns-api: + build: + context: ./mock-cf-dns + image: dstack-e2e-mock-cf-dns:local + networks: + e2e-net: + ipv4_address: 172.31.0.10 + ports: + - "0.0.0.0:${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080}:8080" + environment: + PORT: 8080 + DEBUG: ${DSTACK_E2E_MOCK_DEBUG:-false} + MOCK_CF_ZONES: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test},test,local + MOCK_CF_API_TOKEN: test-token + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] + interval: 5s + timeout: 3s + retries: 10 + + pebble: + # Custom Pebble build used by the existing gateway E2E suite. It supports + # plain HTTP ACME URLs, which keeps the host-side Gateway bootstrap simple. + image: ${DSTACK_E2E_PEBBLE_IMAGE:-kvin/pebble:latest} + command: ["-http", "-dnsserver", "172.31.0.10:53"] + networks: + e2e-net: + ipv4_address: 172.31.0.11 + ports: + - "0.0.0.0:${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000}:14000" + - "0.0.0.0:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000" + environment: + PEBBLE_VA_NOSLEEP: "1" + PEBBLE_VA_ALWAYS_VALID: "1" + depends_on: + mock-cf-dns-api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:14000/dir || curl -fsS http://127.0.0.1:14000/dir >/dev/null"] + interval: 5s + timeout: 3s + retries: 20 + + aesmd: + build: + context: ../../dstack/local-key-provider + dockerfile: build/Dockerfile.aesmd + image: dstack-e2e-aesmd:local + privileged: true + network_mode: host + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + volumes: + - ${DSTACK_E2E_QCNL_CONF:-../../dstack/local-key-provider/build/sgx_default_qcnl.conf}:/etc/sgx_default_qcnl.conf:ro + - aesmd-sock:/var/run/aesmd/ + restart: unless-stopped + + local-keyprovider: + build: + context: ../../dstack + dockerfile: local-key-provider/build/Dockerfile.key-provider + args: + APT_SNAPSHOT: ${APT_SNAPSHOT:-20260423T000000Z} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1776902400} + RUSTUP_VERSION: ${RUSTUP_VERSION:-1.28.2} + RUSTUP_INIT_SHA256: ${RUSTUP_INIT_SHA256:-20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c} + RUST_TOOLCHAIN: ${RUST_TOOLCHAIN:-1.92.0} + image: dstack-e2e-local-keyprovider:local + privileged: true + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + depends_on: + - aesmd + environment: + PCCS_URL: ${PCCS_URL:-https://pccs.phala.network} + LISTEN_ADDR: 0.0.0.0:3443 + volumes: + - aesmd-sock:/var/run/aesmd/ + ports: + - "127.0.0.1:${DSTACK_E2E_KEY_PROVIDER_PORT:-13443}:3443" + healthcheck: + # The provider is a raw TCP service without an HTTP health endpoint. + # Opening the socket proves that the production SGX enclave was loaded; + # the container cannot listen while AESMD/DCAP provisioning is broken. + test: ["CMD-SHELL", "timeout 2 bash -c '- + exec python3 -u -m http.server "${DSTACK_E2E_ARTIFACT_PORT:-38081}" + --bind 0.0.0.0 --directory /suite-state/artifacts + 2> >(tee -a /suite-state/work/artifacts-access.log >&2) + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${DSTACK_E2E_ARTIFACT_PORT:-38081}/"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + vmm: + image: dstack-e2e-runtime:local + privileged: true + pid: host + network_mode: host + devices: + - "/dev/kvm:/dev/kvm" + - "/dev/vhost-vsock:/dev/vhost-vsock" + depends_on: + init-config: + condition: service_completed_successfully + auth: + condition: service_started + artifacts: + condition: service_healthy + local-keyprovider: + condition: service_healthy + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro + - /dev:/dev + - /lib/modules:/lib/modules:ro + environment: + RUST_LOG: ${DSTACK_E2E_VMM_RUST_LOG:-info,dstack_vmm=info} + command: + - bash + - -lc + - exec /workspace/target/release/dstack-vmm -c /suite-state/config/vmm.toml + restart: unless-stopped + + runner: + image: dstack-e2e-runtime:local + network_mode: host + depends_on: + mock-cf-dns-api: + condition: service_healthy + pebble: + condition: service_healthy + auth: + condition: service_started + artifacts: + condition: service_healthy + vmm: + condition: service_started + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ../../dstack:/workspace:ro + environment: + <<: *suite-env + DSTACK_E2E_CLEAN_START: ${DSTACK_E2E_CLEAN_START:-true} + DSTACK_E2E_CLEANUP_AFTER: ${DSTACK_E2E_CLEANUP_AFTER:-false} + DSTACK_E2E_APP_VCPU: ${DSTACK_E2E_APP_VCPU:-2} + DSTACK_E2E_APP_MEMORY: ${DSTACK_E2E_APP_MEMORY:-2048} + command: ["/suite/scripts/runner.sh"] + +networks: + e2e-net: + driver: bridge + ipam: + config: + - subnet: 172.31.0.0/24 + +volumes: + aesmd-sock: diff --git a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile new file mode 100644 index 000000000..8c68f2ec1 --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.12-slim +WORKDIR /app +COPY server.py /app/server.py +EXPOSE 8080/tcp 53/udp +CMD ["python", "/app/server.py"] diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/test-suites/full-stack-compose/mock-cf-dns/server.py new file mode 100755 index 000000000..b3bc07d51 --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/server.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Tiny Cloudflare DNS API + UDP TXT DNS mock for dstack gateway E2E. + +It implements just the Cloudflare endpoints used by certbot's CloudflareClient: + GET /client/v4/zones + GET /client/v4/zones//dns_records?name= + POST /client/v4/zones//dns_records + DELETE /client/v4/zones//dns_records/ + +It also serves TXT answers on UDP/53 so Pebble can validate DNS-01 when the +ACME server is not configured with PEBBLE_VA_ALWAYS_VALID=1. +""" + +from __future__ import annotations + +import json +import os +import re +import socket +import struct +import threading +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +STATE_LOCK = threading.RLock() +RECORDS: list[dict[str, Any]] = [] +NEXT_ID = 1 + + +def _zones() -> list[dict[str, str]]: + raw = os.environ.get("MOCK_CF_ZONES", "e2e.test test local") + names = [ + z.strip().strip(".").lower() for z in re.split(r"[,\s]+", raw) if z.strip() + ] + if not names: + names = ["e2e.test"] + out = [] + seen = set() + for name in names: + if name in seen: + continue + seen.add(name) + out.append({"id": zone_id_for(name), "name": name}) + return out + + +def zone_id_for(name: str) -> str: + """Return a deterministic mock zone ID for a DNS name.""" + safe = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") or "zone" + return f"zone-{safe}" + + +def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None: + data = json.dumps(body, sort_keys=True).encode() + handler.send_response(code) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def _record_content(payload: dict[str, Any]) -> str: + if "content" in payload: + return str(payload.get("content") or "") + data = payload.get("data") or {} + if payload.get("type") == "CAA" and isinstance(data, dict): + return f'{data.get("flags", 0)} {data.get("tag", "issue")} "{data.get("value", "")}"' + return "" + + +def _authorized(handler: BaseHTTPRequestHandler) -> bool: + """Require the configured Cloudflare bearer token for API operations.""" + expected = os.environ.get("MOCK_CF_API_TOKEN", "test-token") + if handler.headers.get("Authorization", "") == f"Bearer {expected}": + return True + _json( + handler, + 403, + {"success": False, "errors": [{"message": "invalid API token"}]}, + ) + return False + + +class Handler(BaseHTTPRequestHandler): + """Handle the mock Cloudflare HTTP API.""" + + server_version = "dstack-mock-cf-dns/0.1" + + def log_message(self, fmt: str, *args: Any) -> None: + """Emit request logs when debug logging is enabled.""" + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + super().log_message(fmt, *args) + + def do_GET(self) -> None: # noqa: N802 + """Handle supported HTTP GET endpoints.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = urllib.parse.parse_qs(parsed.query) + if path == "/health": + _json(self, 200, {"ok": True}) + return + if path == "/api/records": + with STATE_LOCK: + _json(self, 200, {"records": RECORDS}) + return + if path.startswith("/client/v4/") and not _authorized(self): + return + if path == "/client/v4/zones": + zones = _zones() + _json( + self, + 200, + { + "success": True, + "result": zones, + "result_info": { + "page": 1, + "per_page": 50, + "total_pages": 1, + "count": len(zones), + "total_count": len(zones), + }, + }, + ) + return + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if m: + name = (query.get("name") or [""])[0].strip().strip(".").lower() + with STATE_LOCK: + records = [ + r + for r in RECORDS + if not name or r["name"].strip(".").lower() == name + ] + _json( + self, + 200, + { + "success": True, + "result": records, + "result_info": {"page": 1, "per_page": 100, "total_pages": 1}, + }, + ) + return + _json( + self, 404, {"success": False, "errors": [{"message": f"not found: {path}"}]} + ) + + def do_POST(self) -> None: # noqa: N802 + """Create a mock DNS record.""" + global NEXT_ID + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + if not _authorized(self): + return + length = int(self.headers.get("content-length", "0") or "0") + payload = json.loads(self.rfile.read(length) or b"{}") + with STATE_LOCK: + record_id = f"rec-{NEXT_ID}" + NEXT_ID += 1 + record = { + "id": record_id, + "zone_id": m.group(1), + "type": str(payload.get("type") or "TXT").upper(), + "name": str(payload.get("name") or "").strip().strip("."), + "content": _record_content(payload), + "ttl": int(payload.get("ttl") or 60), + "created_on": int(time.time()), + } + RECORDS.append(record) + _json(self, 200, {"success": True, "result": record}) + + def do_DELETE(self) -> None: # noqa: N802 + """Delete a mock DNS record.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + if not _authorized(self): + return + record_id = urllib.parse.unquote(m.group(2)) + with STATE_LOCK: + before = len(RECORDS) + RECORDS[:] = [r for r in RECORDS if r["id"] != record_id] + removed = before != len(RECORDS) + _json( + self, + 200, + {"success": True, "result": {"id": record_id, "removed": removed}}, + ) + + +def parse_qname(packet: bytes, offset: int = 12) -> tuple[str, int]: + """Parse a DNS question name and return its value and end offset.""" + labels: list[str] = [] + while True: + if offset >= len(packet): + raise ValueError("bad qname") + length = packet[offset] + offset += 1 + if length == 0: + break + labels.append(packet[offset : offset + length].decode("ascii", "ignore")) + offset += length + return ".".join(labels).strip(".").lower(), offset + + +def txt_rdata(text: str) -> bytes: + """Encode text as DNS TXT record data.""" + raw = text.encode() + chunks = [raw[i : i + 255] for i in range(0, len(raw), 255)] or [b""] + return b"".join(bytes([len(c)]) + c for c in chunks) + + +def dns_response(packet: bytes) -> bytes: + """Build a DNS response containing matching TXT records.""" + if len(packet) < 12: + return b"" + txid = packet[:2] + qdcount = struct.unpack("!H", packet[4:6])[0] + if qdcount < 1: + return b"" + name, qend = parse_qname(packet) + question = packet[12 : qend + 4] + qtype = ( + struct.unpack("!H", packet[qend : qend + 2])[0] + if qend + 4 <= len(packet) + else 16 + ) + with STATE_LOCK: + answers = [ + r + for r in RECORDS + if r["type"] == "TXT" and r["name"].strip(".").lower() == name + ] + if qtype not in (16, 255): + answers = [] + header = txid + struct.pack("!HHHHH", 0x8180, 1, len(answers), 0, 0) + body = question + for record in answers: + rdata = txt_rdata(record["content"]) + body += ( + b"\xc0\x0c" + + struct.pack("!HHIH", 16, 1, int(record.get("ttl", 60)), len(rdata)) + + rdata + ) + return header + body + + +def dns_loop() -> None: + """Serve mock DNS responses over UDP.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("0.0.0.0", 53)) + while True: + packet, addr = sock.recvfrom(4096) + try: + response = dns_response(packet) + if response: + sock.sendto(response, addr) + except Exception as exc: # pragma: no cover - diagnostic only + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + print(f"dns error from {addr}: {exc}", flush=True) + + +def main() -> None: + """Run the mock Cloudflare API and DNS servers.""" + threading.Thread(target=dns_loop, daemon=True).start() + port = int(os.environ.get("PORT", "8080")) + print(f"mock CF API on :{port}; zones={_zones()}", flush=True) + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/run-upgrade-e2e.sh b/test-suites/full-stack-compose/run-upgrade-e2e.sh new file mode 100755 index 000000000..b0ee43b55 --- /dev/null +++ b/test-suites/full-stack-compose/run-upgrade-e2e.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +REPO_DIR=$(cd -- "$SCRIPT_DIR/../.." && pwd) +STATE_DIR="$SCRIPT_DIR/state" +ENV_FILE=${DSTACK_E2E_ENV_FILE:-$SCRIPT_DIR/.env} + +setting() { + local name=$1 fallback=$2 line value + if [[ -v $name ]]; then + printf '%s' "${!name}" + return + fi + if [[ -f "$ENV_FILE" ]]; then + line=$(grep -E "^[[:space:]]*${name}=" "$ENV_FILE" | tail -n1 || true) + if [[ -n "$line" ]]; then + value=${line#*=} + value=${value%$'\r'} + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value=${value:1:${#value}-2} + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value=${value:1:${#value}-2} + fi + printf '%s' "$value" + return + fi + fi + printf '%s' "$fallback" +} + +OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE \ + dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d) +OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE \ + dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411) +APP_IMAGE=$(setting DSTACK_E2E_APP_IMAGE nginx:alpine) +KEEP_STACK=$(setting DSTACK_E2E_KEEP_STACK true) +CLEAN_STATE=$(setting DSTACK_E2E_UPGRADE_CLEAN_STATE true) +SKIP_BUILD=$(setting DSTACK_E2E_SKIP_CURRENT_BUILD false) +# dstack's build metadata deliberately embeds a 20-hex abbreviated revision. +CURRENT_REV=$(git -C "$REPO_DIR" rev-parse --short=20 HEAD) +CURRENT_VERSION=$(sed -n 's/^version = "\([^"]*\)"/\1/p' \ + "$REPO_DIR/dstack/Cargo.toml" | head -n1) +[[ -n "$CURRENT_VERSION" ]] || { + echo "ERROR: could not read current workspace version" >&2 + exit 1 +} + +COMPOSE=(docker compose -f "$SCRIPT_DIR/compose.yml") +if [[ -f "$ENV_FILE" ]]; then + COMPOSE=(docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR/compose.yml") +fi + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } +compose() { "${COMPOSE[@]}" "$@"; } + +need_bin() { + [[ -x "$1" ]] || die "missing executable $1" +} + +pull_released_image() { + local image=$1 component=$2 + log "pulling released $component image from Docker Hub: $image" + docker pull "$image" || die "cannot pull released $component image $image" +} + +reset_state() { + log "resetting Compose stack and E2E state" + compose down --remove-orphans >/dev/null 2>&1 || true + docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ + 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' +} + +build_current_binaries() { + if [[ "$SKIP_BUILD" == true ]]; then + log "using prebuilt current musl KMS/Gateway binaries" + else + log "building current KMS/Gateway as production-style static musl binaries" + cargo build --manifest-path "$REPO_DIR/dstack/Cargo.toml" \ + --release --target x86_64-unknown-linux-musl \ + -p dstack-kms -p dstack-gateway + fi + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway" +} + +prepare_container_artifacts() { + local artifact_dir="$STATE_DIR/artifacts/images" + local context_dir="$STATE_DIR/image-build" + local rev current_kms_image current_gateway_image + local old_kms_id old_gateway_id current_kms_id current_gateway_id app_id + rev=$(git -C "$REPO_DIR" rev-parse --short=16 HEAD) + current_kms_image="dstack-e2e-kms-current:${rev}" + current_gateway_image="dstack-e2e-gateway-current:${rev}" + mkdir -p "$artifact_dir" "$context_dir/kms" "$context_dir/gateway" + + cp "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" \ + "$context_dir/kms/dstack-kms" + cat > "$context_dir/kms/Dockerfile" < "$context_dir/gateway/Dockerfile" < "$STATE_DIR/artifacts/images.env" </dev/null \ + || die "$OLD_KMS_IMAGE is not KMS 0.5.8" + grep -E '^dstack-gateway v0\.5\.8 \(git:' "$STATE_DIR/work/gateway-old.version.txt" >/dev/null \ + || die "$OLD_GATEWAY_IMAGE is not Gateway 0.5.8" + grep -E "^dstack-kms v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "locally built KMS is not current v$CURRENT_VERSION" + grep -E "^dstack-gateway v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "locally built Gateway is not current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "KMS binary was not built from current revision $CURRENT_REV" + grep -F "$CURRENT_REV" "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "Gateway binary was not built from current revision $CURRENT_REV" +} + +wait_local_key_provider() { + local port deadline status + port=$(setting DSTACK_E2E_KEY_PROVIDER_PORT 13443) + deadline=$((SECONDS + 120)) + log "waiting for production SGX Local-Key-Provider on 127.0.0.1:${port}" + while (( SECONDS < deadline )); do + status=$(compose ps --format json local-keyprovider 2>/dev/null \ + | jq -rs 'map(select(.Service == "local-keyprovider"))[0].Health // ""' 2>/dev/null \ + || true) + if [[ "$status" == healthy ]]; then + log "Local-Key-Provider enclave is healthy" + return 0 + fi + sleep 2 + done + compose logs --tail=200 aesmd local-keyprovider >&2 || true + die "Local-Key-Provider did not become healthy; fix SGX/DCAP/PCCS provisioning rather than disabling attestation" +} + +on_exit() { + local rc=$? + if (( rc != 0 )); then + log "upgrade E2E failed; recent infrastructure logs follow" + compose logs --tail=250 auth artifacts vmm runner >&2 || true + fi + if [[ "$KEEP_STACK" != true ]]; then + compose down --remove-orphans >/dev/null 2>&1 || true + else + log "leaving stack running for inspection (DSTACK_E2E_KEEP_STACK=true)" + fi + exit "$rc" +} +trap on_exit EXIT + +main() { + need_bin "$REPO_DIR/dstack/target/release/dstack" + need_bin "$REPO_DIR/dstack/target/release/dstack-auth" + need_bin "$REPO_DIR/dstack/target/release/dstack-vmm" + need_bin "$REPO_DIR/dstack/target/release/supervisor" + + pull_released_image "$OLD_KMS_IMAGE" kms + pull_released_image "$OLD_GATEWAY_IMAGE" gateway + pull_released_image "$APP_IMAGE" application + build_current_binaries + [[ "$CLEAN_STATE" == true ]] && reset_state + + log "building E2E infrastructure" + compose build init-config mock-cf-dns-api aesmd local-keyprovider + compose run --rm init-config + prepare_container_artifacts + + log "starting authorization, artifact, attestation, ACME and VMM infrastructure" + compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth artifacts + wait_local_key_provider + compose up -d vmm + + log "running production-compatible KMS/Gateway rolling-upgrade E2E" + # Keep the complete runner transcript in state/work even though the runner is + # an ephemeral Compose container. This is especially important for failures + # during deployment, before a per-VM log file exists. + DSTACK_E2E_PHASE=upgrade compose run --rm --no-deps \ + -e DSTACK_E2E_PHASE=upgrade \ + -e DSTACK_E2E_CURRENT_VERSION="$CURRENT_VERSION" \ + -e DSTACK_E2E_CURRENT_REV="$CURRENT_REV" runner \ + 2>&1 | tee "$STATE_DIR/work/runner.log" + + log "upgrade E2E success" + log "artifacts: $STATE_DIR/work" +} + +main "$@" diff --git a/test-suites/full-stack-compose/runtime/Dockerfile b/test-suites/full-stack-compose/runtime/Dockerfile new file mode 100644 index 000000000..3d6fc1ae2 --- /dev/null +++ b/test-suites/full-stack-compose/runtime/Dockerfile @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + software-properties-common \ + && add-apt-repository -y ppa:kobuk-team/tdx-release && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + bash \ + coreutils \ + curl \ + iproute2 \ + iptables \ + iputils-ping \ + jq \ + netcat-openbsd \ + openssl \ + procps \ + python3 \ + python3-venv \ + qemu-system-x86 \ + qemu-utils \ + socat \ + tini \ + wireguard-tools \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +# vmm-cli encrypts deployment environment variables with the KMS key and +# verifies the KMS signature before a production CVM is created. +COPY requirements.txt /tmp/vmm-requirements.txt +RUN python3 -m venv /opt/dstack-venv && \ + /opt/dstack-venv/bin/pip install --no-cache-dir \ + -r /tmp/vmm-requirements.txt && \ + rm /tmp/vmm-requirements.txt +ENV PATH="/opt/dstack-venv/bin:${PATH}" + +WORKDIR /suite +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["bash"] diff --git a/test-suites/full-stack-compose/runtime/requirements.txt b/test-suites/full-stack-compose/runtime/requirements.txt new file mode 100644 index 000000000..4c016c51d --- /dev/null +++ b/test-suites/full-stack-compose/runtime/requirements.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Keep the runtime subset synchronized with dstack/vmm/requirements.txt. +cryptography==46.0.7 +eth-hash[pycryptodome]==0.7.1 +eth-keys==0.7.0 +eth-utils==5.3.0 diff --git a/test-suites/full-stack-compose/scripts/allowlist.py b/test-suites/full-stack-compose/scripts/allowlist.py new file mode 100755 index 000000000..d67865cdf --- /dev/null +++ b/test-suites/full-stack-compose/scripts/allowlist.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Manage the mutable KMS authorization allowlist for the compose E2E suite.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def norm_hex(value: str) -> str: + """Normalize a possibly prefixed hexadecimal string.""" + value = value.strip() + if value.lower().startswith("0x"): + value = value[2:] + return value.lower() + + +def checked_hex(value: str, length: int, label: str) -> str: + """Normalize and validate a fixed-width hexadecimal authorization value.""" + value = norm_hex(value) + if len(value) != length or any(ch not in "0123456789abcdef" for ch in value): + raise ValueError(f"invalid {label}: expected {length} hexadecimal characters") + return value + + +def load(path: Path) -> dict: + """Load an allowlist or return an empty default policy.""" + if path.exists(): + return json.loads(path.read_text()) + return { + "osImages": [], + "gatewayAppId": "", + "kms": {"mrAggregated": [], "devices": [], "allowAnyDevice": False}, + "apps": {}, + } + + +def save(path: Path, data: dict) -> None: + """Atomically write an allowlist to disk.""" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + os.replace(tmp, path) + + +def add_app(args: argparse.Namespace) -> None: + """Add an application and compose hash to the allowlist.""" + path = Path(args.path) + data = load(path) + apps = data.setdefault("apps", {}) + app_id = checked_hex(args.app_id, 40, "app ID") + compose_hash = checked_hex(args.compose_hash, 64, "compose hash") + device_id = checked_hex(args.device_id, 64, "device ID") + entry = apps.setdefault( + app_id, {"composeHashes": [], "devices": [], "allowAnyDevice": False} + ) + hashes = entry.setdefault("composeHashes", []) + if not any(norm_hex(h) == compose_hash for h in hashes): + hashes.append(compose_hash) + devices = entry.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + entry["allowAnyDevice"] = False + if args.gateway_app_id is not None: + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") + save(path, data) + print(json.dumps({"appId": app_id, "composeHash": compose_hash, "path": str(path)})) + + +def set_gateway(args: argparse.Namespace) -> None: + """Set the allowlisted Gateway application ID.""" + path = Path(args.path) + data = load(path) + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") + save(path, data) + print(json.dumps({"gatewayAppId": data["gatewayAppId"], "path": str(path)})) + + +def add_kms(args: argparse.Namespace) -> None: + """Authorize one exact KMS aggregate measurement.""" + path = Path(args.path) + data = load(path) + kms = data.setdefault( + "kms", {"mrAggregated": [], "devices": [], "allowAnyDevice": False} + ) + measurement = checked_hex(args.mr_aggregated, 64, "KMS mrAggregated") + device_id = checked_hex(args.device_id, 64, "device ID") + measurements = kms.setdefault("mrAggregated", []) + if not any(norm_hex(item) == measurement for item in measurements): + measurements.append(measurement) + devices = kms.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + kms["allowAnyDevice"] = False + save(path, data) + print(json.dumps({"mrAggregated": measurement, "path": str(path)})) + + +def main() -> None: + """Parse command-line arguments and update the allowlist.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + p = sub.add_parser("add-app") + p.add_argument("--path", required=True) + p.add_argument("--app-id", required=True) + p.add_argument("--compose-hash", required=True) + p.add_argument("--device-id", required=True) + p.add_argument("--gateway-app-id") + p.set_defaults(func=add_app) + p = sub.add_parser("set-gateway") + p.add_argument("--path", required=True) + p.add_argument("--gateway-app-id", required=True) + p.set_defaults(func=set_gateway) + p = sub.add_parser("add-kms") + p.add_argument("--path", required=True) + p.add_argument("--mr-aggregated", required=True) + p.add_argument("--device-id", required=True) + p.set_defaults(func=add_kms) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py new file mode 100755 index 000000000..6a8cd579a --- /dev/null +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Render measured app-compose manifests used by the full-stack E2E suite.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def image_loader(artifact_port: int, archive: str) -> str: + """Return a pre-launch script that imports one content-addressed image.""" + return f"""set -eu +archive=/tmp/{archive} +curl -fL --retry 5 --retry-delay 1 \\ + http://10.0.2.2:{artifact_port}/images/{archive} -o "$archive" +docker load -i "$archive" +rm -f "$archive" +""" + + +def write_manifest(path: Path, manifest: dict) -> None: + """Write a manifest and print its byte-exact derived identifiers.""" + body = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + path.write_text(body) + digest = hashlib.sha256(body.encode()).hexdigest() + print(json.dumps({"path": str(path), "composeHash": digest, "appId": digest[:40]})) + + +def common_manifest(name: str, docker_compose: str, pre_launch_script: str) -> dict: + """Return the production CVM manifest baseline used by service apps.""" + return { + "docker_compose_file": docker_compose, + "gateway_enabled": False, + "kms_enabled": False, + "local_key_provider_enabled": False, + "manifest_version": 2, + "name": name, + "no_instance_id": True, + "pre_launch_script": pre_launch_script, + "public_logs": True, + "public_sysinfo": True, + "runner": "docker-compose", + "secure_time": True, + "storage_fs": "ext4", + } + + +def kms(args: argparse.Namespace) -> None: + """Render a KMS CVM using the production onboarding and verifier path.""" + config = f"""[rpc] +address = "0.0.0.0" +port = 8000 + +[rpc.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "/kms/certs/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "/kms/certs" +admin_token_hash = "" +pccs_url = "" +enforce_self_authorization = true + +[core.image] +verify = true +cache_dir = "/kms/images" +download_url = "http://10.0.2.2:{args.artifact_port}/os/mr_{{OS_IMAGE_HASH}}.tar.gz" +download_timeout = "2m" + +[core.metrics] +enabled = true + +[core.auth_api] +type = "webhook" + +[core.auth_api.webhook] +url = "http://10.0.2.2:{args.auth_port}" + +[core.onboard] +enabled = true +auto_bootstrap_domain = "" +# Required by KMS 0.5.8. Current KMS ignores this retired field. +quote_enabled = true +address = "0.0.0.0" +port = 8000 +""" + docker_compose = f"""services: + kms: + image: {args.image_ref} + volumes: + - kms-volume:/kms + - /var/run/dstack.sock:/var/run/dstack.sock + ports: + - "8000:8000" + restart: unless-stopped + configs: + - source: kms_config + target: /kms/kms.toml + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' + +volumes: + kms-volume: + +configs: + kms_config: + content: | +""" + "".join(f" {line}\n" for line in config.splitlines()) + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["local_key_provider_enabled"] = True + write_manifest(Path(args.output), manifest) + + +def gateway(args: argparse.Namespace) -> None: + """Render one production Gateway CVM node.""" + docker_compose = f"""services: + gateway: + image: {args.image_ref} + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - /dstack:/dstack + - data:/data + network_mode: host + privileged: true + environment: + - WG_ENDPOINT=${{WG_ENDPOINT}} + - MY_URL=${{MY_URL}} + - BOOTNODE_URL=${{BOOTNODE_URL}} + - WG_IP=${{WG_IP}} + - WG_RESERVED_NET=${{WG_RESERVED_NET}} + - WG_CLIENT_RANGE=${{WG_CLIENT_RANGE}} + - NODE_ID=${{NODE_ID}} + - RUST_LOG=info,certbot=debug + - PCCS_URL=${{PCCS_URL}} + - RPC_DOMAIN=${{RPC_DOMAIN}} + - PROXY_LISTEN_PORT=443 + - PROXY_WORKERS=4 + - SYNC_INTERVAL=1s + - SYNC_TIMEOUT=10s + - SYNC_PERSIST_INTERVAL=1s + - SYNC_CONNECTIONS_ENABLED=true + - SYNC_CONNECTIONS_INTERVAL=1s + - ADMIN_LISTEN_ADDR=0.0.0.0 + - ADMIN_LISTEN_PORT=8001 + - ADMIN_API_TOKEN=${{ADMIN_API_TOKEN}} + restart: always + +volumes: + data: +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["kms_enabled"] = True + manifest["allowed_envs"] = [ + "ADMIN_API_TOKEN", + "BOOTNODE_URL", + "MY_URL", + "NODE_ID", + "PCCS_URL", + "RPC_DOMAIN", + "WG_CLIENT_RANGE", + "WG_ENDPOINT", + "WG_IP", + "WG_RESERVED_NET", + ] + write_manifest(Path(args.output), manifest) + + +def nginx(args: argparse.Namespace) -> None: + """Render a digest-pinned nginx application CVM.""" + docker_compose = f"""services: + web: + image: {args.image_ref} + ports: + - "80:80" +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest.update( + { + "gateway_enabled": True, + "kms_enabled": True, + "no_instance_id": False, + "secure_time": False, + } + ) + if args.attestation_mode != "auto": + # A string v3 manifest makes old guests fail closed instead of silently + # ignoring the requested TDX quote mode. + manifest["manifest_version"] = "3" + manifest["requirements"] = { + "tdx_measure_acpi_tables": args.attestation_mode == "legacy" + } + write_manifest(Path(args.output), manifest) + + +def add_image_args(parser: argparse.ArgumentParser) -> None: + """Add the content-addressed container-image arguments shared by apps.""" + parser.add_argument("--image-ref", required=True) + parser.add_argument("--image-archive", required=True) + parser.add_argument("--artifact-port", required=True, type=int) + parser.add_argument("--name", required=True) + parser.add_argument("--output", required=True) + + +def main() -> None: + """Parse command-line arguments and render a manifest.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + + p = sub.add_parser("kms") + add_image_args(p) + p.add_argument("--auth-port", required=True, type=int) + p.set_defaults(func=kms) + + p = sub.add_parser("gateway") + add_image_args(p) + p.set_defaults(func=gateway) + + p = sub.add_parser("nginx") + add_image_args(p) + p.add_argument( + "--attestation-mode", + choices=("auto", "legacy", "lite"), + default="auto", + ) + p.set_defaults(func=nginx) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/network-probe.sh b/test-suites/full-stack-compose/scripts/network-probe.sh new file mode 100755 index 000000000..9e0387bc7 --- /dev/null +++ b/test-suites/full-stack-compose/scripts/network-probe.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" + +ready="$WORK_DIR/network-probe.ready" +stop="$WORK_DIR/network-probe.stop" +result="$WORK_DIR/network-probe.result.json" +attempts=0 +failures=0 +failovers=0 +cycle=0 +started_ns=$(date +%s%N) + +probe_endpoint() { + local label=$1 instance=$2 port=$3 sni + sni="${instance}-80.${BASE_DOMAIN}" + curl -kfsS --connect-timeout 1 --max-time 2 \ + --connect-to "${sni}:${port}:127.0.0.1:${port}" \ + "https://${sni}:${port}/" 2>/dev/null \ + | grep -qi 'welcome to nginx' +} + +probe_ha() { + local label=$1 instance=$2 first second + if (( cycle % 2 == 0 )); then + first=$GATEWAY1_PROXY_HOST_PORT + second=$GATEWAY2_PROXY_HOST_PORT + else + first=$GATEWAY2_PROXY_HOST_PORT + second=$GATEWAY1_PROXY_HOST_PORT + fi + attempts=$((attempts + 1)) + if probe_endpoint "$label" "$instance" "$first"; then + return 0 + fi + if probe_endpoint "$label" "$instance" "$second"; then + failovers=$((failovers + 1)) + return 0 + fi + failures=$((failures + 1)) + printf '%(%FT%T%z)T route unavailable label=%s preferred=%s fallback=%s\n' \ + -1 "$label" "$first" "$second" >&2 + return 1 +} + +declare -a labels=(legacy lite) +declare -A instances +for label in "${labels[@]}"; do + instances[$label]=$(jq -r '.instance_id // ""' "$WORK_DIR/${label}.info.json") + [[ -n "${instances[$label]}" ]] || { + echo "missing instance id for $label" >&2 + exit 1 + } +done + +# Warm up against both physical Gateway nodes before declaring the HA probe live. +for label in "${labels[@]}"; do + for port in "$GATEWAY1_PROXY_HOST_PORT" "$GATEWAY2_PROXY_HOST_PORT"; do + probe_endpoint "$label" "${instances[$label]}" "$port" || { + echo "warmup failed: $label via Gateway proxy port $port" >&2 + exit 1 + } + done +done +touch "$ready" + +while [[ ! -e "$stop" ]]; do + cycle=$((cycle + 1)) + for label in "${labels[@]}"; do + probe_ha "$label" "${instances[$label]}" || true + done + sleep 0.05 +done + +ended_ns=$(date +%s%N) +duration_ms=$(((ended_ns - started_ns) / 1000000)) +jq -n \ + --argjson duration_ms "$duration_ms" \ + --argjson attempts "$attempts" \ + --argjson failures "$failures" \ + --argjson failovers "$failovers" \ + '{duration_ms:$duration_ms,attempts:$attempts,failures:$failures,successful_failovers:$failovers}' \ + | tee "$result" + +(( failures == 0 )) diff --git a/test-suites/full-stack-compose/scripts/render-config.sh b/test-suites/full-stack-compose/scripts/render-config.sh new file mode 100755 index 000000000..500b61c4b --- /dev/null +++ b/test-suites/full-stack-compose/scripts/render-config.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +RUN_DIR=${DSTACK_E2E_RUN_DIR:-$STATE_DIR/run} +VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} +VOLUMES_DIR=${DSTACK_E2E_VOLUMES_DIR:-$STATE_DIR/volumes} +ARTIFACT_DIR=${DSTACK_E2E_ARTIFACT_DIR:-$STATE_DIR/artifacts} +IMAGE_ROOT=${DSTACK_E2E_IMAGE_ROOT:-/images} +IMAGE_NAME=${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} +PLATFORM=${DSTACK_E2E_PLATFORM:-tdx} + +VMM_PORT=${DSTACK_E2E_VMM_PORT:-29080} +AUTH_PORT=${DSTACK_E2E_AUTH_PORT:-28011} +ARTIFACT_PORT=${DSTACK_E2E_ARTIFACT_PORT:-38081} +KMS_OLD_HOST_PORT=${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} +KMS_LATEST_HOST_PORT=${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} +KMS_RPC_DOMAIN=${DSTACK_E2E_KMS_RPC_DOMAIN:-} +GATEWAY1_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} +GATEWAY1_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} +GATEWAY1_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} +GATEWAY1_WG_HOST_PORT=${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} +GATEWAY2_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} +GATEWAY2_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} +GATEWAY2_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} +GATEWAY2_WG_HOST_PORT=${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} +GATEWAY_APP_ID=${DSTACK_E2E_GATEWAY_APP_ID:-$(printf dstack-e2e-gateway | sha256sum | cut -c1-40)} +KEY_PROVIDER_PORT=${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} +HOST_API_PORT=${DSTACK_E2E_HOST_API_PORT:-20011} +CID_START=${DSTACK_E2E_CID_START:-15000} +QGS_PORT=${DSTACK_E2E_QGS_PORT:-4050} +QEMU_PATH=${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} +BASE_DOMAIN=${DSTACK_E2E_BASE_DOMAIN:-e2e.test} +MOCK_CF_HTTP_PORT=${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} +PEBBLE_HTTP_PORT=${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} +APP_NAME=${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} +SUITE_PREFIX=${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + +mkdir -p \ + "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" \ + "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/os" "$ARTIFACT_DIR/control" \ + "$STATE_DIR/work" +chmod 0777 "$STATE_DIR/work" "$ARTIFACT_DIR" "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/control" + +if [[ ! "$GATEWAY_APP_ID" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: DSTACK_E2E_GATEWAY_APP_ID must be an exact 20-byte hex app id" >&2 + exit 1 +fi +GATEWAY_APP_ID=${GATEWAY_APP_ID,,} + +if [[ -z "$KMS_RPC_DOMAIN" ]]; then + host_ip=$(ip -4 route get 1.1.1.1 2>/dev/null \ + | sed -n 's/.* src \([^ ]*\).*/\1/p' | head -n1) + if [[ ! "$host_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: unable to derive a host IPv4 address; set DSTACK_E2E_KMS_RPC_DOMAIN" >&2 + exit 1 + fi + KMS_RPC_DOMAIN="${host_ip}.nip.io" +fi +if ! getent ahostsv4 "$KMS_RPC_DOMAIN" >/dev/null 2>&1; then + echo "ERROR: DSTACK_E2E_KMS_RPC_DOMAIN does not resolve: $KMS_RPC_DOMAIN" >&2 + exit 1 +fi + +image_dir="$IMAGE_ROOT/$IMAGE_NAME" +if [[ ! -d "$image_dir" ]]; then + echo "ERROR: image directory not found: $image_dir" >&2 + echo "Set DSTACK_E2E_IMAGE_STORE to a host directory containing $IMAGE_NAME/" >&2 + exit 1 +fi + +digest_file=digest.txt +if [[ "$PLATFORM" == "amd-sev-snp" || "$PLATFORM" == "sev-snp" || "$PLATFORM" == "snp" ]]; then + PLATFORM=amd-sev-snp + digest_file=digest.sev.txt +elif [[ "$PLATFORM" != tdx ]]; then + echo "ERROR: unsupported DSTACK_E2E_PLATFORM=$PLATFORM (expected tdx or amd-sev-snp)" >&2 + exit 1 +fi + +if [[ ! -s "$image_dir/$digest_file" ]]; then + echo "ERROR: missing $image_dir/$digest_file; production-compatible E2E never permits an unpinned OS image" >&2 + exit 1 +fi +OS_IMAGE_HASH=$(tr -d '[:space:]' < "$image_dir/$digest_file") +if [[ ! "$OS_IMAGE_HASH" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: invalid OS image digest: $OS_IMAGE_HASH" >&2 + exit 1 +fi + +# Build the exact flat archive consumed by the KMS verifier. Verify the source +# first and include only the measured files named by sha256sum.txt. +( + cd "$image_dir" + sha256sum -c sha256sum.txt + mapfile -t measured_files < <(awk '{print $2}' sha256sum.txt) + for file in "${measured_files[@]}"; do + [[ "$file" != */* && -f "$file" ]] || { + echo "ERROR: unsafe or missing measured image file: $file" >&2 + exit 1 + } + done + tar -czf "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + sha256sum.txt "${measured_files[@]}" +) +mv "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz" + +token_file="$STATE_DIR/gateway-admin-token" +if [[ ! -s "$token_file" ]]; then + (umask 077; openssl rand -hex 24 > "$token_file") +fi +GATEWAY_ADMIN_TOKEN=$(tr -d '[:space:]' < "$token_file") + +cat > "$CONFIG_DIR/auth-allowlist.json" < "$CONFIG_DIR/vmm.toml" < "$STATE_DIR/state.env" < +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} +PHASE=${DSTACK_E2E_PHASE:-upgrade} +CURRENT_VERSION=${DSTACK_E2E_CURRENT_VERSION:?missing current workspace version} +CURRENT_REV=${DSTACK_E2E_CURRENT_REV:?missing current workspace revision} +mkdir -p "$WORK_DIR" +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" +# shellcheck disable=SC1091 +source "$STATE_DIR/artifacts/images.env" + +VMM_URL="http://127.0.0.1:${VMM_PORT}" +VMM_CLI=(python3 /workspace/vmm/src/vmm-cli.py --url "$VMM_URL") +DSTACK_CLI=(/workspace/target/release/dstack --host "$VMM_URL") +ALLOWLIST="$CONFIG_DIR/auth-allowlist.json" +KMS_OLD_URL="https://${KMS_RPC_DOMAIN}:${KMS_OLD_HOST_PORT}" +KMS_LATEST_URL="https://${KMS_RPC_DOMAIN}:${KMS_LATEST_HOST_PORT}" +GATEWAY1_URL="https://${KMS_RPC_DOMAIN}:${GATEWAY1_RPC_HOST_PORT}" +GATEWAY2_URL="https://${KMS_RPC_DOMAIN}:${GATEWAY2_RPC_HOST_PORT}" +probe_pid="" +ATTESTED_DEVICE_ID="" + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +need_bin() { + [[ -x "$1" ]] || die "missing executable $1" +} + +wait_vmm() { + log "waiting for VMM at $VMM_URL" + for _ in $(seq 1 120); do + if "${DSTACK_CLI[@]}" apps -j >/dev/null 2>&1; then + log "VMM ready" + return + fi + sleep 2 + done + die "VMM not ready" +} + +vm_ids_by_prefix() { + local prefix=$1 + "${VMM_CLI[@]}" lsvm --json 2>/dev/null \ + | jq -r --arg p "$prefix" '.[] | select(.name | startswith($p)) | .id' +} + +remove_vm() { + local id=$1 + [[ -n "$id" ]] || return 0 + log "removing VM $id" + "${VMM_CLI[@]}" remove "$id" >/dev/null 2>&1 || true +} + +clean_start() { + [[ "${DSTACK_E2E_CLEAN_START:-true}" == true ]] || return 0 + while read -r id; do + remove_vm "$id" + done < <(vm_ids_by_prefix "$SUITE_PREFIX") + for _ in $(seq 1 90); do + [[ -z "$(vm_ids_by_prefix "$SUITE_PREFIX")" ]] && return 0 + sleep 2 + done + die "stale VMs were not removed" +} + +print_vm_info_safe() { + jq '{id,name,status,uptime,app_id,instance_id,boot_progress,boot_error,image_version,events}' <<<"$1" +} + +wait_boot_done() { + local id=$1 label=$2 timeout=${3:-720} + local deadline=$((SECONDS + timeout)) last="" + while (( SECONDS < deadline )); do + local info status progress error line + info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) + if [[ -n "$info" ]]; then + status=$(jq -r '.status // ""' <<<"$info") + progress=$(jq -r '.boot_progress // ""' <<<"$info") + error=$(jq -r '.boot_error // ""' <<<"$info") + line="status=$status progress=${progress:-none} error=${error:-none}" + if [[ "$line" != "$last" ]]; then + log "$label: $line" + last=$line + fi + if [[ -n "$error" && "$error" != null ]]; then + print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true + die "$label boot failed: $error" + fi + if [[ "$status" == running && "$progress" == "done" ]]; then + printf '%s' "$info" > "$WORK_DIR/${label}.info.json" + return + fi + if [[ "$status" == exited || "$status" == stopped ]]; then + print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true + die "$label exited before boot finished" + fi + fi + sleep 5 + done + "${VMM_CLI[@]}" logs "$id" -n 300 >&2 || true + die "timed out waiting for $label" +} + +wait_vm_stopped() { + local id=$1 deadline=$((SECONDS + 180)) status + while (( SECONDS < deadline )); do + status=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null | jq -r '.status // ""' || true) + [[ "$status" != running && "$status" != starting ]] && return + sleep 2 + done + die "VM $id did not stop" +} + +stop_vm() { + local id=$1 label=$2 + log "stopping $label ($id)" + "${VMM_CLI[@]}" stop "$id" --force >/dev/null + wait_vm_stopped "$id" +} + +start_vm() { + local id=$1 label=$2 + log "starting $label ($id)" + "${VMM_CLI[@]}" start "$id" >/dev/null + wait_boot_done "$id" "$label" +} + +register_app() { + local app_id=$1 hash=$2 + [[ -n "$ATTESTED_DEVICE_ID" ]] || die "cannot authorize app before pinning the attested TDX device" + /suite/scripts/allowlist.py add-app --path "$ALLOWLIST" \ + --app-id "$app_id" --compose-hash "$hash" \ + --device-id "$ATTESTED_DEVICE_ID" >/dev/null +} + +register_kms_measurement() { + local measurement=$1 device_id=$2 + /suite/scripts/allowlist.py add-kms --path "$ALLOWLIST" \ + --mr-aggregated "$measurement" --device-id "$device_id" >/dev/null +} + +render_kms() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py kms \ + --name "dstack-e2e-kms-${stage}" \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --auth-port "$AUTH_PORT" \ + --output "$WORK_DIR/kms-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/kms-${stage}.manifest-meta.json" +} + +deploy_kms_onboard() { + local stage=$1 port=$2 out vm_id + log "deploying $stage KMS CVM in Local-Key-Provider mode" + if ! out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-kms-${stage}" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/kms-${stage}.app-compose.json" \ + --port "tcp:0.0.0.0:${port}:8000" \ + --vcpu "${DSTACK_E2E_KMS_VCPU:-4}" \ + --memory "${DSTACK_E2E_KMS_MEMORY:-4096}" \ + --disk "${DSTACK_E2E_KMS_DISK:-30}" 2>&1); then + die "failed to deploy $stage KMS CVM: $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/kms-${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse $stage KMS VM id" + printf '%s' "$vm_id" > "$WORK_DIR/kms-${stage}.vm_id" + wait_boot_done "$vm_id" "kms-${stage}" +} + +wait_onboard() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) url + url="http://127.0.0.1:${port}/prpc/Onboard.GetAttestationInfo?json" + log "waiting for $stage KMS quote-enabled onboarding endpoint" + while (( SECONDS < deadline )); do + if curl -fsS "$url" -o "$WORK_DIR/kms-${stage}.attestation-info.json"; then + jq -e '((.mr_aggregated // .mrAggregated // "") | length) > 0 and ((.device_id // .deviceId // "") | length) > 0' \ + "$WORK_DIR/kms-${stage}.attestation-info.json" >/dev/null && return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS onboarding endpoint not ready" +} + +authorize_kms_from_attestation() { + local stage=$1 measurement device_id + measurement=$(jq -r '.mr_aggregated // .mrAggregated' "$WORK_DIR/kms-${stage}.attestation-info.json") + device_id=$(jq -r '.device_id // .deviceId' "$WORK_DIR/kms-${stage}.attestation-info.json") + [[ "$measurement" =~ ^[0-9a-fA-F]+$ ]] || die "invalid $stage KMS mrAggregated" + [[ "$device_id" =~ ^[0-9a-fA-F]{64}$ ]] || die "invalid $stage KMS device ID" + if [[ -n "$ATTESTED_DEVICE_ID" && "$device_id" != "$ATTESTED_DEVICE_ID" ]]; then + die "$stage KMS quote came from unexpected TDX device $device_id" + fi + ATTESTED_DEVICE_ID=$device_id + register_kms_measurement "$measurement" "$device_id" + log "authorized exact $stage KMS mrAggregated=${measurement} on device=${device_id}" +} + +onboard_rpc() { + local port=$1 method=$2 data=$3 out=$4 + curl -fsS -X POST -H 'Content-Type: application/json' \ + "http://127.0.0.1:${port}/prpc/Onboard.${method}?json" \ + --data-raw "$data" -o "$out" + jq -e 'has("error") | not' "$out" >/dev/null \ + || die "Onboard.${method} returned an error: $(cat "$out")" + log "Onboard.${method} completed" +} + +finish_onboarding() { + local stage=$1 port=$2 + curl -fsS "http://127.0.0.1:${port}/finish" > "$WORK_DIR/kms-${stage}.finish.txt" + wait_kms_tls "$stage" "$port" +} + +wait_kms_tls() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) + log "waiting for $stage KMS TLS service" + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${port}/prpc/GetMeta?json" \ + -o "$WORK_DIR/kms-${stage}.ready-meta.json"; then + log "$stage KMS TLS service ready" + return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS TLS service not ready" +} + +kms_rpc_get() { + local port=$1 method=$2 + curl -kfsS "https://127.0.0.1:${port}/prpc/${method}?json" +} + +kms_rpc_post() { + local port=$1 method=$2 data=$3 + curl -kfsS -X POST -H 'Content-Type: application/json' \ + "https://127.0.0.1:${port}/prpc/${method}?json" --data-raw "$data" +} + +capture_kms_identity() { + local stage=$1 port=$2 app_id=$3 ca_pem ca_spki k256_pubkey + ca_pem="$WORK_DIR/kms-${stage}.ca.pem" + kms_rpc_get "$port" GetMeta | tee "$WORK_DIR/kms-${stage}.meta.raw.json" \ + | jq -S '{ca_cert, k256_pubkey}' > "$WORK_DIR/kms-${stage}.meta.json" + jq -er '.ca_cert' "$WORK_DIR/kms-${stage}.meta.json" > "$ca_pem" + openssl x509 -in "$ca_pem" -noout -checkend 0 >/dev/null + ca_spki=$(openssl x509 -in "$ca_pem" -pubkey -noout \ + | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 \ + | awk '{print $NF}') + k256_pubkey=$(jq -er '.k256_pubkey' "$WORK_DIR/kms-${stage}.meta.json") + jq -nS --arg ca_spki_sha256 "$ca_spki" --arg k256_pubkey "$k256_pubkey" \ + '{ca_spki_sha256:$ca_spki_sha256,k256_pubkey:$k256_pubkey}' \ + > "$WORK_DIR/kms-${stage}.identity.json" + kms_rpc_post "$port" GetAppEnvEncryptPubKey \ + "$(jq -cn --arg id "$app_id" '{app_id:$id}')" \ + | tee "$WORK_DIR/kms-${stage}.app-key.raw.json" \ + | jq -S '{public_key}' > "$WORK_DIR/kms-${stage}.app-key.json" + jq -e '.ca_spki_sha256 != "" and .k256_pubkey != ""' \ + "$WORK_DIR/kms-${stage}.identity.json" >/dev/null + jq -e '.public_key != ""' "$WORK_DIR/kms-${stage}.app-key.json" >/dev/null +} + +assert_kms_identity_unchanged() { + # Onboarding deliberately reissues the self-signed CA certificate with a new + # validity end time. The transferable identity is its private key/SPKI, not + # the DER bytes of that freshly issued certificate. + diff -u "$WORK_DIR/kms-old.identity.json" "$WORK_DIR/kms-latest.identity.json" \ + || die "KMS CA key or root k256 key changed during 0.5.8 -> current onboarding" + diff -u "$WORK_DIR/kms-old.app-key.json" "$WORK_DIR/kms-latest.app-key.json" \ + || die "per-app environment encryption key changed during KMS upgrade" + log "KMS CA SPKI, root k256 key and per-app environment key are unchanged" +} + +render_gateway_manifests() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py gateway \ + --name dstack-e2e-gateway \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --output "$WORK_DIR/gateway-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/gateway-${stage}.manifest-meta.json" + register_app "$GATEWAY_APP_ID" "$(jq -r .composeHash <<<"$meta")" +} + +write_gateway_env() { + local node=$1 rpc_port wg_port third_octet bootnode + if [[ "$node" == 1 ]]; then + rpc_port=$GATEWAY1_RPC_HOST_PORT + wg_port=$GATEWAY1_WG_HOST_PORT + third_octet=0 + bootnode="" + else + rpc_port=$GATEWAY2_RPC_HOST_PORT + wg_port=$GATEWAY2_WG_HOST_PORT + third_octet=64 + bootnode="$GATEWAY1_URL" + fi + cat > "$WORK_DIR/gateway${node}.env" <&1); then + die "failed to deploy Gateway node $node ($stage): $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/gateway${node}.${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse Gateway node $node VM id" + printf '%s' "$vm_id" > "$WORK_DIR/gateway${node}.vm_id" + wait_boot_done "$vm_id" "gateway${node}-${stage}" + wait_gateway "$node" +} + +gateway_version() { + local node=$1 stage=$2 rpc _ + read -r rpc _ < <(gateway_ports "$node") + curl -kfsS -D "$WORK_DIR/gateway${node}-${stage}.headers" -o /dev/null \ + "https://127.0.0.1:${rpc}/prpc/Info?json" + tr -d '\r' < "$WORK_DIR/gateway${node}-${stage}.headers" \ + | awk 'tolower($1)=="x-app-version:" {$1=""; sub(/^ /,""); print}' \ + > "$WORK_DIR/gateway${node}-${stage}.version.txt" + [[ -s "$WORK_DIR/gateway${node}-${stage}.version.txt" ]] \ + || die "Gateway node $node did not return X-App-Version" + if [[ "$stage" == old ]]; then + grep -E '^v0\.5\.8 \(git:' "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run released v0.5.8" + else + grep -E "^v${CURRENT_VERSION//./\\.} \\(git:" \ + "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current revision $CURRENT_REV" + fi + log "Gateway node $node $stage: $(cat "$WORK_DIR/gateway${node}-${stage}.version.txt")" +} + +admin_curl() { + local node=$1 method=$2 data=${3:-'{}'} admin out code + read -r _ admin _ < <(gateway_ports "$node") + out=$(mktemp) + code=$(curl -sS -o "$out" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ + -H 'Content-Type: application/json' \ + "http://127.0.0.1:${admin}/prpc/Admin.${method}?json" \ + --data-raw "$data" || true) + if [[ "$code" =~ ^2 ]]; then + cat "$out" + rm -f "$out" + return + fi + log "Gateway $node Admin.${method} failed HTTP $code: $(cat "$out")" >&2 + rm -f "$out" + return 1 +} + +wait_gateway() { + local node=$1 rpc deadline=$((SECONDS + 300)) + read -r rpc _ < <(gateway_ports "$node") + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${rpc}/prpc/Info?json" >/dev/null 2>&1 \ + && admin_curl "$node" Status >/dev/null 2>&1; then + log "Gateway node $node ready" + return + fi + sleep 3 + done + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/gateway${node}.vm_id")" -n 300 >&2 || true + die "Gateway node $node not ready" +} + +bootstrap_gateway() { + log "configuring Gateway cluster through node 1" + admin_curl 1 SetCertbotConfig \ + "$(jq -cn --arg u "http://10.0.2.2:${PEBBLE_HTTP_PORT}/dir" \ + '{acme_url:$u, renew_before_expiration_secs:3600}')" >/dev/null + admin_curl 1 CreateDnsCredential \ + "$(jq -cn --arg u "http://10.0.2.2:${MOCK_CF_HTTP_PORT}/client/v4" \ + '{name:"mock-cloudflare",provider_type:"cloudflare",cf_api_token:"test-token",cf_api_url:$u,set_as_default:true,dns_txt_ttl:1,max_dns_wait:0}')" \ + >/dev/null + admin_curl 1 AddZtDomain \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,port:443,priority:100}')" \ + >/dev/null + admin_curl 1 RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')" \ + | tee "$WORK_DIR/gateway-renew-cert.json" + wait_gateway_cert 1 + wait_gateway_cluster_sync + wait_gateway_cert 2 +} + +wait_gateway_cluster_sync() { + local deadline=$((SECONDS + 240)) domains certbot + log "waiting for Gateway WaveKV config/certificate sync to node 2" + while (( SECONDS < deadline )); do + domains=$(admin_curl 2 ListZtDomains 2>/dev/null || true) + certbot=$(admin_curl 2 GetCertbotConfig 2>/dev/null || true) + if jq -e --arg d "$BASE_DOMAIN" '.domains[]? | (.domain // .config.domain) == $d' \ + <<<"$domains" >/dev/null 2>&1 \ + && jq -e '.acme_url != ""' <<<"$certbot" >/dev/null 2>&1; then + log "Gateway cluster state synced" + return + fi + sleep 3 + done + die "Gateway node 2 did not receive synced cluster state" +} + +wait_gateway_cert() { + local node=$1 proxy deadline=$((SECONDS + 300)) sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") + while (( SECONDS < deadline )); do + if echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -ext subjectAltName 2>/dev/null \ + | grep -Fq "*.${BASE_DOMAIN}"; then + log "Gateway node $node wildcard certificate active" + return + fi + sleep 3 + done + die "Gateway node $node wildcard certificate not active" +} + +gateway_cert_fingerprint() { + local node=$1 proxy sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") + echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -fingerprint -sha256 2>/dev/null +} + +wait_gateway_persisted() { + local node=$1 deadline=$((SECONDS + 120)) status + while (( SECONDS < deadline )); do + status=$(admin_curl "$node" WaveKvStatus 2>/dev/null || true) + if jq -e '.persistent.dirty == false and .persistent.n_keys > 0' <<<"$status" >/dev/null 2>&1; then + return + fi + sleep 2 + done + die "Gateway node $node persistent store did not flush" +} + +capture_gateway_state() { + local node=$1 stage=$2 + wait_gateway_persisted "$node" + admin_curl "$node" Status | tee "$WORK_DIR/gateway${node}-${stage}.status.raw.json" \ + | jq -S '. as $status | { + id: $status.id, + uuid: ([$status.nodes[] | select(.id == $status.id)][0].uuid), + hosts: ([$status.hosts[] | {instance_id,ip,app_id,base_domain}] | sort_by(.instance_id)) + }' \ + > "$WORK_DIR/gateway${node}-${stage}.status.json" + admin_curl "$node" GetCertbotConfig | jq -S \ + '{acme_url,renew_interval_secs,renew_before_expiration_secs,renew_timeout_secs}' \ + > "$WORK_DIR/gateway${node}-${stage}.certbot.json" + admin_curl "$node" ListDnsCredentials | jq -S \ + '{default_id,credentials:([.credentials[]|{ + id,name,provider_type, + cf_api_token_set: ((.cf_api_token // "") | length > 0), + cf_zone_id,cf_api_url,dns_txt_ttl,max_dns_wait + }]|sort_by(.id))}' \ + > "$WORK_DIR/gateway${node}-${stage}.dns.json" + admin_curl "$node" ListZtDomains | jq -S \ + '{domains:([.domains[]|(.config // .)]|sort_by(.domain))}' \ + > "$WORK_DIR/gateway${node}-${stage}.domains.json" + gateway_cert_fingerprint "$node" > "$WORK_DIR/gateway${node}-${stage}.cert.txt" +} + +assert_gateway_state_unchanged() { + local node=$1 file + capture_gateway_state "$node" after + for file in status certbot dns domains; do + diff -u "$WORK_DIR/gateway${node}-before.${file}.json" \ + "$WORK_DIR/gateway${node}-after.${file}.json" \ + || die "Gateway node $node $file state changed across upgrade" + done + diff -u "$WORK_DIR/gateway${node}-before.cert.txt" "$WORK_DIR/gateway${node}-after.cert.txt" \ + || die "Gateway node $node wildcard certificate changed across upgrade" + log "Gateway node $node durable state is unchanged" +} + +assert_gateway_dns_credential_usable() { + local node=$1 result deadline=$((SECONDS + 90)) + log "proving Gateway node $node retained the Cloudflare credential" + while (( SECONDS < deadline )); do + result=$(admin_curl "$node" RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')") + printf '%s\n' "$result" > "$WORK_DIR/gateway${node}-post-upgrade-renew.json" + if jq -e '.renewed == true and .not_after > 0' <<<"$result" >/dev/null; then + wait_gateway_cert "$node" + return + fi + # WaveKV's distributed certificate lock may still be held by the peer's + # immediately preceding issuance. + sleep 3 + done + die "Gateway node $node could not issue with its persisted DNS credential" +} + +render_app() { + local label=$1 mode=$2 meta app_id hash + meta=$(/suite/scripts/app_compose.py nginx \ + --name "${APP_NAME}-${label}" \ + --image-ref "$APP_IMAGE_ID" \ + --image-archive app.tar \ + --artifact-port "$ARTIFACT_PORT" \ + --attestation-mode "$mode" \ + --output "$WORK_DIR/${label}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/${label}.manifest-meta.json" + app_id=$(jq -r .appId <<<"$meta") + hash=$(jq -r .composeHash <<<"$meta") + printf '%s' "$app_id" > "$WORK_DIR/${label}.app_id" + register_app "$app_id" "$hash" +} + +deploy_app() { + local label=$1 mode=$2 kms_url=$3 out vm_id + render_app "$label" "$mode" + log "deploying $label app CVM (forced TDX $mode)" + if ! out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-${label}" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/${label}.app-compose.json" \ + --kms-url "$kms_url" \ + --gateway-url "$GATEWAY1_URL" \ + --gateway-url "$GATEWAY2_URL" \ + --vcpu "${DSTACK_E2E_APP_VCPU:-2}" \ + --memory "${DSTACK_E2E_APP_MEMORY:-2048}" \ + --disk "${DSTACK_E2E_APP_DISK:-20}" 2>&1); then + die "failed to deploy $label app CVM: $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/${label}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse $label VM id" + printf '%s' "$vm_id" > "$WORK_DIR/${label}.vm_id" + wait_boot_done "$vm_id" "$label" + assert_attestation_mode "$label" "$mode" +} + +assert_attestation_mode() { + local label=$1 expected=$2 id sys_config actual + id=$(cat "$WORK_DIR/${label}.vm_id") + sys_config="$VM_DIR/$id/shared/.sys-config.json" + [[ -s "$sys_config" ]] || die "missing $label sys config" + actual=$(jq -r '.vm_config | fromjson | .tdx_attestation_variant // "legacy"' "$sys_config") + [[ "$actual" == "$expected" ]] || die "$label mode=$actual, expected $expected" + jq -e '(.kms_urls | length) > 0 and (.gateway_urls | length) == 2' "$sys_config" >/dev/null + log "$label completed boot-time KMS key provisioning in TDX $actual mode" +} + +verify_app_via_gateway() { + local label=$1 node=$2 instance sni proxy url deadline + instance=$(jq -r '.instance_id' "$WORK_DIR/${label}.info.json") + read -r _ _ proxy _ < <(gateway_ports "$node") + sni="${instance}-80.${BASE_DOMAIN}" + url="https://${sni}:${proxy}/" + deadline=$((SECONDS + 300)) + while (( SECONDS < deadline )); do + if curl -kfsS --max-time 5 \ + --connect-to "${sni}:${proxy}:127.0.0.1:${proxy}" "$url" \ + | grep -qi 'welcome to nginx'; then + log "$label reachable through Gateway node $node" + return + fi + sleep 3 + done + die "$label not reachable through Gateway node $node" +} + +verify_all_routes() { + local label node + for label in legacy lite; do + for node in 1 2; do + verify_app_via_gateway "$label" "$node" + done + done +} + +restart_legacy_on_latest_kms() { + local id + id=$(cat "$WORK_DIR/legacy.vm_id") + stop_vm "$id" legacy + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" legacy + assert_attestation_mode legacy legacy +} + +upgrade_gateway_node() { + local node=$1 id + id=$(cat "$WORK_DIR/gateway${node}.vm_id") + stop_vm "$id" "Gateway node $node" + "${VMM_CLI[@]}" update-app-compose "$id" "$WORK_DIR/gateway-latest.app-compose.json" >/dev/null + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" "gateway${node}-latest" + wait_gateway "$node" + wait_gateway_cert "$node" + gateway_version "$node" latest + # The newly restarted node must carry traffic before its peer is touched. + verify_app_via_gateway legacy "$node" + verify_app_via_gateway lite "$node" + assert_gateway_state_unchanged "$node" +} + +start_network_probe() { + rm -f "$WORK_DIR/network-probe."{ready,stop,result.json,log} + /suite/scripts/network-probe.sh > "$WORK_DIR/network-probe.log" 2>&1 & + probe_pid=$! + for _ in $(seq 1 120); do + [[ -e "$WORK_DIR/network-probe.ready" ]] && return + kill -0 "$probe_pid" 2>/dev/null || { + cat "$WORK_DIR/network-probe.log" >&2 + die "HA network probe exited during warmup" + } + sleep 1 + done + die "HA network probe did not become ready" +} + +stop_network_probe() { + touch "$WORK_DIR/network-probe.stop" + if ! wait "$probe_pid"; then + cat "$WORK_DIR/network-probe.log" >&2 + die "CVM traffic had downtime during rolling Gateway upgrade" + fi + probe_pid="" + cat "$WORK_DIR/network-probe.log" + jq -e '.attempts > 0 and .failures == 0 and .successful_failovers > 0' \ + "$WORK_DIR/network-probe.result.json" >/dev/null \ + || die "HA probe reported downtime" +} + +assert_no_insecure_shortcuts() { + log "auditing rendered manifests for production trust settings" + local manifest + local forbidden='quote_enabled[[:space:]]*=[[:space:]]*false|enforce_self_authorization[[:space:]]*=[[:space:]]*false|verify[[:space:]]*=[[:space:]]*false' + if grep -R -E "$forbidden" "$WORK_DIR"/*.app-compose.json; then + die "rendered app manifest contains a forbidden development trust setting" + fi + + for manifest in "$WORK_DIR/kms-old.app-compose.json" \ + "$WORK_DIR/kms-latest.app-compose.json"; do + if ! python3 - "$manifest" <<'PY' +import json +import sys +import tomllib + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +assert manifest["local_key_provider_enabled"] is True +assert manifest["kms_enabled"] is False + +# Parse the actual kms.toml embedded in the rendered Compose YAML. Checking +# TOML values instead of matching adjacent strings keeps this guard strict +# while allowing comments and blank lines in the production-style config. +marker = "configs:\n kms_config:\n content: |\n" +compose = manifest["docker_compose_file"] +assert compose.count(marker) == 1 +indented_config = compose.split(marker, 1)[1] +config_lines = [] +for line in indented_config.splitlines(): + if line and not line.startswith(" "): + break + config_lines.append(line[6:] if line else "") +config = tomllib.loads("\n".join(config_lines)) + +core = config["core"] +assert core["enforce_self_authorization"] is True +assert core["image"]["verify"] is True +assert core["auth_api"]["type"] == "webhook" +assert core["onboard"]["enabled"] is True +assert core["onboard"]["quote_enabled"] is True +PY + then + die "$manifest does not use quote-attested production KMS settings" + fi + done + + for manifest in "$WORK_DIR/gateway-old.app-compose.json" \ + "$WORK_DIR/gateway-latest.app-compose.json"; do + jq -e '.kms_enabled == true and .local_key_provider_enabled == false' \ + "$manifest" >/dev/null \ + || die "$manifest does not obtain its keys from KMS" + done + + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == true + ' "$WORK_DIR/legacy.app-compose.json" >/dev/null \ + || die "legacy app manifest did not force the legacy TDX verifier" + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == false + ' "$WORK_DIR/lite.app-compose.json" >/dev/null \ + || die "lite app manifest did not force the lite TDX verifier" + + jq -e --arg id "$GATEWAY_APP_ID" \ + --arg device "$ATTESTED_DEVICE_ID" \ + '.gatewayAppId == $id and .gatewayAppId != "any" + and (.osImages | length) == 1 + and (.kms.mrAggregated | length) == 2 + and (.kms.allowAnyDevice == false) + and (.kms.devices == [$device]) + and (.apps | length) == 3 + and ([.apps[].allowAnyDevice] | all(. == false)) + and ([.apps[].devices] | all(. == [$device]))' \ + "$ALLOWLIST" >/dev/null +} + +save_vm_logs() { + local name file id + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] || continue + name=$(basename "$file" .vm_id) + id=$(cat "$file") + "${VMM_CLI[@]}" logs "$id" -n 2000 > "$WORK_DIR/${name}.vm.log" 2>&1 || true + done +} + +cleanup_after() { + [[ "${DSTACK_E2E_CLEANUP_AFTER:-false}" == true ]] || return 0 + local file + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] && remove_vm "$(cat "$file")" + done +} + +phase_upgrade() { + [[ "$PLATFORM" == tdx ]] || die "upgrade phase requires TDX" + wait_vmm + clean_start + + render_kms old "$OLD_KMS_IMAGE_ID" kms-0.5.8.tar + deploy_kms_onboard old "$KMS_OLD_HOST_PORT" + wait_onboard old "$KMS_OLD_HOST_PORT" + authorize_kms_from_attestation old + onboard_rpc "$KMS_OLD_HOST_PORT" Bootstrap \ + "$(jq -cn --arg d "$KMS_RPC_DOMAIN" '{domain:$d}')" \ + "$WORK_DIR/kms-old.bootstrap.json" + jq -e '(.attestation // "") | length > 0' "$WORK_DIR/kms-old.bootstrap.json" >/dev/null \ + || die "KMS 0.5.8 bootstrap did not return quote-bound attestation" + log "KMS 0.5.8 bootstrap returned quote-bound attestation" + finish_onboarding old "$KMS_OLD_HOST_PORT" + + render_gateway_manifests old "$OLD_GATEWAY_IMAGE_ID" gateway-0.5.8.tar + render_gateway_manifests latest "$CURRENT_GATEWAY_IMAGE_ID" gateway-current.tar + write_gateway_env 1 + write_gateway_env 2 + deploy_gateway 1 old "$KMS_OLD_URL" + gateway_version 1 old + deploy_gateway 2 old "$KMS_OLD_URL" + gateway_version 2 old + bootstrap_gateway + + deploy_app legacy legacy "$KMS_OLD_URL" + verify_app_via_gateway legacy 1 + verify_app_via_gateway legacy 2 + capture_kms_identity old "$KMS_OLD_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" + + # Production KMS upgrades are rolling, quote-attested replication into a new + # Local-Key-Provider CVM, not an unmeasured binary swap in the old CVM. + render_kms latest "$CURRENT_KMS_IMAGE_ID" kms-current.tar + deploy_kms_onboard latest "$KMS_LATEST_HOST_PORT" + wait_onboard latest "$KMS_LATEST_HOST_PORT" + authorize_kms_from_attestation latest + onboard_rpc "$KMS_LATEST_HOST_PORT" Onboard \ + "$(jq -cn --arg s "$KMS_OLD_URL" --arg d "$KMS_RPC_DOMAIN" '{source_url:$s,domain:$d}')" \ + "$WORK_DIR/kms-latest.onboard.json" + # Current Onboard returns only the replicated public key. Quote evidence is + # carried by the mutual RA-TLS connection and authorized above through the + # target's exact quote-derived measurement and physical TDX device ID. + finish_onboarding latest "$KMS_LATEST_HOST_PORT" + capture_kms_identity latest "$KMS_LATEST_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" + assert_kms_identity_unchanged + + stop_vm "$(cat "$WORK_DIR/kms-old.vm_id")" "KMS 0.5.8 after cutover" + restart_legacy_on_latest_kms + deploy_app lite lite "$KMS_LATEST_URL" + verify_all_routes + + capture_gateway_state 1 before + capture_gateway_state 2 before + start_network_probe + upgrade_gateway_node 1 + upgrade_gateway_node 2 + stop_network_probe + verify_all_routes + # The current API redacts stored tokens, so metadata equality alone cannot + # prove the secret survived. The mock rejects any token other than the exact + # original value; a forced issuance through each upgraded node proves it. + assert_gateway_dns_credential_usable 1 + assert_gateway_dns_credential_usable 2 + + assert_no_insecure_shortcuts + save_vm_logs + # dstack-kms runs in an inner container, whose stdout is not part of the CVM + # serial log returned by VMM. Each KMS has a fresh data disk, so two 200 GETs + # for the exact measured archive prove that both verifiers downloaded it. + local os_archive_gets + os_archive_gets=$(grep -Fc \ + "GET /os/mr_${OS_IMAGE_HASH}.tar.gz HTTP/1.1\" 200" \ + "$WORK_DIR/artifacts-access.log" || true) + (( os_archive_gets >= 2 )) \ + || die "expected verified OS image downloads by both KMS versions, saw ${os_archive_gets}" + if grep -E 'Image verification is disabled|self-authorization is disabled' \ + "$WORK_DIR/kms-"*.vm.log; then + die "KMS logs contain a forbidden disabled verification path" + fi + + log "production-compatible KMS/Gateway rolling-upgrade E2E success" + cleanup_after +} + +on_exit() { + local rc=$? + if [[ -n "$probe_pid" ]] && kill -0 "$probe_pid" 2>/dev/null; then + touch "$WORK_DIR/network-probe.stop" 2>/dev/null || true + wait "$probe_pid" 2>/dev/null || true + fi + if (( rc != 0 )); then + save_vm_logs || true + fi + exit "$rc" +} +trap on_exit EXIT + +main() { + need_bin /workspace/target/release/dstack + [[ -s "$STATE_DIR/artifacts/images.env" ]] || die "missing prepared image metadata" + case "$PHASE" in + upgrade) phase_upgrade ;; + *) die "unknown DSTACK_E2E_PHASE=$PHASE" ;; + esac + log "Work artifacts: $WORK_DIR" +} + +main "$@" diff --git a/test-suites/full-stack-compose/state/.gitkeep b/test-suites/full-stack-compose/state/.gitkeep new file mode 100644 index 000000000..e69de29bb