From 3d88c7b87e3904649f02a2ea5298acb126d76c95 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 8 Jul 2026 08:05:54 -0700 Subject: [PATCH 1/4] Add host-side compose TDX E2E suite --- .../gateway/dstack-app/bootstrap-cluster.sh | 34 +- dstack/gateway/dstack-app/deploy-to-vmm.sh | 6 + .../Dockerfile.key-provider | 15 +- test-suites/full-stack-compose/.env.example | 50 ++ test-suites/full-stack-compose/.gitignore | 8 + test-suites/full-stack-compose/README.md | 114 ++++ test-suites/full-stack-compose/compose.yml | 266 ++++++++++ .../full-stack-compose/mock-cf-dns/Dockerfile | 9 + .../full-stack-compose/mock-cf-dns/server.py | 274 ++++++++++ .../full-stack-compose/runtime/Dockerfile | 36 ++ .../full-stack-compose/scripts/allowlist.py | 91 ++++ .../full-stack-compose/scripts/app_compose.py | 63 +++ .../scripts/render-config.sh | 498 ++++++++++++++++++ .../full-stack-compose/scripts/runner.sh | 295 +++++++++++ test-suites/full-stack-compose/state/.gitkeep | 0 15 files changed, 1748 insertions(+), 11 deletions(-) create mode 100644 test-suites/full-stack-compose/.env.example create mode 100644 test-suites/full-stack-compose/.gitignore create mode 100644 test-suites/full-stack-compose/README.md create mode 100644 test-suites/full-stack-compose/compose.yml create mode 100644 test-suites/full-stack-compose/mock-cf-dns/Dockerfile create mode 100755 test-suites/full-stack-compose/mock-cf-dns/server.py create mode 100644 test-suites/full-stack-compose/runtime/Dockerfile create mode 100755 test-suites/full-stack-compose/scripts/allowlist.py create mode 100755 test-suites/full-stack-compose/scripts/app_compose.py create mode 100755 test-suites/full-stack-compose/scripts/render-config.sh create mode 100755 test-suites/full-stack-compose/scripts/runner.sh create mode 100644 test-suites/full-stack-compose/state/.gitkeep diff --git a/dstack/gateway/dstack-app/bootstrap-cluster.sh b/dstack/gateway/dstack-app/bootstrap-cluster.sh index f1ad6b48b..90e7ed477 100755 --- a/dstack/gateway/dstack-app/bootstrap-cluster.sh +++ b/dstack/gateway/dstack-app/bootstrap-cluster.sh @@ -33,7 +33,7 @@ echo "Waiting for gateway admin API at $ADMIN_ADDR..." max_retries=60 retry=0 while [ $retry -lt $max_retries ]; do - if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Status" >/dev/null 2>&1; then + if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.Status" >/dev/null 2>&1; then break fi retry=$((retry + 1)) @@ -48,29 +48,45 @@ fi echo "Admin API ready, bootstrapping configuration..." -# Set ACME URL -if [ "$ACME_STAGING" = "yes" ]; then +# Set ACME URL. ACME_URL is useful for local E2E suites (for example Pebble). +if [ -n "${ACME_URL:-}" ]; then + : +elif [ "$ACME_STAGING" = "yes" ]; then ACME_URL="https://acme-staging-v02.api.letsencrypt.org/directory" else ACME_URL="https://acme-v02.api.letsencrypt.org/directory" fi echo "Setting certbot config (ACME URL: $ACME_URL)..." -curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \ +curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.SetCertbotConfig" \ -H "Content-Type: application/json" \ -d '{"acme_url":"'"$ACME_URL"'","renew_interval_secs":3600,"renew_before_expiration_secs":864000,"renew_timeout_secs":300}' >/dev/null \ && echo " Certbot config set" || echo " WARN: failed to set certbot config" # Create DNS credential if CF_API_TOKEN is provided and no credentials exist yet if [ -n "$CF_API_TOKEN" ]; then - existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListDnsCredentials" 2>/dev/null) + existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListDnsCredentials" 2>/dev/null) cred_count=$(echo "$existing" | jq -r '.credentials | length' 2>/dev/null || echo "0") if [ "$cred_count" = "0" ]; then echo "Creating default DNS credential..." - curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \ + dns_payload=$(jq -cn \ + --arg token "$CF_API_TOKEN" \ + --arg api_url "${CF_API_URL:-}" \ + --argjson ttl "${DNS_TXT_TTL:-60}" \ + --argjson max_wait "${MAX_DNS_WAIT:-300}" \ + '{ + name: "cloudflare", + provider_type: "cloudflare", + cf_api_token: $token, + set_as_default: true, + dns_txt_ttl: $ttl, + max_dns_wait: $max_wait + } + + (if $api_url == "" then {} else {cf_api_url: $api_url} end)') + curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.CreateDnsCredential" \ -H "Content-Type: application/json" \ - -d '{"name":"cloudflare","provider_type":"cloudflare","cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' >/dev/null \ + -d "$dns_payload" >/dev/null \ && echo " DNS credential created" || echo " WARN: failed to create DNS credential" else echo " DNS credentials already exist ($cred_count), skipping" @@ -81,12 +97,12 @@ fi # Add ZT-Domain if SRV_DOMAIN is provided and domain doesn't exist yet if [ -n "$SRV_DOMAIN" ]; then - existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" 2>/dev/null) + existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListZtDomains" 2>/dev/null) has_domain=$(echo "$existing" | jq -r '.domains[]? | select(.domain=="'"$SRV_DOMAIN"'") | .domain' 2>/dev/null) if [ -z "$has_domain" ]; then echo "Adding ZT-Domain: $SRV_DOMAIN..." - curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \ + curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.AddZtDomain" \ -H "Content-Type: application/json" \ -d '{"domain":"'"$SRV_DOMAIN"'","port":443,"priority":100}' >/dev/null \ && echo " ZT-Domain added" || echo " WARN: failed to add ZT-Domain" diff --git a/dstack/gateway/dstack-app/deploy-to-vmm.sh b/dstack/gateway/dstack-app/deploy-to-vmm.sh index 5ae76a87c..8e363f52b 100755 --- a/dstack/gateway/dstack-app/deploy-to-vmm.sh +++ b/dstack/gateway/dstack-app/deploy-to-vmm.sh @@ -64,6 +64,12 @@ NODE_ID=1 # Whether to use ACME staging (yes/no) ACME_STAGING=no +# Optional ACME directory override for local E2E (for example Pebble). +# ACME_URL=http://10.0.2.2:14000/dir +# Optional Cloudflare API override for local E2E mock. +# CF_API_URL=http://10.0.2.2:18080/client/v4 +# DNS_TXT_TTL=60 +# MAX_DNS_WAIT=300 # Networking mode: bridge or user (default: user) # NET_MODE=bridge diff --git a/dstack/key-provider-build/Dockerfile.key-provider b/dstack/key-provider-build/Dockerfile.key-provider index ba2975788..fb44e35b7 100644 --- a/dstack/key-provider-build/Dockerfile.key-provider +++ b/dstack/key-provider-build/Dockerfile.key-provider @@ -59,9 +59,20 @@ RUN git clone https://github.com/MoeMahhouk/gramine-sealing-key-provider && \ WORKDIR /gramine-sealing-key-provider COPY Cargo.lock . -# Build key provider, generate keys, and build manifest +# Build key provider, generate keys, and build manifest. +# +# Do not use `gramine-sgx-gen-private-key` here. On some hosts the Python +# cryptography/OpenSSL stack in the Gramine image fails while generating the +# RSA key with a generic "Unknown OpenSSL error". Generate the exact same kind +# of Gramine signing key directly with OpenSSL instead: +# - 3072-bit RSA +# - public exponent 3 +# - traditional PEM accepted by gramine-sgx-sign RUN make target/release/gramine-sealing-key-provider && \ - gramine-sgx-gen-private-key && \ + mkdir -p /root/.config/gramine && \ + chmod 0700 /root/.config /root/.config/gramine && \ + openssl genrsa -traditional -3 -out /root/.config/gramine/enclave-key.pem 3072 && \ + chmod 0600 /root/.config/gramine/enclave-key.pem && \ make RUST_LOG=info RUN gramine-sgx-sigstruct-view --output-format json gramine-sealing-key-provider.sig diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example new file mode 100644 index 000000000..67575d15e --- /dev/null +++ b/test-suites/full-stack-compose/.env.example @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Directory containing unpacked dstack guest images: //{bzImage,initramfs...,rootfs...,digest.txt} +# In meta-dstack this is usually ../../../build/images from this directory, or an absolute path. +DSTACK_E2E_IMAGE_STORE=../../../build/images +DSTACK_E2E_IMAGE_NAME=dstack-0.6.0 +DSTACK_E2E_PLATFORM=tdx + +# App image used inside the only CVM launched by this suite. +DSTACK_E2E_APP_IMAGE=nginx:alpine + +# Mock ACME/DNS + externally visible test domain for Gateway SNI. +DSTACK_E2E_BASE_DOMAIN=e2e.test +DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest + +# Host ports. Defaults are chosen to avoid the common dstackup ports. +DSTACK_E2E_VMM_PORT=29080 +DSTACK_E2E_AUTH_PORT=28011 +DSTACK_E2E_KMS_HOST_PORT=28082 +DSTACK_E2E_GATEWAY_RPC_HOST_PORT=28000 +DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT=28001 +DSTACK_E2E_GATEWAY_PROXY_HOST_PORT=28443 +DSTACK_E2E_GATEWAY_WG_HOST_PORT=28120 +DSTACK_E2E_KEY_PROVIDER_PORT=13443 +DSTACK_E2E_HOST_API_PORT=20011 +DSTACK_E2E_MOCK_CF_HTTP_PORT=38080 +DSTACK_E2E_PEBBLE_HTTP_PORT=34000 +DSTACK_E2E_PEBBLE_MGMT_PORT=35000 + +# Dev-compose gateway policy. "any" matches the tdxlab host-side gateway mode: +# app CVMs skip gateway app-id pinning but still use KMS authorization for apps. +DSTACK_E2E_GATEWAY_APP_ID=any +DSTACK_E2E_GATEWAY_WG_INTERFACE=wg-ds-e2e +DSTACK_E2E_GATEWAY_WG_IP=10.8.0.1/16 +DSTACK_E2E_GATEWAY_WG_RESERVED_NET=10.8.0.1/32 +DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE=10.8.0.0/18 + +# VMM/QEMU knobs. +DSTACK_E2E_CID_START=15000 +DSTACK_E2E_QGS_PORT=4050 +DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64 + +# SGX QCNL config used by AESMD. On hosts with a local PCCS, point this at the +# host config, e.g. /etc/sgx_default_qcnl.conf. +DSTACK_E2E_QCNL_CONF=../../dstack/key-provider-build/sgx_default_qcnl.conf + +# Runner behaviour. +DSTACK_E2E_CLEAN_START=true +DSTACK_E2E_CLEANUP_AFTER=false diff --git a/test-suites/full-stack-compose/.gitignore b/test-suites/full-stack-compose/.gitignore new file mode 100644 index 000000000..89e702f8d --- /dev/null +++ b/test-suites/full-stack-compose/.gitignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.env +/state/* +!/state/.gitkeep +__pycache__/ +*.pyc diff --git a/test-suites/full-stack-compose/README.md b/test-suites/full-stack-compose/README.md new file mode 100644 index 000000000..5c913a275 --- /dev/null +++ b/test-suites/full-stack-compose/README.md @@ -0,0 +1,114 @@ + + +# dstack full-stack compose E2E suite + +This directory contains a Docker Compose based test suite that mirrors the +`tdxlab`/`dstack-k8s deploy/compose` shape: host-side services run as normal +processes/containers, and QEMU is used only for the real app CVM under test. + +```text +docker compose + ├─ mock-cf-dns-api # tiny Cloudflare API + TXT DNS mock + ├─ pebble # ACME test server (HTTP-capable custom Pebble image) + ├─ aesmd + local-keyprovider + ├─ dstack-auth # KMS auth webhook backed by auth-allowlist.json + ├─ dstack-kms # host-side dev KMS, not a CVM + ├─ dstack-gateway # host-side dev Gateway, not a CVM + ├─ dstack-vmm # launches real TDX/SNP app CVMs via QEMU + └─ runner + ├─ configures Gateway certbot to use mock CF + Pebble + ├─ deploys nginx test app CVM with kms_enabled=true + gateway_enabled=true + └─ verifies HTTPS Gateway -> app routing +``` + +It is intended for a **real TDX/SGX host** (or SEV-SNP host after setting the +platform/image knobs). It is not a pure unit-test environment; QEMU still starts +a confidential VM for the app. + +## Prerequisites + +1. From this directory, build the host binaries from the dstack checkout: + + ```bash + cargo build --release \ + --manifest-path ../../dstack/Cargo.toml \ + -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms \ + -p dstack-gateway -p supervisor + ``` + +2. Make sure the host has the required devices/services: + + - `/dev/kvm` + - Intel TDX support and QGS for TDX (`DSTACK_E2E_QGS_PORT`, default `4050`) + - `/dev/sgx_enclave` and `/dev/sgx_provision` for the local key provider + - an SGX QCNL config that works on the host. By default the suite uses + `../../dstack/key-provider-build/sgx_default_qcnl.conf`; on hosts with a local + PCCS, set `DSTACK_E2E_QCNL_CONF=/etc/sgx_default_qcnl.conf`. + - `/dev/vhost-vsock` for guest↔host vsock services + - a TDX-capable `qemu-system-x86_64` available inside the runtime container. + The default runtime image follows `../dstack-k8s/deploy/compose` and + installs QEMU from `ppa:kobuk-team/tdx-release`. + - WireGuard kernel support on the host. The Gateway service runs privileged + with host networking and creates `DSTACK_E2E_GATEWAY_WG_INTERFACE`. + +3. Provide an unpacked guest image store. From `meta-dstack` this is usually + `../../../build/images`; otherwise copy `.env.example` and set: + + ```bash + cp .env.example .env + $EDITOR .env + ``` + +No Gateway image publishing is needed in this architecture: `dstack-gateway` and +`dstack-kms` are run from the local `target/release/` binaries on the host side. + +## Run + +From this directory, run: + +```bash +DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build --abort-on-container-exit runner +``` + +For iterative debugging, keep the stack running: + +```bash +DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build +``` + +The runner leaves the app VM running by default for inspection. Set +`DSTACK_E2E_CLEANUP_AFTER=true` to remove suite VMs at the end. Stale VMs whose +names start with `DSTACK_E2E_NAME_PREFIX` are removed at the start when +`DSTACK_E2E_CLEAN_START=true`. + +## What is verified + +The runner checks: + +1. Host-side KMS is reachable over TLS. +2. Host-side Gateway Admin API accepts configuration. +3. Gateway obtains a wildcard certificate for `*.DSTACK_E2E_BASE_DOMAIN` from + Pebble using the mock Cloudflare DNS API. +4. VMM API becomes reachable. +5. A real nginx app CVM boots with `kms_enabled=true` and `gateway_enabled=true`. +6. `curl -k` through the Gateway SNI route returns the nginx welcome page. + +Artifacts are written under `state/work/`. + +## Notes / flexibility gaps + +- This is a dev compose KMS/Gateway setup. KMS and Gateway key material is + generated under `state/`; the allowlist returns `gatewayAppId = "any"`, which + matches the host-side tdxlab-style deployment and avoids requiring Gateway to + be a CVM. +- `dstackup install` is systemd-oriented; this suite renders `vmm.toml`, + `kms.toml`, `gateway.toml`, and `auth-allowlist.json` itself. +- Running VMM in a container depends on the container image having a QEMU build + that supports the target TEE platform. The runtime Dockerfile uses Intel's + kobuk-team TDX PPA for QEMU, matching the dstack-k8s compose deployment. +- Host API is vsock-only by design. The suite therefore uses privileged host + networking and a separate `DSTACK_E2E_HOST_API_PORT` to avoid conflicts with + other VMM instances. diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml new file mode 100644 index 000000000..e12e74446 --- /dev/null +++ b/test-suites/full-stack-compose/compose.yml @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: dstack-full-e2e + +x-suite-env: &suite-env + DSTACK_E2E_STATE_DIR: /suite-state + DSTACK_E2E_CONFIG_DIR: /suite-state/config + DSTACK_E2E_RUN_DIR: /suite-state/run + DSTACK_E2E_VM_DIR: /suite-state/vm + DSTACK_E2E_VOLUMES_DIR: /suite-state/volumes + DSTACK_E2E_IMAGE_ROOT: /images + DSTACK_E2E_IMAGE_NAME: ${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} + DSTACK_E2E_PLATFORM: ${DSTACK_E2E_PLATFORM:-tdx} + DSTACK_E2E_VMM_PORT: ${DSTACK_E2E_VMM_PORT:-29080} + DSTACK_E2E_AUTH_PORT: ${DSTACK_E2E_AUTH_PORT:-28011} + DSTACK_E2E_KMS_HOST_PORT: ${DSTACK_E2E_KMS_HOST_PORT:-28082} + DSTACK_E2E_GATEWAY_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} + DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} + DSTACK_E2E_GATEWAY_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} + DSTACK_E2E_GATEWAY_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY_WG_HOST_PORT:-28120} + DSTACK_E2E_GATEWAY_WG_INTERFACE: ${DSTACK_E2E_GATEWAY_WG_INTERFACE:-wg-ds-e2e} + DSTACK_E2E_GATEWAY_WG_IP: ${DSTACK_E2E_GATEWAY_WG_IP:-10.8.0.1/16} + DSTACK_E2E_GATEWAY_WG_RESERVED_NET: ${DSTACK_E2E_GATEWAY_WG_RESERVED_NET:-10.8.0.1/32} + DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE: ${DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE:-10.8.0.0/18} + DSTACK_E2E_GATEWAY_APP_ID: ${DSTACK_E2E_GATEWAY_APP_ID:-any} + DSTACK_E2E_KEY_PROVIDER_PORT: ${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} + DSTACK_E2E_HOST_API_PORT: ${DSTACK_E2E_HOST_API_PORT:-20011} + DSTACK_E2E_CID_START: ${DSTACK_E2E_CID_START:-15000} + DSTACK_E2E_QGS_PORT: ${DSTACK_E2E_QGS_PORT:-4050} + DSTACK_E2E_QEMU_PATH: ${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} + DSTACK_E2E_BASE_DOMAIN: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test} + DSTACK_E2E_MOCK_CF_HTTP_PORT: ${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} + DSTACK_E2E_PEBBLE_HTTP_PORT: ${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} + DSTACK_E2E_APP_IMAGE: ${DSTACK_E2E_APP_IMAGE:-nginx:alpine} + DSTACK_E2E_APP_NAME: ${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} + DSTACK_E2E_NAME_PREFIX: ${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + DSTACK_E2E_ALLOW_UNPINNED_IMAGE: ${DSTACK_E2E_ALLOW_UNPINNED_IMAGE:-false} + +services: + init-config: + build: + context: ./runtime + image: dstack-e2e-runtime:local + network_mode: host + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../build/images}:/images:ro + environment: + <<: *suite-env + command: ["/suite/scripts/render-config.sh"] + + mock-cf-dns-api: + build: + context: ./mock-cf-dns + image: dstack-e2e-mock-cf-dns:local + networks: + e2e-net: + ipv4_address: 172.31.0.10 + ports: + - "127.0.0.1:${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080}:8080" + environment: + PORT: 8080 + DEBUG: ${DSTACK_E2E_MOCK_DEBUG:-false} + MOCK_CF_ZONES: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test},test,local + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] + interval: 5s + timeout: 3s + retries: 10 + + pebble: + # Custom Pebble build used by the existing gateway E2E suite. It supports + # plain HTTP ACME URLs, which keeps the host-side Gateway bootstrap simple. + image: ${DSTACK_E2E_PEBBLE_IMAGE:-kvin/pebble:latest} + command: ["-http", "-dnsserver", "172.31.0.10:53"] + networks: + e2e-net: + ipv4_address: 172.31.0.11 + ports: + - "127.0.0.1:${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000}:14000" + - "127.0.0.1:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000" + environment: + PEBBLE_VA_NOSLEEP: "1" + PEBBLE_VA_ALWAYS_VALID: "1" + depends_on: + mock-cf-dns-api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://127.0.0.1:14000/dir || curl -fsS http://127.0.0.1:14000/dir >/dev/null"] + interval: 5s + timeout: 3s + retries: 20 + + aesmd: + build: + context: ../../dstack/key-provider-build + dockerfile: Dockerfile.aesmd + image: dstack-e2e-aesmd:local + privileged: true + network_mode: host + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + volumes: + - ${DSTACK_E2E_QCNL_CONF:-../../dstack/key-provider-build/sgx_default_qcnl.conf}:/etc/sgx_default_qcnl.conf:ro + - aesmd-sock:/var/run/aesmd/ + restart: unless-stopped + + local-keyprovider: + build: + context: ../../dstack/key-provider-build + dockerfile: Dockerfile.key-provider + args: + APT_SNAPSHOT: ${APT_SNAPSHOT:-20260423T000000Z} + RUSTUP_VERSION: ${RUSTUP_VERSION:-1.28.2} + RUSTUP_INIT_SHA256: ${RUSTUP_INIT_SHA256:-20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c} + RUST_TOOLCHAIN: ${RUST_TOOLCHAIN:-1.85.1} + image: dstack-e2e-local-keyprovider:local + privileged: true + devices: + - "/dev/sgx_enclave:/dev/sgx_enclave" + - "/dev/sgx_provision:/dev/sgx_provision" + depends_on: + - aesmd + volumes: + - aesmd-sock:/var/run/aesmd/ + ports: + - "127.0.0.1:${DSTACK_E2E_KEY_PROVIDER_PORT:-13443}:3443" + restart: unless-stopped + + auth: + image: dstack-e2e-runtime:local + network_mode: host + depends_on: + init-config: + condition: service_completed_successfully + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + command: + - bash + - -lc + - exec /workspace/target/release/dstack-auth --config /suite-state/config/auth-allowlist.json --address 127.0.0.1 --port ${DSTACK_E2E_AUTH_PORT:-28011} + restart: unless-stopped + + kms: + image: dstack-e2e-runtime:local + network_mode: host + depends_on: + init-config: + condition: service_completed_successfully + auth: + condition: service_started + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + environment: + RUST_LOG: ${DSTACK_E2E_KMS_RUST_LOG:-info,dstack_kms=info} + command: + - bash + - -lc + - exec /workspace/target/release/dstack-kms -c /suite-state/config/kms.toml + healthcheck: + test: ["CMD-SHELL", "curl -kfsS https://127.0.0.1:${DSTACK_E2E_KMS_HOST_PORT:-28082}/metrics >/dev/null"] + interval: 5s + timeout: 3s + retries: 30 + restart: unless-stopped + + gateway: + image: dstack-e2e-runtime:local + privileged: true + network_mode: host + depends_on: + init-config: + condition: service_completed_successfully + kms: + condition: service_healthy + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + - /lib/modules:/lib/modules:ro + environment: + RUST_LOG: ${DSTACK_E2E_GATEWAY_RUST_LOG:-info,dstack_gateway=debug,certbot=debug} + command: + - bash + - -lc + - exec /workspace/target/release/dstack-gateway -c /suite-state/config/gateway.toml + healthcheck: + test: ["CMD-SHELL", "curl -kfsS https://127.0.0.1:${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000}/health >/dev/null"] + interval: 5s + timeout: 3s + retries: 30 + restart: unless-stopped + + vmm: + image: dstack-e2e-runtime:local + privileged: true + pid: host + network_mode: host + devices: + - "/dev/kvm:/dev/kvm" + - "/dev/vhost-vsock:/dev/vhost-vsock" + depends_on: + init-config: + condition: service_completed_successfully + auth: + condition: service_started + kms: + condition: service_healthy + gateway: + condition: service_healthy + local-keyprovider: + condition: service_started + volumes: + - ./state:/suite-state + - ../../dstack:/workspace:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../build/images}:/images:ro + - /dev:/dev + - /lib/modules:/lib/modules:ro + environment: + RUST_LOG: ${DSTACK_E2E_VMM_RUST_LOG:-info,dstack_vmm=info} + command: + - bash + - -lc + - exec /workspace/target/release/dstack-vmm -c /suite-state/config/vmm.toml + restart: unless-stopped + + runner: + image: dstack-e2e-runtime:local + network_mode: host + depends_on: + mock-cf-dns-api: + condition: service_healthy + pebble: + condition: service_healthy + kms: + condition: service_healthy + gateway: + condition: service_healthy + vmm: + condition: service_started + volumes: + - ./state:/suite-state + - ./scripts:/suite/scripts:ro + - ../../dstack:/workspace:ro + environment: + DSTACK_E2E_STATE_DIR: /suite-state + DSTACK_E2E_CLEAN_START: ${DSTACK_E2E_CLEAN_START:-true} + DSTACK_E2E_CLEANUP_AFTER: ${DSTACK_E2E_CLEANUP_AFTER:-false} + DSTACK_E2E_APP_VCPU: ${DSTACK_E2E_APP_VCPU:-2} + DSTACK_E2E_APP_MEMORY: ${DSTACK_E2E_APP_MEMORY:-2048} + command: ["/suite/scripts/runner.sh"] + +networks: + e2e-net: + driver: bridge + ipam: + config: + - subnet: 172.31.0.0/24 + +volumes: + aesmd-sock: diff --git a/test-suites/full-stack-compose/mock-cf-dns/Dockerfile b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile new file mode 100644 index 000000000..8c68f2ec1 --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/Dockerfile @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.12-slim +WORKDIR /app +COPY server.py /app/server.py +EXPOSE 8080/tcp 53/udp +CMD ["python", "/app/server.py"] diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/test-suites/full-stack-compose/mock-cf-dns/server.py new file mode 100755 index 000000000..344522ccd --- /dev/null +++ b/test-suites/full-stack-compose/mock-cf-dns/server.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Tiny Cloudflare DNS API + UDP TXT DNS mock for dstack gateway E2E. + +It implements just the Cloudflare endpoints used by certbot's CloudflareClient: + GET /client/v4/zones + GET /client/v4/zones//dns_records?name= + POST /client/v4/zones//dns_records + DELETE /client/v4/zones//dns_records/ + +It also serves TXT answers on UDP/53 so Pebble can validate DNS-01 when the +ACME server is not configured with PEBBLE_VA_ALWAYS_VALID=1. +""" + +from __future__ import annotations + +import json +import os +import re +import socket +import struct +import threading +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +STATE_LOCK = threading.RLock() +RECORDS: list[dict[str, Any]] = [] +NEXT_ID = 1 + + +def _zones() -> list[dict[str, str]]: + raw = os.environ.get("MOCK_CF_ZONES", "e2e.test test local") + names = [ + z.strip().strip(".").lower() for z in re.split(r"[,\s]+", raw) if z.strip() + ] + if not names: + names = ["e2e.test"] + out = [] + seen = set() + for name in names: + if name in seen: + continue + seen.add(name) + out.append({"id": zone_id_for(name), "name": name}) + return out + + +def zone_id_for(name: str) -> str: + """Return a deterministic mock zone ID for a DNS name.""" + safe = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") or "zone" + return f"zone-{safe}" + + +def _json(handler: BaseHTTPRequestHandler, code: int, body: Any) -> None: + data = json.dumps(body, sort_keys=True).encode() + handler.send_response(code) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(data))) + handler.end_headers() + handler.wfile.write(data) + + +def _record_content(payload: dict[str, Any]) -> str: + if "content" in payload: + return str(payload.get("content") or "") + data = payload.get("data") or {} + if payload.get("type") == "CAA" and isinstance(data, dict): + return f'{data.get("flags", 0)} {data.get("tag", "issue")} "{data.get("value", "")}"' + return "" + + +class Handler(BaseHTTPRequestHandler): + """Handle the mock Cloudflare HTTP API.""" + + server_version = "dstack-mock-cf-dns/0.1" + + def log_message(self, fmt: str, *args: Any) -> None: + """Emit request logs when debug logging is enabled.""" + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + super().log_message(fmt, *args) + + def do_GET(self) -> None: # noqa: N802 + """Handle supported HTTP GET endpoints.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + query = urllib.parse.parse_qs(parsed.query) + if path == "/health": + _json(self, 200, {"ok": True}) + return + if path == "/api/records": + with STATE_LOCK: + _json(self, 200, {"records": RECORDS}) + return + if path == "/client/v4/zones": + zones = _zones() + _json( + self, + 200, + { + "success": True, + "result": zones, + "result_info": { + "page": 1, + "per_page": 50, + "total_pages": 1, + "count": len(zones), + "total_count": len(zones), + }, + }, + ) + return + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if m: + name = (query.get("name") or [""])[0].strip().strip(".").lower() + with STATE_LOCK: + records = [ + r + for r in RECORDS + if not name or r["name"].strip(".").lower() == name + ] + _json( + self, + 200, + { + "success": True, + "result": records, + "result_info": {"page": 1, "per_page": 100, "total_pages": 1}, + }, + ) + return + _json( + self, 404, {"success": False, "errors": [{"message": f"not found: {path}"}]} + ) + + def do_POST(self) -> None: # noqa: N802 + """Create a mock DNS record.""" + global NEXT_ID + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + length = int(self.headers.get("content-length", "0") or "0") + payload = json.loads(self.rfile.read(length) or b"{}") + with STATE_LOCK: + record_id = f"rec-{NEXT_ID}" + NEXT_ID += 1 + record = { + "id": record_id, + "zone_id": m.group(1), + "type": str(payload.get("type") or "TXT").upper(), + "name": str(payload.get("name") or "").strip().strip("."), + "content": _record_content(payload), + "ttl": int(payload.get("ttl") or 60), + "created_on": int(time.time()), + } + RECORDS.append(record) + _json(self, 200, {"success": True, "result": record}) + + def do_DELETE(self) -> None: # noqa: N802 + """Delete a mock DNS record.""" + parsed = urllib.parse.urlparse(self.path) + path = parsed.path.rstrip("/") or "/" + m = re.fullmatch(r"/client/v4/zones/([^/]+)/dns_records/([^/]+)", path) + if not m: + _json( + self, + 404, + {"success": False, "errors": [{"message": f"not found: {path}"}]}, + ) + return + record_id = urllib.parse.unquote(m.group(2)) + with STATE_LOCK: + before = len(RECORDS) + RECORDS[:] = [r for r in RECORDS if r["id"] != record_id] + removed = before != len(RECORDS) + _json( + self, + 200, + {"success": True, "result": {"id": record_id, "removed": removed}}, + ) + + +def parse_qname(packet: bytes, offset: int = 12) -> tuple[str, int]: + """Parse a DNS question name and return its value and end offset.""" + labels: list[str] = [] + while True: + if offset >= len(packet): + raise ValueError("bad qname") + length = packet[offset] + offset += 1 + if length == 0: + break + labels.append(packet[offset : offset + length].decode("ascii", "ignore")) + offset += length + return ".".join(labels).strip(".").lower(), offset + + +def txt_rdata(text: str) -> bytes: + """Encode text as DNS TXT record data.""" + raw = text.encode() + chunks = [raw[i : i + 255] for i in range(0, len(raw), 255)] or [b""] + return b"".join(bytes([len(c)]) + c for c in chunks) + + +def dns_response(packet: bytes) -> bytes: + """Build a DNS response containing matching TXT records.""" + if len(packet) < 12: + return b"" + txid = packet[:2] + qdcount = struct.unpack("!H", packet[4:6])[0] + if qdcount < 1: + return b"" + name, qend = parse_qname(packet) + question = packet[12 : qend + 4] + qtype = ( + struct.unpack("!H", packet[qend : qend + 2])[0] + if qend + 4 <= len(packet) + else 16 + ) + with STATE_LOCK: + answers = [ + r + for r in RECORDS + if r["type"] == "TXT" and r["name"].strip(".").lower() == name + ] + if qtype not in (16, 255): + answers = [] + header = txid + struct.pack("!HHHHH", 0x8180, 1, len(answers), 0, 0) + body = question + for record in answers: + rdata = txt_rdata(record["content"]) + body += ( + b"\xc0\x0c" + + struct.pack("!HHIH", 16, 1, int(record.get("ttl", 60)), len(rdata)) + + rdata + ) + return header + body + + +def dns_loop() -> None: + """Serve mock DNS responses over UDP.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("0.0.0.0", 53)) + while True: + packet, addr = sock.recvfrom(4096) + try: + response = dns_response(packet) + if response: + sock.sendto(response, addr) + except Exception as exc: # pragma: no cover - diagnostic only + if os.environ.get("DEBUG", "").lower() in {"1", "true", "yes"}: + print(f"dns error from {addr}: {exc}", flush=True) + + +def main() -> None: + """Run the mock Cloudflare API and DNS servers.""" + threading.Thread(target=dns_loop, daemon=True).start() + port = int(os.environ.get("PORT", "8080")) + print(f"mock CF API on :{port}; zones={_zones()}", flush=True) + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/runtime/Dockerfile b/test-suites/full-stack-compose/runtime/Dockerfile new file mode 100644 index 000000000..12cd5f737 --- /dev/null +++ b/test-suites/full-stack-compose/runtime/Dockerfile @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + software-properties-common \ + && add-apt-repository -y ppa:kobuk-team/tdx-release && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + bash \ + coreutils \ + curl \ + iproute2 \ + iptables \ + iputils-ping \ + jq \ + netcat-openbsd \ + openssl \ + procps \ + python3 \ + qemu-system-x86 \ + qemu-utils \ + socat \ + tini \ + wireguard-tools \ + xz-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /suite +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["bash"] diff --git a/test-suites/full-stack-compose/scripts/allowlist.py b/test-suites/full-stack-compose/scripts/allowlist.py new file mode 100755 index 000000000..8f599169b --- /dev/null +++ b/test-suites/full-stack-compose/scripts/allowlist.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Manage the mutable KMS authorization allowlist for the compose E2E suite.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def norm_hex(value: str) -> str: + """Normalize a possibly prefixed hexadecimal string.""" + value = value.strip() + if value.lower().startswith("0x"): + value = value[2:] + return value.lower() + + +def load(path: Path) -> dict: + """Load an allowlist or return an empty default policy.""" + if path.exists(): + return json.loads(path.read_text()) + return { + "osImages": [], + "gatewayAppId": "", + "kms": {"mrAggregated": [], "devices": [], "allowAnyDevice": True}, + "apps": {}, + } + + +def save(path: Path, data: dict) -> None: + """Atomically write an allowlist to disk.""" + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") + os.replace(tmp, path) + + +def add_app(args: argparse.Namespace) -> None: + """Add an application and compose hash to the allowlist.""" + path = Path(args.path) + data = load(path) + apps = data.setdefault("apps", {}) + app_id = norm_hex(args.app_id) + compose_hash = norm_hex(args.compose_hash) + entry = apps.setdefault( + app_id, {"composeHashes": [], "devices": [], "allowAnyDevice": True} + ) + hashes = entry.setdefault("composeHashes", []) + if not any(norm_hex(h) == compose_hash for h in hashes): + hashes.append(compose_hash) + entry.setdefault("devices", []) + entry["allowAnyDevice"] = True + if args.gateway_app_id is not None: + data["gatewayAppId"] = norm_hex(args.gateway_app_id) + save(path, data) + print(json.dumps({"appId": app_id, "composeHash": compose_hash, "path": str(path)})) + + +def set_gateway(args: argparse.Namespace) -> None: + """Set the allowlisted Gateway application ID.""" + path = Path(args.path) + data = load(path) + data["gatewayAppId"] = norm_hex(args.gateway_app_id) + save(path, data) + print(json.dumps({"gatewayAppId": data["gatewayAppId"], "path": str(path)})) + + +def main() -> None: + """Parse command-line arguments and update the allowlist.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + p = sub.add_parser("add-app") + p.add_argument("--path", required=True) + p.add_argument("--app-id", required=True) + p.add_argument("--compose-hash", required=True) + p.add_argument("--gateway-app-id") + p.set_defaults(func=add_app) + p = sub.add_parser("set-gateway") + p.add_argument("--path", required=True) + p.add_argument("--gateway-app-id", required=True) + p.set_defaults(func=set_gateway) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py new file mode 100755 index 000000000..02d0ec42c --- /dev/null +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Render app-compose manifests used by the full-stack E2E suite.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def write_manifest(path: Path, manifest: dict) -> None: + """Write a manifest and print its derived identifiers.""" + body = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + path.write_text(body) + digest = hashlib.sha256(body.encode()).hexdigest() + print(json.dumps({"path": str(path), "composeHash": digest, "appId": digest[:40]})) + + +def nginx(args: argparse.Namespace) -> None: + """Render an nginx app-compose manifest.""" + docker_compose = f"""services: + web: + image: {args.app_image} + ports: + - "80:80" +""" + manifest = { + "docker_compose_file": docker_compose, + "gateway_enabled": True, + "kms_enabled": True, + "local_key_provider_enabled": False, + "manifest_version": 2, + "name": args.name, + "no_instance_id": False, + "public_logs": True, + "public_sysinfo": True, + "runner": "docker-compose", + "secure_time": False, + } + write_manifest(Path(args.output), manifest) + + +def main() -> None: + """Parse command-line arguments and render a manifest.""" + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(required=True) + + p = sub.add_parser("nginx") + p.add_argument("--name", required=True) + p.add_argument("--app-image", required=True) + p.add_argument("--output", required=True) + p.set_defaults(func=nginx) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test-suites/full-stack-compose/scripts/render-config.sh b/test-suites/full-stack-compose/scripts/render-config.sh new file mode 100755 index 000000000..b22764269 --- /dev/null +++ b/test-suites/full-stack-compose/scripts/render-config.sh @@ -0,0 +1,498 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +RUN_DIR=${DSTACK_E2E_RUN_DIR:-$STATE_DIR/run} +VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} +VOLUMES_DIR=${DSTACK_E2E_VOLUMES_DIR:-$STATE_DIR/volumes} +IMAGE_ROOT=${DSTACK_E2E_IMAGE_ROOT:-/images} +IMAGE_NAME=${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} +PLATFORM=${DSTACK_E2E_PLATFORM:-tdx} + +VMM_PORT=${DSTACK_E2E_VMM_PORT:-29080} +AUTH_PORT=${DSTACK_E2E_AUTH_PORT:-28011} +KMS_HOST_PORT=${DSTACK_E2E_KMS_HOST_PORT:-28082} +GATEWAY_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} +GATEWAY_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} +GATEWAY_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} +GATEWAY_WG_HOST_PORT=${DSTACK_E2E_GATEWAY_WG_HOST_PORT:-28120} +GATEWAY_WG_INTERFACE=${DSTACK_E2E_GATEWAY_WG_INTERFACE:-wg-ds-e2e} +GATEWAY_WG_IP=${DSTACK_E2E_GATEWAY_WG_IP:-10.8.0.1/16} +GATEWAY_WG_RESERVED_NET=${DSTACK_E2E_GATEWAY_WG_RESERVED_NET:-10.8.0.1/32} +GATEWAY_WG_CLIENT_RANGE=${DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE:-10.8.0.0/18} +GATEWAY_APP_ID=${DSTACK_E2E_GATEWAY_APP_ID:-any} +KEY_PROVIDER_PORT=${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} +HOST_API_PORT=${DSTACK_E2E_HOST_API_PORT:-20011} +CID_START=${DSTACK_E2E_CID_START:-15000} +QGS_PORT=${DSTACK_E2E_QGS_PORT:-4050} +QEMU_PATH=${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} +BASE_DOMAIN=${DSTACK_E2E_BASE_DOMAIN:-e2e.test} +MOCK_CF_HTTP_PORT=${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} +PEBBLE_HTTP_PORT=${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} +APP_IMAGE=${DSTACK_E2E_APP_IMAGE:-nginx:alpine} +APP_NAME=${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} +SUITE_PREFIX=${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} + +KMS_CERT_DIR=${DSTACK_E2E_KMS_CERT_DIR:-$STATE_DIR/kms-certs} +GATEWAY_CERT_DIR=${DSTACK_E2E_GATEWAY_CERT_DIR:-$STATE_DIR/gateway-certs} +GATEWAY_DATA_DIR=${DSTACK_E2E_GATEWAY_DATA_DIR:-$STATE_DIR/gateway-data} + +mkdir -p \ + "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" "$STATE_DIR/work" \ + "$KMS_CERT_DIR" "$GATEWAY_CERT_DIR" "$GATEWAY_DATA_DIR" + +image_dir="$IMAGE_ROOT/$IMAGE_NAME" +if [[ ! -d "$image_dir" ]]; then + echo "ERROR: image directory not found: $image_dir" >&2 + echo "Set DSTACK_E2E_IMAGE_STORE to a host directory containing $IMAGE_NAME/" >&2 + exit 1 +fi + +digest_file="digest.txt" +if [[ "$PLATFORM" == "amd-sev-snp" || "$PLATFORM" == "sev-snp" || "$PLATFORM" == "snp" ]]; then + PLATFORM="amd-sev-snp" + digest_file="digest.sev.txt" +elif [[ "$PLATFORM" != "tdx" ]]; then + echo "ERROR: unsupported DSTACK_E2E_PLATFORM=$PLATFORM (expected tdx or amd-sev-snp)" >&2 + exit 1 +fi + +OS_IMAGE_HASH="" +if [[ -s "$image_dir/$digest_file" ]]; then + OS_IMAGE_HASH=$(tr -d '[:space:]' < "$image_dir/$digest_file") +elif [[ "${DSTACK_E2E_ALLOW_UNPINNED_IMAGE:-false}" == "true" ]]; then + echo "WARN: missing $image_dir/$digest_file; auth allowlist will not pin OS image" >&2 +else + echo "ERROR: missing $image_dir/$digest_file" >&2 + echo "Set DSTACK_E2E_ALLOW_UNPINNED_IMAGE=true only for local experiments." >&2 + exit 1 +fi + +ADMIN_TOKEN_FILE="$STATE_DIR/gateway-admin-token" +if [[ ! -s "$ADMIN_TOKEN_FILE" ]]; then + openssl rand -hex 24 > "$ADMIN_TOKEN_FILE" + chmod 0600 "$ADMIN_TOKEN_FILE" || true +fi +GATEWAY_ADMIN_TOKEN=$(cat "$ADMIN_TOKEN_FILE") + +openssl_gen_ec_key() { + local path=$1 + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:prime256v1 -out "$path" >/dev/null 2>&1 + chmod 0600 "$path" || true +} + +openssl_gen_ca() { + local key=$1 cert=$2 subject=$3 + openssl_gen_ec_key "$key" + openssl req -x509 -new -key "$key" -sha256 -days 3650 -out "$cert" \ + -subj "$subject" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:1" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" >/dev/null 2>&1 +} + +openssl_gen_kms_rpc_cert() { + local dir=$1 tmp csr + tmp=$(mktemp) + csr="$dir/rpc.csr" + cat > "$tmp" <<'OPENSSL_CONF' +[req] +prompt = no +distinguished_name = dn +req_extensions = v3_req + +[dn] +CN = 10.0.2.2 + +[v3_req] +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = serverAuth,clientAuth +subjectAltName = @alt_names +1.3.6.1.4.1.62397.1.4 = DER:04:07:6B:6D:73:3A:72:70:63 + +[alt_names] +IP.1 = 10.0.2.2 +IP.2 = 127.0.0.1 +DNS.1 = localhost +OPENSSL_CONF + openssl_gen_ec_key "$dir/rpc.key" + openssl req -new -key "$dir/rpc.key" -out "$csr" -config "$tmp" >/dev/null 2>&1 + openssl x509 -req -in "$csr" -CA "$dir/root-ca.crt" -CAkey "$dir/root-ca.key" \ + -CAcreateserial -out "$dir/rpc.crt" -days 3650 -sha256 \ + -extfile "$tmp" -extensions v3_req >/dev/null 2>&1 + rm -f "$tmp" "$csr" +} + +openssl_gen_gateway_rpc_cert() { + local dir=$1 tmp + tmp=$(mktemp) + cat > "$tmp" <<'OPENSSL_CONF' +[req] +prompt = no +distinguished_name = dn +x509_extensions = v3_req + +[dn] +CN = 10.0.2.2 + +[v3_req] +basicConstraints = critical,CA:FALSE +keyUsage = critical,digitalSignature +extendedKeyUsage = serverAuth +subjectAltName = @alt_names + +[alt_names] +IP.1 = 10.0.2.2 +IP.2 = 127.0.0.1 +DNS.1 = localhost +OPENSSL_CONF + openssl_gen_ec_key "$dir/gateway-rpc.key" + openssl req -x509 -new -key "$dir/gateway-rpc.key" -sha256 -days 3650 \ + -out "$dir/gateway-rpc.crt" -config "$tmp" -extensions v3_req >/dev/null 2>&1 + rm -f "$tmp" +} + +write_k256_key() { + local path=$1 + python3 - "$path" <<'PY' +import os +import sys +# secp256k1 curve order +N = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 +path = sys.argv[1] +while True: + b = os.urandom(32) + x = int.from_bytes(b, "big") + if 1 <= x < N: + with open(path, "wb") as f: + f.write(b) + break +PY + chmod 0600 "$path" || true +} + +if [[ ! -s "$KMS_CERT_DIR/root-ca.key" || ! -s "$KMS_CERT_DIR/root-ca.crt" || \ + ! -s "$KMS_CERT_DIR/tmp-ca.key" || ! -s "$KMS_CERT_DIR/tmp-ca.crt" || \ + ! -s "$KMS_CERT_DIR/rpc.key" || ! -s "$KMS_CERT_DIR/rpc.crt" || \ + ! -s "$KMS_CERT_DIR/root-k256.key" ]]; then + echo "Generating dev KMS key material in $KMS_CERT_DIR" + rm -f "$KMS_CERT_DIR"/{root-ca.key,root-ca.crt,root-ca.srl,tmp-ca.key,tmp-ca.crt,rpc.key,rpc.crt,rpc.csr,root-k256.key,rpc-domain} + openssl_gen_ca "$KMS_CERT_DIR/root-ca.key" "$KMS_CERT_DIR/root-ca.crt" "/O=Dstack/CN=Dstack KMS CA" + openssl_gen_ca "$KMS_CERT_DIR/tmp-ca.key" "$KMS_CERT_DIR/tmp-ca.crt" "/O=Dstack/CN=Dstack Client Temp CA" + openssl_gen_kms_rpc_cert "$KMS_CERT_DIR" + write_k256_key "$KMS_CERT_DIR/root-k256.key" + printf '10.0.2.2' > "$KMS_CERT_DIR/rpc-domain" +fi + +if [[ ! -s "$GATEWAY_CERT_DIR/gateway-rpc.key" || ! -s "$GATEWAY_CERT_DIR/gateway-rpc.crt" ]]; then + echo "Generating dev Gateway RPC certificate in $GATEWAY_CERT_DIR" + rm -f "$GATEWAY_CERT_DIR"/{gateway-rpc.key,gateway-rpc.crt} + openssl_gen_gateway_rpc_cert "$GATEWAY_CERT_DIR" +fi + +WG_KEY_FILE="$GATEWAY_DATA_DIR/wg.key" +if [[ ! -s "$WG_KEY_FILE" ]]; then + (umask 077; wg genkey > "$WG_KEY_FILE") + chmod 0600 "$WG_KEY_FILE" || true +fi +GATEWAY_WG_PRIVATE_KEY=$(tr -d '[:space:]' < "$WG_KEY_FILE") +GATEWAY_WG_PUBLIC_KEY=$(printf '%s' "$GATEWAY_WG_PRIVATE_KEY" | wg pubkey) + +cat > "$CONFIG_DIR/auth-allowlist.json" < "$CONFIG_DIR/kms.toml" < "$CONFIG_DIR/gateway.toml" < "$CONFIG_DIR/vmm.toml" < "$STATE_DIR/state.env" < +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +mkdir -p "$WORK_DIR" +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" + +VMM_URL="http://127.0.0.1:${VMM_PORT}" +VMM_CLI=(python3 /workspace/vmm/src/vmm-cli.py --url "$VMM_URL") +DSTACK_CLI=(/workspace/target/release/dstack --host "$VMM_URL") +ALLOWLIST="$CONFIG_DIR/auth-allowlist.json" + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +need_bin() { + [[ -x "$1" ]] || die "missing executable $1; run: cargo build --release -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms -p dstack-gateway -p supervisor" +} +need_bin /workspace/target/release/dstack +need_bin /workspace/target/release/dstack-auth +need_bin /workspace/target/release/dstack-vmm +need_bin /workspace/target/release/dstack-kms +need_bin /workspace/target/release/dstack-gateway +need_bin /workspace/target/release/supervisor + +wait_vmm() { + log "waiting for VMM at $VMM_URL" + for _ in $(seq 1 90); do + if "${DSTACK_CLI[@]}" apps -j >/dev/null 2>&1; then + log "VMM ready" + return 0 + fi + sleep 2 + done + die "VMM not ready" +} + +vm_ids_by_prefix() { + local prefix=$1 + "${VMM_CLI[@]}" lsvm --json 2>/dev/null | jq -r --arg p "$prefix" '.[] | select(.name | startswith($p)) | .id' +} + +remove_vm() { + local id=$1 + [[ -n "$id" ]] || return 0 + log "removing VM $id" + "${VMM_CLI[@]}" remove "$id" >/dev/null 2>&1 || true +} + +clean_start() { + if [[ "${DSTACK_E2E_CLEAN_START:-true}" != "true" ]]; then + return 0 + fi + log "cleaning stale VMs with prefix ${SUITE_PREFIX}" + while read -r id; do + remove_vm "$id" + done < <(vm_ids_by_prefix "$SUITE_PREFIX") + sleep 2 +} + +wait_boot_done() { + local id=$1 label=$2 timeout=${3:-480} + local deadline=$((SECONDS + timeout)) + local last="" + while (( SECONDS < deadline )); do + local info status progress error instance + info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) + if [[ -n "$info" ]]; then + status=$(jq -r '.status // ""' <<<"$info") + progress=$(jq -r '.boot_progress // ""' <<<"$info") + error=$(jq -r '.boot_error // ""' <<<"$info") + instance=$(jq -r '.instance_id // ""' <<<"$info") + local line="status=$status progress=${progress:-none} instance=${instance:-none} error=${error:-none}" + if [[ "$line" != "$last" ]]; then + log "$label: $line" + last=$line + fi + if [[ -n "$error" && "$error" != "null" ]]; then + print_vm_info_safe "$info" >&2 || true + die "$label boot failed: $error" + fi + if [[ "$status" == "running" && "$progress" == "done" ]]; then + printf '%s' "$info" > "$WORK_DIR/${label}.info.json" + return 0 + fi + if [[ "$status" == "exited" || "$status" == "stopped" ]]; then + print_vm_info_safe "$info" >&2 || true + die "$label exited before boot finished" + fi + fi + sleep 5 + done + print_vm_info_safe "$("${VMM_CLI[@]}" info "$id" --json)" >&2 || true + die "timed out waiting for $label" +} + +print_vm_info_safe() { + jq '{ + id, + name, + status, + uptime, + app_id, + instance_id, + boot_progress, + boot_error, + shutdown_progress, + image_version, + events + }' <<<"$1" +} + +compose_meta() { + jq -r '"\(.appId) \(.composeHash)"' +} + +register_app() { + local app_id=$1 hash=$2 gateway_id=${3:-} + local args=(/suite/scripts/allowlist.py add-app --path "$ALLOWLIST" --app-id "$app_id" --compose-hash "$hash") + if [[ -n "$gateway_id" ]]; then + args+=(--gateway-app-id "$gateway_id") + fi + python3 "${args[@]}" >/dev/null +} + +set_gateway_app_id() { + python3 /suite/scripts/allowlist.py set-gateway --path "$ALLOWLIST" --gateway-app-id "$1" >/dev/null +} + + +wait_kms() { + log "waiting for KMS at https://127.0.0.1:${KMS_HOST_PORT}" + for _ in $(seq 1 90); do + if curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/metrics" >/dev/null 2>&1; then + log "KMS ready" + return 0 + fi + sleep 2 + done + die "KMS not ready" +} + +admin_curl() { + local method=$1 + local data + if [[ $# -ge 2 ]]; then + data=$2 + else + data='{}' + fi + local out code + out=$(mktemp) + code=$(curl -sS -o "$out" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ + -H "Content-Type: application/json" \ + "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.${method}" \ + --data-raw "$data" || true) + if [[ "$code" =~ ^2 ]]; then + cat "$out" + rm -f "$out" + return 0 + fi + log "Admin.${method} failed HTTP ${code}: $(cat "$out")" >&2 + rm -f "$out" + return 1 +} + +wait_gateway_admin() { + log "waiting for Gateway admin API on 127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}" + for _ in $(seq 1 90); do + if curl -fsS -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ + "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.Status" >/dev/null 2>&1; then + log "Gateway admin ready" + return 0 + fi + sleep 2 + done + die "Gateway admin API not ready" +} + +bootstrap_gateway_certbot() { + local acme_url="http://127.0.0.1:${PEBBLE_HTTP_PORT}/dir" + local cf_url="http://127.0.0.1:${MOCK_CF_HTTP_PORT}/client/v4" + log "configuring Gateway certbot: acme=$acme_url cf=$cf_url domain=$BASE_DOMAIN" + admin_curl SetCertbotConfig "$(jq -cn --arg u "$acme_url" '{acme_url:$u}')" >/dev/null + # Idempotency for reruns: duplicate credentials/domains are harmless but noisy, so ignore create conflicts. + admin_curl CreateDnsCredential "$(jq -cn --arg u "$cf_url" '{name:"mock-cloudflare", provider_type:"cloudflare", cf_api_token:"test-token", cf_api_url:$u, set_as_default:true, dns_txt_ttl:1, max_dns_wait:0}')" >/dev/null || true + admin_curl AddZtDomain "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, port:443, priority:100}')" >/dev/null || true + log "requesting wildcard cert for *.${BASE_DOMAIN}" + admin_curl RenewZtDomainCert "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, force:true}')" | tee "$WORK_DIR/renew-cert.json" +} + +wait_gateway_cert() { + local sni="gateway.${BASE_DOMAIN}" deadline=$((SECONDS + ${DSTACK_E2E_CERT_TIMEOUT:-240})) + log "waiting for Gateway TLS certificate with SNI $sni" + while (( SECONDS < deadline )); do + if echo | timeout 8 openssl s_client \ + -connect "127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ + -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -ext subjectAltName 2>/dev/null \ + | grep -Fq "*.${BASE_DOMAIN}"; then + log "Gateway wildcard certificate is active" + return 0 + fi + sleep 5 + done + die "timed out waiting for Gateway certificate" +} + +deploy_app() { + local out meta app_id hash vm_id + log "rendering nginx test app-compose" + meta=$(python3 /suite/scripts/app_compose.py nginx \ + --name "$APP_NAME" \ + --app-image "$APP_IMAGE" \ + --output "$WORK_DIR/app-compose.json") + read -r app_id hash < <(jq -r '"\(.appId) \(.composeHash)"' <<<"$meta") + local gateway_id + gateway_id=$(jq -r '.gatewayAppId' "$ALLOWLIST") + log "registering test app in auth allowlist app_id=$app_id gateway_app_id=$gateway_id" + register_app "$app_id" "$hash" "$gateway_id" + log "deploying nginx app CVM" + out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-app" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/app-compose.json" \ + --vcpu "${DSTACK_E2E_APP_VCPU:-2}" \ + --memory "${DSTACK_E2E_APP_MEMORY:-2048}" \ + --disk "${DSTACK_E2E_APP_DISK:-20}") + printf '%s\n' "$out" | tee "$WORK_DIR/app.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse app VM id" + echo "$vm_id" > "$WORK_DIR/app.vm_id" + wait_boot_done "$vm_id" app "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" +} + +verify_app_via_gateway() { + local app_info instance_id sni url deadline + app_info=$(cat "$WORK_DIR/app.info.json") + instance_id=$(jq -r '.instance_id // ""' <<<"$app_info") + [[ -n "$instance_id" && "$instance_id" != "null" ]] || die "app has no instance_id" + sni="${instance_id}-80.${BASE_DOMAIN}" + url="https://${sni}:${GATEWAY_PROXY_HOST_PORT}/" + log "verifying app through Gateway: $url" + deadline=$((SECONDS + ${DSTACK_E2E_APP_HTTP_TIMEOUT:-240})) + while (( SECONDS < deadline )); do + if curl -fsS -k --connect-to "${sni}:${GATEWAY_PROXY_HOST_PORT}:127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ + "$url" > "$WORK_DIR/app-http.out" 2>"$WORK_DIR/app-http.err"; then + if grep -qi "welcome to nginx" "$WORK_DIR/app-http.out"; then + log "Gateway -> app HTTP check passed" + head -c 160 "$WORK_DIR/app-http.out" | tr '\n' ' '; echo + return 0 + fi + fi + sleep 5 + done + cat "$WORK_DIR/app-http.err" >&2 || true + die "timed out verifying app through Gateway" +} + +cleanup_after() { + if [[ "${DSTACK_E2E_CLEANUP_AFTER:-false}" != "true" ]]; then + return 0 + fi + log "DSTACK_E2E_CLEANUP_AFTER=true: removing suite VMs" + for f in app.vm_id gateway.vm_id kms.vm_id; do + if [[ -s "$WORK_DIR/$f" ]]; then + remove_vm "$(cat "$WORK_DIR/$f")" + fi + done +} + +main() { + wait_vmm + clean_start + wait_kms + wait_gateway_admin + bootstrap_gateway_certbot + wait_gateway_cert + deploy_app + verify_app_via_gateway + log "E2E success" + log "VMM dashboard: $VMM_URL" + log "Gateway proxy: https://*.${BASE_DOMAIN}:${GATEWAY_PROXY_HOST_PORT} (connect to 127.0.0.1:${GATEWAY_PROXY_HOST_PORT})" + log "Gateway admin: http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT} (token in $STATE_DIR/gateway-admin-token)" + log "Work artifacts: $WORK_DIR" + cleanup_after +} + +main "$@" diff --git a/test-suites/full-stack-compose/state/.gitkeep b/test-suites/full-stack-compose/state/.gitkeep new file mode 100644 index 000000000..e69de29bb From 777749fc0e33d1e8aa23a3cda2af066d79f881ca Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 13 Jul 2026 01:03:51 -0700 Subject: [PATCH 2/4] Add KMS and Gateway upgrade E2E coverage --- test-suites/full-stack-compose/.env.example | 15 +- test-suites/full-stack-compose/README.md | 75 +++- test-suites/full-stack-compose/compose.yml | 29 +- .../full-stack-compose/run-upgrade-e2e.sh | 209 ++++++++++ .../full-stack-compose/scripts/app_compose.py | 13 + .../scripts/network-probe.sh | 74 ++++ .../scripts/render-config.sh | 25 +- .../full-stack-compose/scripts/runner.sh | 384 ++++++++++++++---- 8 files changed, 724 insertions(+), 100 deletions(-) create mode 100755 test-suites/full-stack-compose/run-upgrade-e2e.sh create mode 100755 test-suites/full-stack-compose/scripts/network-probe.sh diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example index 67575d15e..079299a95 100644 --- a/test-suites/full-stack-compose/.env.example +++ b/test-suites/full-stack-compose/.env.example @@ -7,9 +7,15 @@ DSTACK_E2E_IMAGE_STORE=../../../build/images DSTACK_E2E_IMAGE_NAME=dstack-0.6.0 DSTACK_E2E_PLATFORM=tdx -# App image used inside the only CVM launched by this suite. +# App image used inside the CVMs launched by this suite. DSTACK_E2E_APP_IMAGE=nginx:alpine +# Released images used by run-upgrade-e2e.sh. The KMS case intentionally pins +# exactly 0.5.7; the script fails before starting the stack if Docker Hub does +# not provide that tag. Gateway 0.5.8 is the default published upgrade source. +DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.7 +DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8 + # Mock ACME/DNS + externally visible test domain for Gateway SNI. DSTACK_E2E_BASE_DOMAIN=e2e.test DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest @@ -18,6 +24,9 @@ DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest DSTACK_E2E_VMM_PORT=29080 DSTACK_E2E_AUTH_PORT=28011 DSTACK_E2E_KMS_HOST_PORT=28082 +# 0.5.x KMS encodes rpc-domain as a DNS SAN even when it looks like an IP. +# Use a DNS name that resolves to QEMU's 10.0.2.2 host gateway. +DSTACK_E2E_KMS_RPC_DOMAIN=10.0.2.2.nip.io DSTACK_E2E_GATEWAY_RPC_HOST_PORT=28000 DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT=28001 DSTACK_E2E_GATEWAY_PROXY_HOST_PORT=28443 @@ -48,3 +57,7 @@ DSTACK_E2E_QCNL_CONF=../../dstack/key-provider-build/sgx_default_qcnl.conf # Runner behaviour. DSTACK_E2E_CLEAN_START=true DSTACK_E2E_CLEANUP_AFTER=false + +# Upgrade-driver behaviour. Keep the completed stack by default for inspection. +DSTACK_E2E_UPGRADE_CLEAN_STATE=true +DSTACK_E2E_KEEP_STACK=true diff --git a/test-suites/full-stack-compose/README.md b/test-suites/full-stack-compose/README.md index 5c913a275..993fed942 100644 --- a/test-suites/full-stack-compose/README.md +++ b/test-suites/full-stack-compose/README.md @@ -20,8 +20,9 @@ docker compose ├─ dstack-vmm # launches real TDX/SNP app CVMs via QEMU └─ runner ├─ configures Gateway certbot to use mock CF + Pebble - ├─ deploys nginx test app CVM with kms_enabled=true + gateway_enabled=true - └─ verifies HTTPS Gateway -> app routing + ├─ deploys legacy and lite TDX nginx CVMs + ├─ verifies KMS key provisioning and HTTPS Gateway -> app routing + └─ snapshots durable KMS/Gateway state for upgrade assertions ``` It is intended for a **real TDX/SGX host** (or SEV-SNP host after setting the @@ -62,8 +63,9 @@ a confidential VM for the app. $EDITOR .env ``` -No Gateway image publishing is needed in this architecture: `dstack-gateway` and -`dstack-kms` are run from the local `target/release/` binaries on the host side. +The normal latest-only run needs no published KMS/Gateway image: +`dstack-gateway` and `dstack-kms` run from the local `target/release/` binaries. +The upgrade run additionally pulls its old KMS/Gateway images from Docker Hub. ## Run @@ -73,13 +75,64 @@ From this directory, run: DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build --abort-on-container-exit runner ``` +On TDX this latest-only path deploys two otherwise identical CVMs. Their v3 +manifest requirements force `tdx_measure_acpi_tables=true` (legacy) and +`false` (lite), respectively. Both must reach `boot_progress=done`, which is +after the boot-time `GetAppKey` request, and both must serve nginx through the +Gateway. + +## Run the upgrade scenario + +The host-side driver is required because the runner container cannot replace +its own KMS/Gateway dependencies: + +```bash +DOCKER_BUILDKIT=0 ./run-upgrade-e2e.sh +``` + +Defaults: + +- old KMS: `dstacktee/dstack-kms:0.5.7` (pulled from Docker Hub) +- old Gateway: `dstacktee/dstack-gateway:0.5.8` (pulled from Docker Hub) +- latest KMS, Gateway, VMM, guest image: current checkout/build + +The driver deliberately fails during the image-pull preflight if the exact old +image is unavailable. Override images only for intentional matrix runs: + +```bash +DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.6 \ +DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8 \ + ./run-upgrade-e2e.sh +``` + +The upgrade sequence is: + +1. Boot a forced-legacy CVM against the old KMS and old Gateway. +2. Record the KMS CA, root k256 public key, and that app's derived environment + encryption public key. +3. Replace only KMS with the current binary while retaining its cert/key dir. +4. Require the recorded identities to be byte-for-byte stable; force-reboot the + legacy CVM so it must provision keys from the new KMS; then boot a forced-lite + CVM from scratch against the new KMS. +5. Flush and record Gateway WaveKV state: node UUID, CVM registrations/IPs, + certbot config, DNS credential, ZT domain, and wildcard cert fingerprint. +6. Start rapid direct HTTP probes to both CVM WireGuard IPs, replace only the + Gateway process, and require **zero failed probe cycles**. +7. Require all recorded Gateway state and the cert fingerprint to survive and + verify both SNI routes through the new Gateway. + +The zero-downtime assertion is specifically the CVM WireGuard data plane. The +Gateway process owns the public TLS listener, so a single-node stop/start does +not claim that the external listener itself has zero downtime. External routing +is asserted immediately before and after the upgrade. + For iterative debugging, keep the stack running: ```bash DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build ``` -The runner leaves the app VM running by default for inspection. Set +The runner leaves app VMs running by default for inspection. Set `DSTACK_E2E_CLEANUP_AFTER=true` to remove suite VMs at the end. Stale VMs whose names start with `DSTACK_E2E_NAME_PREFIX` are removed at the start when `DSTACK_E2E_CLEAN_START=true`. @@ -93,8 +146,16 @@ The runner checks: 3. Gateway obtains a wildcard certificate for `*.DSTACK_E2E_BASE_DOMAIN` from Pebble using the mock Cloudflare DNS API. 4. VMM API becomes reachable. -5. A real nginx app CVM boots with `kms_enabled=true` and `gateway_enabled=true`. -6. `curl -k` through the Gateway SNI route returns the nginx welcome page. +5. Real nginx app CVMs boot with `kms_enabled=true` and + `gateway_enabled=true`. +6. On TDX, v3 requirements resolve one CVM to legacy attestation and one to + lite; both complete boot-time KMS key provisioning. +7. `curl -k` through each Gateway SNI route returns the nginx welcome page. +8. KMS and Gateway state can be captured in canonical, diffable artifacts. + +`run-upgrade-e2e.sh` additionally checks KMS key continuity, Gateway persistence, +old-to-new config/storage compatibility, and zero-failure CVM WireGuard traffic +during the Gateway replacement. Artifacts are written under `state/work/`. diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml index e12e74446..47e078cda 100644 --- a/test-suites/full-stack-compose/compose.yml +++ b/test-suites/full-stack-compose/compose.yml @@ -16,6 +16,7 @@ x-suite-env: &suite-env DSTACK_E2E_VMM_PORT: ${DSTACK_E2E_VMM_PORT:-29080} DSTACK_E2E_AUTH_PORT: ${DSTACK_E2E_AUTH_PORT:-28011} DSTACK_E2E_KMS_HOST_PORT: ${DSTACK_E2E_KMS_HOST_PORT:-28082} + DSTACK_E2E_KMS_RPC_DOMAIN: ${DSTACK_E2E_KMS_RPC_DOMAIN:-10.0.2.2.nip.io} DSTACK_E2E_GATEWAY_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} DSTACK_E2E_GATEWAY_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} @@ -37,6 +38,7 @@ x-suite-env: &suite-env DSTACK_E2E_APP_NAME: ${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} DSTACK_E2E_NAME_PREFIX: ${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} DSTACK_E2E_ALLOW_UNPINNED_IMAGE: ${DSTACK_E2E_ALLOW_UNPINNED_IMAGE:-false} + DSTACK_E2E_PHASE: ${DSTACK_E2E_PHASE:-full} services: init-config: @@ -147,7 +149,11 @@ services: restart: unless-stopped kms: - image: dstack-e2e-runtime:local + # The upgrade driver swaps this between the released Docker Hub image and + # the binary built from the current checkout. Keep the executable path + # configurable because the two images install it in different locations. + image: ${DSTACK_E2E_KMS_IMAGE:-dstack-e2e-runtime:local} + entrypoint: ["${DSTACK_E2E_KMS_BIN:-/workspace/target/release/dstack-kms}"] network_mode: host depends_on: init-config: @@ -159,19 +165,19 @@ services: - ../../dstack:/workspace:ro environment: RUST_LOG: ${DSTACK_E2E_KMS_RUST_LOG:-info,dstack_kms=info} - command: - - bash - - -lc - - exec /workspace/target/release/dstack-kms -c /suite-state/config/kms.toml + command: ["-c", "/suite-state/config/kms.toml"] healthcheck: - test: ["CMD-SHELL", "curl -kfsS https://127.0.0.1:${DSTACK_E2E_KMS_HOST_PORT:-28082}/metrics >/dev/null"] + test: ["CMD-SHELL", "timeout 3 sh -c \"printf 'GET /prpc/GetMeta?json HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n' | openssl s_client -quiet -connect 127.0.0.1:${DSTACK_E2E_KMS_HOST_PORT:-28082} 2>/dev/null | grep -q '^HTTP/1.1 200'\""] interval: 5s timeout: 3s retries: 30 restart: unless-stopped gateway: - image: dstack-e2e-runtime:local + # See kms above. The old image entrypoint is intentionally bypassed so + # both versions consume the exact same config and persistent data dir. + image: ${DSTACK_E2E_GATEWAY_IMAGE:-dstack-e2e-runtime:local} + entrypoint: ["${DSTACK_E2E_GATEWAY_BIN:-/workspace/target/release/dstack-gateway}"] privileged: true network_mode: host depends_on: @@ -185,12 +191,9 @@ services: - /lib/modules:/lib/modules:ro environment: RUST_LOG: ${DSTACK_E2E_GATEWAY_RUST_LOG:-info,dstack_gateway=debug,certbot=debug} - command: - - bash - - -lc - - exec /workspace/target/release/dstack-gateway -c /suite-state/config/gateway.toml + command: ["-c", "/suite-state/config/gateway.toml"] healthcheck: - test: ["CMD-SHELL", "curl -kfsS https://127.0.0.1:${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000}/health >/dev/null"] + test: ["CMD-SHELL", "timeout 3 sh -c \"printf 'GET /prpc/Info?json HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n' | openssl s_client -quiet -connect 127.0.0.1:${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} 2>/dev/null | grep -q '^HTTP/1.1 200'\""] interval: 5s timeout: 3s retries: 30 @@ -248,7 +251,7 @@ services: - ./scripts:/suite/scripts:ro - ../../dstack:/workspace:ro environment: - DSTACK_E2E_STATE_DIR: /suite-state + <<: *suite-env DSTACK_E2E_CLEAN_START: ${DSTACK_E2E_CLEAN_START:-true} DSTACK_E2E_CLEANUP_AFTER: ${DSTACK_E2E_CLEANUP_AFTER:-false} DSTACK_E2E_APP_VCPU: ${DSTACK_E2E_APP_VCPU:-2} diff --git a/test-suites/full-stack-compose/run-upgrade-e2e.sh b/test-suites/full-stack-compose/run-upgrade-e2e.sh new file mode 100755 index 000000000..ea17b308b --- /dev/null +++ b/test-suites/full-stack-compose/run-upgrade-e2e.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +REPO_DIR=$(cd -- "$SCRIPT_DIR/../.." && pwd) +STATE_DIR="$SCRIPT_DIR/state" +WORK_DIR="$STATE_DIR/work" +ENV_FILE=${DSTACK_E2E_ENV_FILE:-$SCRIPT_DIR/.env} + +setting() { + local name=$1 fallback=$2 line value + if [[ -v $name ]]; then + printf '%s' "${!name}" + return + fi + if [[ -f "$ENV_FILE" ]]; then + line=$(grep -E "^[[:space:]]*${name}=" "$ENV_FILE" | tail -n1 || true) + if [[ -n "$line" ]]; then + value=${line#*=} + value=${value%$'\r'} + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value=${value:1:${#value}-2} + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value=${value:1:${#value}-2} + fi + printf '%s' "$value" + return + fi + fi + printf '%s' "$fallback" +} + +OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE dstacktee/dstack-kms:0.5.7) +OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE dstacktee/dstack-gateway:0.5.8) +LATEST_RUNTIME_IMAGE=$(setting DSTACK_E2E_LATEST_RUNTIME_IMAGE dstack-e2e-runtime:local) +OLD_KMS_BIN=$(setting DSTACK_E2E_OLD_KMS_BIN /usr/local/bin/dstack-kms) +OLD_GATEWAY_BIN=$(setting DSTACK_E2E_OLD_GATEWAY_BIN /usr/local/bin/dstack-gateway) +LATEST_KMS_BIN=/workspace/target/release/dstack-kms +LATEST_GATEWAY_BIN=/workspace/target/release/dstack-gateway +KEEP_STACK=$(setting DSTACK_E2E_KEEP_STACK true) +CLEAN_STATE=$(setting DSTACK_E2E_UPGRADE_CLEAN_STATE true) +KMS_HOST_PORT=$(setting DSTACK_E2E_KMS_HOST_PORT 28082) +GATEWAY_RPC_HOST_PORT=$(setting DSTACK_E2E_GATEWAY_RPC_HOST_PORT 28000) +GATEWAY_WG_INTERFACE=$(setting DSTACK_E2E_GATEWAY_WG_INTERFACE wg-ds-e2e) +probe_pid="" + +COMPOSE=(docker compose -f "$SCRIPT_DIR/compose.yml") +if [[ -f "$ENV_FILE" ]]; then + COMPOSE=(docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR/compose.yml") +fi + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +compose() { + "${COMPOSE[@]}" "$@" +} + +need_bin() { + [[ -x "$1" ]] || die "missing current binary $1; build the release binaries documented in README.md" +} + +pull_released_image() { + local image=$1 component=$2 + log "pulling old $component image from Docker Hub: $image" + if ! docker pull "$image"; then + die "cannot pull $image from Docker Hub; set DSTACK_E2E_OLD_${component^^}_IMAGE only if an exact replacement was intentionally published" + fi +} + +wait_http() { + local name=$1 url=$2 deadline=$((SECONDS + 180)) + while (( SECONDS < deadline )); do + if curl -kfsS "$url" >/dev/null 2>&1; then + log "$name ready" + return 0 + fi + sleep 2 + done + compose logs --tail=200 "$name" >&2 || true + die "$name did not become ready: $url" +} + +container_version() { + local service=$1 binary=$2 stage=$3 id image version + id=$(compose ps -q "$service") + [[ -n "$id" ]] || die "$service container not found" + image=$(docker inspect -f '{{.Config.Image}}' "$id") + version=$(docker exec "$id" "$binary" --version 2>&1) + jq -n --arg stage "$stage" --arg service "$service" --arg image "$image" \ + --arg image_id "$(docker inspect -f '{{.Image}}' "$id")" --arg version "$version" \ + '{stage:$stage, service:$service, image:$image, image_id:$image_id, version:$version}' \ + > "$WORK_DIR/${service}-${stage}.version.json" + log "$service $stage: $version ($image)" +} + +run_phase() { + local phase=$1 + log "running runner phase: $phase" + DSTACK_E2E_PHASE="$phase" compose run --rm --no-deps \ + -e DSTACK_E2E_PHASE="$phase" runner +} + +reset_state() { + log "resetting Compose stack and E2E state" + compose down --remove-orphans >/dev/null 2>&1 || true + docker run --rm --privileged --network host alpine:3.22 sh -c \ + 'ip link delete "$1" 2>/dev/null || true' -- "$GATEWAY_WG_INTERFACE" + docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ + 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' +} + +on_exit() { + local rc=$? + if [[ -n "$probe_pid" ]] && kill -0 "$probe_pid" 2>/dev/null; then + touch "$WORK_DIR/network-probe.stop" 2>/dev/null || true + wait "$probe_pid" 2>/dev/null || true + fi + if (( rc != 0 )); then + log "upgrade E2E failed; recent service logs follow" + compose logs --tail=250 kms gateway vmm runner >&2 || true + fi + if [[ "$KEEP_STACK" != "true" ]]; then + compose down --remove-orphans >/dev/null 2>&1 || true + else + log "leaving stack running for inspection (DSTACK_E2E_KEEP_STACK=true)" + fi + exit "$rc" +} +trap on_exit EXIT + +main() { + need_bin "$REPO_DIR/dstack/target/release/dstack" + need_bin "$REPO_DIR/dstack/target/release/dstack-auth" + need_bin "$REPO_DIR/dstack/target/release/dstack-vmm" + need_bin "$REPO_DIR/dstack/target/release/dstack-kms" + need_bin "$REPO_DIR/dstack/target/release/dstack-gateway" + need_bin "$REPO_DIR/dstack/target/release/supervisor" + + pull_released_image "$OLD_KMS_IMAGE" kms + pull_released_image "$OLD_GATEWAY_IMAGE" gateway + [[ "$CLEAN_STATE" == "true" ]] && reset_state + + log "building support containers" + compose build init-config mock-cf-dns-api aesmd local-keyprovider + compose up init-config + compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth + + log "starting old KMS and Gateway" + DSTACK_E2E_KMS_IMAGE="$OLD_KMS_IMAGE" DSTACK_E2E_KMS_BIN="$OLD_KMS_BIN" \ + compose up -d --no-deps --force-recreate kms + wait_http kms "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" + DSTACK_E2E_GATEWAY_IMAGE="$OLD_GATEWAY_IMAGE" DSTACK_E2E_GATEWAY_BIN="$OLD_GATEWAY_BIN" \ + compose up -d --no-deps --force-recreate gateway + wait_http gateway "https://127.0.0.1:${GATEWAY_RPC_HOST_PORT}/prpc/Info?json" + compose up -d --no-deps vmm + mkdir -p "$WORK_DIR" + container_version kms "$OLD_KMS_BIN" old + container_version gateway "$OLD_GATEWAY_BIN" old + run_phase prepare-old + + log "upgrading KMS in place to the current checkout" + compose stop -t 30 kms + DSTACK_E2E_KMS_IMAGE="$LATEST_RUNTIME_IMAGE" DSTACK_E2E_KMS_BIN="$LATEST_KMS_BIN" \ + compose up -d --no-deps --force-recreate kms + wait_http kms "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" + container_version kms "$LATEST_KMS_BIN" latest + run_phase after-kms-upgrade + + log "starting strict direct-WireGuard availability probe" + rm -f "$WORK_DIR/network-probe.ready" "$WORK_DIR/network-probe.stop" + compose run --rm --no-deps runner /suite/scripts/network-probe.sh \ + >"$WORK_DIR/network-probe.log" 2>&1 & + probe_pid=$! + for _ in $(seq 1 120); do + if [[ -e "$WORK_DIR/network-probe.ready" ]]; then + break + fi + kill -0 "$probe_pid" 2>/dev/null || { + cat "$WORK_DIR/network-probe.log" >&2 + die "network probe exited during warmup" + } + sleep 1 + done + [[ -e "$WORK_DIR/network-probe.ready" ]] || die "network probe did not become ready" + + log "upgrading Gateway in place while CVM traffic remains active" + compose stop -t 30 gateway + DSTACK_E2E_GATEWAY_IMAGE="$LATEST_RUNTIME_IMAGE" DSTACK_E2E_GATEWAY_BIN="$LATEST_GATEWAY_BIN" \ + compose up -d --no-deps --force-recreate gateway + wait_http gateway "https://127.0.0.1:${GATEWAY_RPC_HOST_PORT}/prpc/Info?json" + container_version gateway "$LATEST_GATEWAY_BIN" latest + sleep 3 + touch "$WORK_DIR/network-probe.stop" + if ! wait "$probe_pid"; then + cat "$WORK_DIR/network-probe.log" >&2 + die "CVM WireGuard data plane was interrupted during Gateway upgrade" + fi + probe_pid="" + cat "$WORK_DIR/network-probe.log" + run_phase after-gateway-upgrade + + log "upgrade E2E success" + log "artifacts: $WORK_DIR" +} + +main "$@" diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py index 02d0ec42c..c5be78c57 100755 --- a/test-suites/full-stack-compose/scripts/app_compose.py +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -41,6 +41,14 @@ def nginx(args: argparse.Namespace) -> None: "runner": "docker-compose", "secure_time": False, } + if args.attestation_mode != "auto": + # A string v3 manifest makes old guests fail closed instead of silently + # ignoring the requirement. The current VMM resolves this field into + # the corresponding legacy/lite VM attestation config. + manifest["manifest_version"] = "3" + manifest["requirements"] = { + "tdx_measure_acpi_tables": args.attestation_mode == "legacy" + } write_manifest(Path(args.output), manifest) @@ -53,6 +61,11 @@ def main() -> None: p.add_argument("--name", required=True) p.add_argument("--app-image", required=True) p.add_argument("--output", required=True) + p.add_argument( + "--attestation-mode", + choices=("auto", "legacy", "lite"), + default="auto", + ) p.set_defaults(func=nginx) args = parser.parse_args() diff --git a/test-suites/full-stack-compose/scripts/network-probe.sh b/test-suites/full-stack-compose/scripts/network-probe.sh new file mode 100755 index 000000000..a8de95028 --- /dev/null +++ b/test-suites/full-stack-compose/scripts/network-probe.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} +WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} +TARGETS=${DSTACK_E2E_NETWORK_TARGETS:-$WORK_DIR/network-targets.tsv} +READY=$WORK_DIR/network-probe.ready +STOP=$WORK_DIR/network-probe.stop +RESULT=$WORK_DIR/network-probe.result.json +FAILURES=$WORK_DIR/network-probe.failures.log +INTERVAL=${DSTACK_E2E_NETWORK_PROBE_INTERVAL:-0.05} +REQUEST_TIMEOUT=${DSTACK_E2E_NETWORK_PROBE_TIMEOUT:-1} +WARMUP_SUCCESSES=${DSTACK_E2E_NETWORK_PROBE_WARMUP:-20} + +log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } +die() { log "ERROR: $*" >&2; exit 1; } + +[[ -s "$TARGETS" ]] || die "missing network targets: $TARGETS" +rm -f "$READY" "$STOP" "$RESULT" "$FAILURES" + +probe_all() { + local count=0 label ip body + while IFS=$'\t' read -r label ip; do + [[ -n "$label" && -n "$ip" ]] || continue + body=$(curl -fsS --max-time "$REQUEST_TIMEOUT" "http://${ip}:80/" 2>&1) \ + && grep -qi 'welcome to nginx' <<<"$body" \ + || return 1 + count=$((count + 1)) + done < "$TARGETS" + (( count > 0 )) +} + +log "warming up direct CVM WireGuard probes" +successes=0 +deadline=$((SECONDS + 120)) +while (( successes < WARMUP_SUCCESSES )); do + (( SECONDS < deadline )) || die "network probe warmup timed out" + if probe_all; then + successes=$((successes + 1)) + else + successes=0 + fi + sleep "$INTERVAL" +done + +started_at=$(date +%s%3N) +printf 'ready\n' > "$READY" +log "probe ready; strict zero-failure window started" + +attempts=0 +failures=0 +while [[ ! -e "$STOP" ]]; do + attempts=$((attempts + 1)) + if ! probe_all; then + failures=$((failures + 1)) + printf '%s\tattempt=%s\n' "$(date --iso-8601=ns)" "$attempts" >> "$FAILURES" + fi + sleep "$INTERVAL" +done + +finished_at=$(date +%s%3N) +jq -n \ + --argjson started_at_ms "$started_at" \ + --argjson finished_at_ms "$finished_at" \ + --argjson attempts "$attempts" \ + --argjson failures "$failures" \ + '{started_at_ms:$started_at_ms, finished_at_ms:$finished_at_ms, duration_ms:($finished_at_ms-$started_at_ms), attempts:$attempts, failures:$failures}' \ + | tee "$RESULT" + +(( attempts > 0 )) || die "network probe recorded no attempts" +(( failures == 0 )) || die "CVM WireGuard data plane had $failures failed probe cycle(s)" +log "zero-downtime CVM network assertion passed" diff --git a/test-suites/full-stack-compose/scripts/render-config.sh b/test-suites/full-stack-compose/scripts/render-config.sh index b22764269..168b19945 100755 --- a/test-suites/full-stack-compose/scripts/render-config.sh +++ b/test-suites/full-stack-compose/scripts/render-config.sh @@ -15,6 +15,7 @@ PLATFORM=${DSTACK_E2E_PLATFORM:-tdx} VMM_PORT=${DSTACK_E2E_VMM_PORT:-29080} AUTH_PORT=${DSTACK_E2E_AUTH_PORT:-28011} KMS_HOST_PORT=${DSTACK_E2E_KMS_HOST_PORT:-28082} +KMS_RPC_DOMAIN=${DSTACK_E2E_KMS_RPC_DOMAIN:-10.0.2.2.nip.io} GATEWAY_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} GATEWAY_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} GATEWAY_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} @@ -43,6 +44,9 @@ GATEWAY_DATA_DIR=${DSTACK_E2E_GATEWAY_DATA_DIR:-$STATE_DIR/gateway-data} mkdir -p \ "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" "$STATE_DIR/work" \ "$KMS_CERT_DIR" "$GATEWAY_CERT_DIR" "$GATEWAY_DATA_DIR" +# The host-side upgrade driver writes orchestration metadata and probe markers +# here while service containers also write artifacts as root. +chmod 0777 "$STATE_DIR/work" image_dir="$IMAGE_ROOT/$IMAGE_NAME" if [[ ! -d "$image_dir" ]]; then @@ -97,7 +101,7 @@ openssl_gen_kms_rpc_cert() { local dir=$1 tmp csr tmp=$(mktemp) csr="$dir/rpc.csr" - cat > "$tmp" <<'OPENSSL_CONF' + cat > "$tmp" </dev/null 2>&1 @@ -184,7 +189,7 @@ if [[ ! -s "$KMS_CERT_DIR/root-ca.key" || ! -s "$KMS_CERT_DIR/root-ca.crt" || \ openssl_gen_ca "$KMS_CERT_DIR/tmp-ca.key" "$KMS_CERT_DIR/tmp-ca.crt" "/O=Dstack/CN=Dstack Client Temp CA" openssl_gen_kms_rpc_cert "$KMS_CERT_DIR" write_k256_key "$KMS_CERT_DIR/root-k256.key" - printf '10.0.2.2' > "$KMS_CERT_DIR/rpc-domain" + printf '%s' "$KMS_RPC_DOMAIN" > "$KMS_CERT_DIR/rpc-domain" fi if [[ ! -s "$GATEWAY_CERT_DIR/gateway-rpc.key" || ! -s "$GATEWAY_CERT_DIR/gateway-rpc.crt" ]]; then @@ -255,6 +260,9 @@ url = "http://127.0.0.1:${AUTH_PORT}" [core.onboard] enabled = false auto_bootstrap_domain = "" +# Kept for compatibility with the 0.5.x config schema. Current KMS versions +# ignore this retired field. +quote_enabled = false address = "127.0.0.1" port = ${KMS_HOST_PORT} EOF_KMS @@ -357,9 +365,9 @@ interval = "5s" timeout = "10s" bootnode = "" data_dir = "${GATEWAY_DATA_DIR}" -persist_interval = "5s" +persist_interval = "1s" sync_connections_enabled = true -sync_connections_interval = "5s" +sync_connections_interval = "1s" EOF_GATEWAY cat > "$CONFIG_DIR/vmm.toml" <&2; exit 1; } need_bin() { [[ -x "$1" ]] || die "missing executable $1; run: cargo build --release -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms -p dstack-gateway -p supervisor" } -need_bin /workspace/target/release/dstack -need_bin /workspace/target/release/dstack-auth -need_bin /workspace/target/release/dstack-vmm -need_bin /workspace/target/release/dstack-kms -need_bin /workspace/target/release/dstack-gateway -need_bin /workspace/target/release/supervisor + +need_current_bins() { + need_bin /workspace/target/release/dstack + need_bin /workspace/target/release/dstack-auth + need_bin /workspace/target/release/dstack-vmm + need_bin /workspace/target/release/dstack-kms + need_bin /workspace/target/release/dstack-gateway + need_bin /workspace/target/release/supervisor +} + +require_tdx() { + [[ "$PLATFORM" == "tdx" ]] || die "$PHASE requires DSTACK_E2E_PLATFORM=tdx" +} wait_vmm() { log "waiting for VMM at $VMM_URL" @@ -60,7 +69,27 @@ clean_start() { while read -r id; do remove_vm "$id" done < <(vm_ids_by_prefix "$SUITE_PREFIX") - sleep 2 + for _ in $(seq 1 60); do + [[ -z "$(vm_ids_by_prefix "$SUITE_PREFIX")" ]] && return 0 + sleep 2 + done + die "stale VMs were not removed" +} + +print_vm_info_safe() { + jq '{ + id, + name, + status, + uptime, + app_id, + instance_id, + boot_progress, + boot_error, + shutdown_progress, + image_version, + events + }' <<<"$1" } wait_boot_done() { @@ -99,24 +128,40 @@ wait_boot_done() { die "timed out waiting for $label" } -print_vm_info_safe() { - jq '{ - id, - name, - status, - uptime, - app_id, - instance_id, - boot_progress, - boot_error, - shutdown_progress, - image_version, - events - }' <<<"$1" +wait_vm_stopped() { + local id=$1 deadline=$((SECONDS + 120)) + while (( SECONDS < deadline )); do + local status + status=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null | jq -r '.status // ""' || true) + if [[ "$status" != "running" && "$status" != "starting" ]]; then + return 0 + fi + sleep 2 + done + die "VM $id did not stop" } -compose_meta() { - jq -r '"\(.appId) \(.composeHash)"' +wait_vm_running() { + local id=$1 label=$2 deadline=$((SECONDS + 90)) last="" + while (( SECONDS < deadline )); do + local info status error + info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) + if [[ -n "$info" ]]; then + status=$(jq -r '.status // ""' <<<"$info") + error=$(jq -r '.boot_error // ""' <<<"$info") + if [[ "$status" != "$last" ]]; then + log "$label restart: status=${status:-unknown}" + last=$status + fi + if [[ -n "$error" && "$error" != "null" ]]; then + print_vm_info_safe "$info" >&2 || true + die "$label restart failed: $error" + fi + [[ "$status" == "running" ]] && return 0 + fi + sleep 1 + done + die "$label did not enter running state after restart" } register_app() { @@ -128,15 +173,10 @@ register_app() { python3 "${args[@]}" >/dev/null } -set_gateway_app_id() { - python3 /suite/scripts/allowlist.py set-gateway --path "$ALLOWLIST" --gateway-app-id "$1" >/dev/null -} - - wait_kms() { log "waiting for KMS at https://127.0.0.1:${KMS_HOST_PORT}" for _ in $(seq 1 90); do - if curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/metrics" >/dev/null 2>&1; then + if curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" >/dev/null 2>&1; then log "KMS ready" return 0 fi @@ -145,20 +185,49 @@ wait_kms() { die "KMS not ready" } +kms_rpc_get() { + local method=$1 + curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/prpc/KMS.${method}?json" +} + +kms_rpc_post() { + local method=$1 data=$2 + curl -kfsS -X POST -H 'Content-Type: application/json' \ + "https://127.0.0.1:${KMS_HOST_PORT}/prpc/KMS.${method}?json" \ + --data-raw "$data" +} + +capture_kms_identity() { + local stage=$1 app_id + app_id=$(cat "$WORK_DIR/legacy.app_id") + log "capturing KMS identity ($stage) for app_id=$app_id" + kms_rpc_get GetMeta | tee "$WORK_DIR/kms-${stage}.meta.raw.json" \ + | jq -S '{ca_cert, k256_pubkey}' > "$WORK_DIR/kms-${stage}.meta.json" + kms_rpc_post GetAppEnvEncryptPubKey "$(jq -cn --arg id "$app_id" '{app_id:$id}')" \ + | tee "$WORK_DIR/kms-${stage}.app-key.raw.json" \ + | jq -S '{public_key}' > "$WORK_DIR/kms-${stage}.app-key.json" + jq -e '.ca_cert != "" and .k256_pubkey != ""' "$WORK_DIR/kms-${stage}.meta.json" >/dev/null + jq -e '.public_key != ""' "$WORK_DIR/kms-${stage}.app-key.json" >/dev/null +} + +assert_kms_identity_unchanged() { + capture_kms_identity after + diff -u "$WORK_DIR/kms-before.meta.json" "$WORK_DIR/kms-after.meta.json" \ + || die "KMS CA or root k256 public key changed across upgrade" + diff -u "$WORK_DIR/kms-before.app-key.json" "$WORK_DIR/kms-after.app-key.json" \ + || die "per-app environment encryption key changed across KMS upgrade" + log "KMS persistent identity and per-app key are unchanged" +} + admin_curl() { local method=$1 - local data - if [[ $# -ge 2 ]]; then - data=$2 - else - data='{}' - fi + local data=${2:-'{}'} local out code out=$(mktemp) code=$(curl -sS -o "$out" -w '%{http_code}' -X POST \ -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ - -H "Content-Type: application/json" \ - "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.${method}" \ + -H 'Content-Type: application/json' \ + "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.${method}?json" \ --data-raw "$data" || true) if [[ "$code" =~ ^2 ]]; then cat "$out" @@ -173,8 +242,7 @@ admin_curl() { wait_gateway_admin() { log "waiting for Gateway admin API on 127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}" for _ in $(seq 1 90); do - if curl -fsS -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ - "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.Status" >/dev/null 2>&1; then + if admin_curl Status >/dev/null 2>&1; then log "Gateway admin ready" return 0 fi @@ -187,12 +255,15 @@ bootstrap_gateway_certbot() { local acme_url="http://127.0.0.1:${PEBBLE_HTTP_PORT}/dir" local cf_url="http://127.0.0.1:${MOCK_CF_HTTP_PORT}/client/v4" log "configuring Gateway certbot: acme=$acme_url cf=$cf_url domain=$BASE_DOMAIN" - admin_curl SetCertbotConfig "$(jq -cn --arg u "$acme_url" '{acme_url:$u}')" >/dev/null - # Idempotency for reruns: duplicate credentials/domains are harmless but noisy, so ignore create conflicts. + # Pebble's test certificates are short-lived. Keep the renewal threshold + # below their lifetime so a Gateway restart tests certificate restoration + # instead of intentionally rotating the certificate during startup. + admin_curl SetCertbotConfig "$(jq -cn --arg u "$acme_url" '{acme_url:$u, renew_before_expiration_secs:3600}')" >/dev/null admin_curl CreateDnsCredential "$(jq -cn --arg u "$cf_url" '{name:"mock-cloudflare", provider_type:"cloudflare", cf_api_token:"test-token", cf_api_url:$u, set_as_default:true, dns_txt_ttl:1, max_dns_wait:0}')" >/dev/null || true admin_curl AddZtDomain "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, port:443, priority:100}')" >/dev/null || true log "requesting wildcard cert for *.${BASE_DOMAIN}" - admin_curl RenewZtDomainCert "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, force:true}')" | tee "$WORK_DIR/renew-cert.json" + admin_curl RenewZtDomainCert "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, force:true}')" \ + | tee "$WORK_DIR/renew-cert.json" } wait_gateway_cert() { @@ -212,55 +283,154 @@ wait_gateway_cert() { die "timed out waiting for Gateway certificate" } +gateway_cert_fingerprint() { + local sni="gateway.${BASE_DOMAIN}" + echo | timeout 8 openssl s_client \ + -connect "127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ + -servername "$sni" 2>/dev/null \ + | openssl x509 -noout -fingerprint -sha256 2>/dev/null +} + +wait_gateway_persisted() { + log "waiting for Gateway WaveKV persistence" + local deadline=$((SECONDS + 90)) status + while (( SECONDS < deadline )); do + status=$(admin_curl WaveKvStatus 2>/dev/null || true) + if [[ -n "$status" ]] \ + && jq -e '.persistent.dirty == false and .persistent.n_keys > 0' <<<"$status" >/dev/null 2>&1; then + log "Gateway persistent store is clean ($(jq -r '.persistent.n_keys' <<<"$status") keys)" + return 0 + fi + sleep 1 + done + die "Gateway persistent store did not flush" +} + +capture_gateway_state() { + local stage=$1 + log "capturing Gateway durable state ($stage)" + admin_curl Status | tee "$WORK_DIR/gateway-${stage}.status.raw.json" \ + | jq -S '{uuid, hosts: ([.hosts[] | {instance_id, ip, app_id, base_domain}] | sort_by(.instance_id))}' \ + > "$WORK_DIR/gateway-${stage}.status.json" + admin_curl GetCertbotConfig | tee "$WORK_DIR/gateway-${stage}.certbot.raw.json" \ + | jq -S '{acme_url, renew_interval_secs, renew_before_expiration_secs, renew_timeout_secs}' \ + > "$WORK_DIR/gateway-${stage}.certbot.json" + admin_curl ListDnsCredentials | tee "$WORK_DIR/gateway-${stage}.dns.raw.json" \ + | jq -S '{default_id, credentials: ([.credentials[] | {id, name, provider_type, cf_zone_id, cf_api_url, dns_txt_ttl, max_dns_wait}] | sort_by(.id))}' \ + > "$WORK_DIR/gateway-${stage}.dns.json" + admin_curl ListZtDomains | tee "$WORK_DIR/gateway-${stage}.domains.raw.json" \ + | jq -S '{domains: ([.domains[] | .config] | sort_by(.domain))}' \ + > "$WORK_DIR/gateway-${stage}.domains.json" + gateway_cert_fingerprint > "$WORK_DIR/gateway-${stage}.cert-fingerprint.txt" +} + +assert_gateway_state_unchanged() { + capture_gateway_state after + local file + for file in status certbot dns domains; do + diff -u "$WORK_DIR/gateway-before.${file}.json" "$WORK_DIR/gateway-after.${file}.json" \ + || die "Gateway $file state changed across upgrade" + done + diff -u "$WORK_DIR/gateway-before.cert-fingerprint.txt" "$WORK_DIR/gateway-after.cert-fingerprint.txt" \ + || die "Gateway wildcard certificate changed across upgrade" + log "Gateway registrations, identity, certbot config, DNS/domain config and certificate are unchanged" +} + deploy_app() { + local label=$1 attestation_mode=$2 local out meta app_id hash vm_id - log "rendering nginx test app-compose" + log "rendering $label nginx app-compose (TDX $attestation_mode)" meta=$(python3 /suite/scripts/app_compose.py nginx \ - --name "$APP_NAME" \ + --name "${APP_NAME}-${label}" \ --app-image "$APP_IMAGE" \ - --output "$WORK_DIR/app-compose.json") + --attestation-mode "$attestation_mode" \ + --output "$WORK_DIR/${label}.app-compose.json") read -r app_id hash < <(jq -r '"\(.appId) \(.composeHash)"' <<<"$meta") + printf '%s' "$app_id" > "$WORK_DIR/${label}.app_id" local gateway_id gateway_id=$(jq -r '.gatewayAppId' "$ALLOWLIST") - log "registering test app in auth allowlist app_id=$app_id gateway_app_id=$gateway_id" + log "registering $label app in auth allowlist app_id=$app_id gateway_app_id=$gateway_id" register_app "$app_id" "$hash" "$gateway_id" - log "deploying nginx app CVM" + log "deploying $label nginx app CVM" out=$("${VMM_CLI[@]}" deploy \ - --name "${SUITE_PREFIX}-app" \ + --name "${SUITE_PREFIX}-${label}" \ --image "$IMAGE_NAME" \ - --compose "$WORK_DIR/app-compose.json" \ + --compose "$WORK_DIR/${label}.app-compose.json" \ --vcpu "${DSTACK_E2E_APP_VCPU:-2}" \ --memory "${DSTACK_E2E_APP_MEMORY:-2048}" \ --disk "${DSTACK_E2E_APP_DISK:-20}") - printf '%s\n' "$out" | tee "$WORK_DIR/app.deploy.log" + printf '%s\n' "$out" | tee "$WORK_DIR/${label}.deploy.log" vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) - [[ -n "$vm_id" ]] || die "failed to parse app VM id" - echo "$vm_id" > "$WORK_DIR/app.vm_id" - wait_boot_done "$vm_id" app "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" + [[ -n "$vm_id" ]] || die "failed to parse $label app VM id" + printf '%s' "$vm_id" > "$WORK_DIR/${label}.vm_id" + wait_boot_done "$vm_id" "$label" "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" + assert_attestation_mode "$label" "$attestation_mode" +} + +assert_attestation_mode() { + local label=$1 expected=$2 id sys_config actual + id=$(cat "$WORK_DIR/${label}.vm_id") + sys_config="$VM_DIR/$id/shared/.sys-config.json" + [[ -s "$sys_config" ]] || die "missing $label sys config: $sys_config" + actual=$(jq -r '.vm_config | fromjson | .tdx_attestation_variant // "legacy"' "$sys_config") + [[ "$actual" == "$expected" ]] \ + || die "$label resolved attestation mode is $actual, expected $expected" + jq -e '.kms_urls | length > 0' "$sys_config" >/dev/null \ + || die "$label has no KMS URL" + log "$label booted with resolved TDX mode=$actual and completed KMS key provisioning" +} + +restart_app_after_kms_upgrade() { + local label=$1 id + id=$(cat "$WORK_DIR/${label}.vm_id") + log "force-stopping $label CVM to require a fresh boot-time KMS key request" + "${VMM_CLI[@]}" stop "$id" --force >/dev/null + wait_vm_stopped "$id" + log "restarting $label CVM against upgraded KMS" + "${VMM_CLI[@]}" start "$id" >/dev/null + # The Start RPC returns before the supervisor updates the persisted VM state. + # Avoid treating that short, expected `exited` window as a failed boot. + wait_vm_running "$id" "$label" + wait_boot_done "$id" "$label" "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" + assert_attestation_mode "$label" legacy } verify_app_via_gateway() { - local app_info instance_id sni url deadline - app_info=$(cat "$WORK_DIR/app.info.json") + local label=$1 app_info instance_id sni url deadline + app_info=$(cat "$WORK_DIR/${label}.info.json") instance_id=$(jq -r '.instance_id // ""' <<<"$app_info") - [[ -n "$instance_id" && "$instance_id" != "null" ]] || die "app has no instance_id" + [[ -n "$instance_id" && "$instance_id" != "null" ]] || die "$label app has no instance_id" sni="${instance_id}-80.${BASE_DOMAIN}" url="https://${sni}:${GATEWAY_PROXY_HOST_PORT}/" - log "verifying app through Gateway: $url" + log "verifying $label app through Gateway: $url" deadline=$((SECONDS + ${DSTACK_E2E_APP_HTTP_TIMEOUT:-240})) while (( SECONDS < deadline )); do if curl -fsS -k --connect-to "${sni}:${GATEWAY_PROXY_HOST_PORT}:127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ - "$url" > "$WORK_DIR/app-http.out" 2>"$WORK_DIR/app-http.err"; then - if grep -qi "welcome to nginx" "$WORK_DIR/app-http.out"; then - log "Gateway -> app HTTP check passed" - head -c 160 "$WORK_DIR/app-http.out" | tr '\n' ' '; echo + "$url" > "$WORK_DIR/${label}.http.out" 2>"$WORK_DIR/${label}.http.err"; then + if grep -qi 'welcome to nginx' "$WORK_DIR/${label}.http.out"; then + log "$label Gateway -> app HTTP check passed" return 0 fi fi sleep 5 done - cat "$WORK_DIR/app-http.err" >&2 || true - die "timed out verifying app through Gateway" + cat "$WORK_DIR/${label}.http.err" >&2 || true + die "timed out verifying $label app through Gateway" +} + +write_network_targets() { + local status label instance ip + status=$(admin_curl Status) + : > "$WORK_DIR/network-targets.tsv" + for label in legacy lite; do + instance=$(jq -r '.instance_id' "$WORK_DIR/${label}.info.json") + ip=$(jq -r --arg id "$instance" '.hosts[] | select(.instance_id == $id) | .ip' <<<"$status") + [[ -n "$ip" && "$ip" != "null" ]] || die "Gateway has no WireGuard IP for $label ($instance)" + printf '%s\t%s\n' "$label" "$ip" >> "$WORK_DIR/network-targets.tsv" + curl -fsS --max-time 3 "http://${ip}:80/" | grep -qi 'welcome to nginx' \ + || die "cannot reach $label directly over WireGuard at $ip" + done + log "CVM WireGuard targets: $(tr '\n' ' ' < "$WORK_DIR/network-targets.tsv")" } cleanup_after() { @@ -268,28 +438,94 @@ cleanup_after() { return 0 fi log "DSTACK_E2E_CLEANUP_AFTER=true: removing suite VMs" - for f in app.vm_id gateway.vm_id kms.vm_id; do - if [[ -s "$WORK_DIR/$f" ]]; then - remove_vm "$(cat "$WORK_DIR/$f")" - fi + local f + for f in "$WORK_DIR"/*.vm_id; do + [[ -s "$f" ]] && remove_vm "$(cat "$f")" done } -main() { +prepare_common() { wait_vmm clean_start wait_kms wait_gateway_admin bootstrap_gateway_certbot wait_gateway_cert - deploy_app - verify_app_via_gateway +} + +phase_full() { + prepare_common + if [[ "$PLATFORM" == "tdx" ]]; then + deploy_app legacy legacy + deploy_app lite lite + verify_app_via_gateway legacy + verify_app_via_gateway lite + capture_kms_identity full + wait_gateway_persisted + capture_gateway_state full + write_network_targets + else + deploy_app app auto + verify_app_via_gateway app + fi log "E2E success" - log "VMM dashboard: $VMM_URL" - log "Gateway proxy: https://*.${BASE_DOMAIN}:${GATEWAY_PROXY_HOST_PORT} (connect to 127.0.0.1:${GATEWAY_PROXY_HOST_PORT})" - log "Gateway admin: http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT} (token in $STATE_DIR/gateway-admin-token)" - log "Work artifacts: $WORK_DIR" cleanup_after } +phase_prepare_old() { + require_tdx + prepare_common + deploy_app legacy legacy + verify_app_via_gateway legacy + capture_kms_identity before + printf 'ok\n' > "$WORK_DIR/prepare-old.done" + log "old KMS/Gateway phase complete" +} + +phase_after_kms_upgrade() { + require_tdx + wait_vmm + wait_kms + wait_gateway_admin + assert_kms_identity_unchanged + restart_app_after_kms_upgrade legacy + verify_app_via_gateway legacy + deploy_app lite lite + verify_app_via_gateway lite + wait_gateway_persisted + capture_gateway_state before + write_network_targets + printf 'ok\n' > "$WORK_DIR/after-kms-upgrade.done" + log "KMS upgrade and latest lite/legacy key-provisioning phase complete" +} + +phase_after_gateway_upgrade() { + require_tdx + wait_vmm + wait_kms + wait_gateway_admin + wait_gateway_cert + assert_gateway_state_unchanged + verify_app_via_gateway legacy + verify_app_via_gateway lite + write_network_targets + printf 'ok\n' > "$WORK_DIR/after-gateway-upgrade.done" + log "Gateway upgrade compatibility phase complete" + cleanup_after +} + +main() { + need_current_bins + log "running E2E phase=$PHASE" + case "$PHASE" in + full) phase_full ;; + prepare-old) phase_prepare_old ;; + after-kms-upgrade) phase_after_kms_upgrade ;; + after-gateway-upgrade) phase_after_gateway_upgrade ;; + *) die "unknown DSTACK_E2E_PHASE=$PHASE" ;; + esac + log "VMM dashboard: $VMM_URL" + log "Work artifacts: $WORK_DIR" +} + main "$@" From 4a5ae2d9e996d7f5ef09e7fc24c6f6c9dba76e22 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 13 Jul 2026 03:44:00 -0700 Subject: [PATCH 3/4] Make upgrade E2E production-faithful --- dstack/verifier/src/verification.rs | 58 +- dstack/vmm/src/tests/test_vmm_cli.py | 59 ++ dstack/vmm/src/vmm-cli.py | 5 + test-suites/full-stack-compose/.env.example | 60 +- test-suites/full-stack-compose/README.md | 261 ++--- test-suites/full-stack-compose/compose.yml | 117 +-- .../full-stack-compose/mock-cf-dns/server.py | 19 + .../full-stack-compose/run-upgrade-e2e.sh | 281 ++--- .../full-stack-compose/runtime/Dockerfile | 10 + .../runtime/requirements.txt | 8 + .../full-stack-compose/scripts/allowlist.py | 53 +- .../full-stack-compose/scripts/app_compose.py | 218 +++- .../scripts/network-probe.sh | 121 ++- .../scripts/render-config.sh | 440 ++------ .../full-stack-compose/scripts/runner.sh | 994 ++++++++++++------ 15 files changed, 1582 insertions(+), 1122 deletions(-) create mode 100644 dstack/vmm/src/tests/test_vmm_cli.py create mode 100644 test-suites/full-stack-compose/runtime/requirements.txt diff --git a/dstack/verifier/src/verification.rs b/dstack/verifier/src/verification.rs index 9faf03459..f4da88b93 100644 --- a/dstack/verifier/src/verification.rs +++ b/dstack/verifier/src/verification.rs @@ -392,6 +392,31 @@ impl CvmVerifier { .is_some_and(|digest| digest == expected)) } + fn prune_unlisted_image_files(extracted_dir: &Path, files_doc: &str) -> Result<()> { + let listed_files: Vec<&OsStr> = files_doc + .lines() + .flat_map(|line| line.split_whitespace().nth(1)) + .map(|s| s.as_ref()) + .collect(); + let files = fs_err::read_dir(extracted_dir).context("Failed to read directory")?; + for file in files { + let file = file.context("Failed to read directory entry")?; + let filename = file.file_name(); + // sha256sum.txt is the content-addressed OS identity and is needed + // again when a legacy TDX quote is verified from the cache. + if filename != OsStr::new("sha256sum.txt") + && !listed_files.contains(&filename.as_os_str()) + { + if file.path().is_dir() { + fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; + } else { + fs_err::remove_file(file.path()).context("Failed to remove file")?; + } + } + } + Ok(()) + } + fn tdx_acpi_hashes_from_event_log(event_log: &[TdxEvent]) -> Result { let rtmr0_events = event_log .iter() @@ -1108,23 +1133,7 @@ impl CvmVerifier { let sha256sum_path = extracted_dir.join("sha256sum.txt"); let files_doc = fs_err::read_to_string(&sha256sum_path).context("Failed to read sha256sum.txt")?; - let listed_files: Vec<&OsStr> = files_doc - .lines() - .flat_map(|line| line.split_whitespace().nth(1)) - .map(|s| s.as_ref()) - .collect(); - let files = fs_err::read_dir(&extracted_dir).context("Failed to read directory")?; - for file in files { - let file = file.context("Failed to read directory entry")?; - let filename = file.file_name(); - if !listed_files.contains(&filename.as_os_str()) { - if file.path().is_dir() { - fs_err::remove_dir_all(file.path()).context("Failed to remove directory")?; - } else { - fs_err::remove_file(file.path()).context("Failed to remove file")?; - } - } - } + Self::prune_unlisted_image_files(&extracted_dir, &files_doc)?; // All image modes are addressed by sha256(sha256sum.txt). Extra // measurement CBOR files are ordinary sha256sum.txt entries and do not @@ -1247,6 +1256,21 @@ mod tests { assert!(decode_key_provider_info(b"not json").is_none()); } + #[test] + fn image_cache_pruning_keeps_checksum_identity() { + let dir = tempfile::tempdir().expect("temp image directory"); + let files_doc = "00 metadata.json\n"; + fs_err::write(dir.path().join("sha256sum.txt"), files_doc).unwrap(); + fs_err::write(dir.path().join("metadata.json"), "{}").unwrap(); + fs_err::write(dir.path().join("unmeasured"), "remove me").unwrap(); + + CvmVerifier::prune_unlisted_image_files(dir.path(), files_doc).unwrap(); + + assert!(dir.path().join("sha256sum.txt").exists()); + assert!(dir.path().join("metadata.json").exists()); + assert!(!dir.path().join("unmeasured").exists()); + } + #[tokio::test] async fn verifies_sev_snp_attestation_fixture_without_image_download() { let request: VerificationRequest = diff --git a/dstack/vmm/src/tests/test_vmm_cli.py b/dstack/vmm/src/tests/test_vmm_cli.py new file mode 100644 index 000000000..f3c33c371 --- /dev/null +++ b/dstack/vmm/src/tests/test_vmm_cli.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +"""Focused regression tests for vmm-cli update request construction.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import unittest +from pathlib import Path + + +def load_vmm_cli(): + """Load the executable vmm-cli.py as a normal Python module.""" + path = Path(__file__).resolve().parents[1] / "vmm-cli.py" + spec = importlib.util.spec_from_file_location("dstack_vmm_cli", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +VMM_CLI = load_vmm_cli() + + +class UpdateVmTests(unittest.TestCase): + """Verify update flags are not silently discarded by the CLI.""" + + def test_kms_urls_are_sent_to_upgrade_app(self) -> None: + """Send both the update flag and requested KMS URL list.""" + cli = VMM_CLI.VmmCLI("http://127.0.0.1:8080") + calls: list[tuple[str, dict]] = [] + cli.rpc_call = lambda method, params=None: calls.append((method, params)) or {} + + with contextlib.redirect_stdout(io.StringIO()): + cli.update_vm("vm-id", kms_urls=["https://kms.example:8000"]) + + self.assertEqual( + calls, + [ + ( + "UpgradeApp", + { + "id": "vm-id", + "update_kms_urls": True, + "kms_urls": ["https://kms.example:8000"], + }, + ) + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/vmm/src/vmm-cli.py b/dstack/vmm/src/vmm-cli.py index 290ee2440..e393e1323 100755 --- a/dstack/vmm/src/vmm-cli.py +++ b/dstack/vmm/src/vmm-cli.py @@ -1127,6 +1127,11 @@ def update_vm( upgrade_params["user_config"] = user_config updates.append("user config") + if kms_urls is not None: + upgrade_params["update_kms_urls"] = True + upgrade_params["kms_urls"] = kms_urls + updates.append(f"KMS URLs ({len(kms_urls)})") + # handle port updates - only update if --port or --no-ports is specified if no_ports or ports is not None: if no_ports: diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example index 079299a95..28c113a2e 100644 --- a/test-suites/full-stack-compose/.env.example +++ b/test-suites/full-stack-compose/.env.example @@ -1,63 +1,63 @@ # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -# Directory containing unpacked dstack guest images: //{bzImage,initramfs...,rootfs...,digest.txt} -# In meta-dstack this is usually ../../../build/images from this directory, or an absolute path. -DSTACK_E2E_IMAGE_STORE=../../../build/images +# Directory containing unpacked dstack guest images: +# //{bzImage,initramfs...,rootfs...,digest.txt,sha256sum.txt} +DSTACK_E2E_IMAGE_STORE=../../../meta-dstack/build/images DSTACK_E2E_IMAGE_NAME=dstack-0.6.0 DSTACK_E2E_PLATFORM=tdx -# App image used inside the CVMs launched by this suite. +# Exact released upgrade sources. Both are digest-pinned on Docker Hub and +# checked by their in-image --version output before any CVM is launched. +DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d +DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411 DSTACK_E2E_APP_IMAGE=nginx:alpine -# Released images used by run-upgrade-e2e.sh. The KMS case intentionally pins -# exactly 0.5.7; the script fails before starting the stack if Docker Hub does -# not provide that tag. Gateway 0.5.8 is the default published upgrade source. -DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.7 -DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8 - # Mock ACME/DNS + externally visible test domain for Gateway SNI. DSTACK_E2E_BASE_DOMAIN=e2e.test DSTACK_E2E_PEBBLE_IMAGE=kvin/pebble:latest -# Host ports. Defaults are chosen to avoid the common dstackup ports. +# Host ports. Defaults are chosen to avoid common dstackup ports. DSTACK_E2E_VMM_PORT=29080 DSTACK_E2E_AUTH_PORT=28011 -DSTACK_E2E_KMS_HOST_PORT=28082 -# 0.5.x KMS encodes rpc-domain as a DNS SAN even when it looks like an IP. -# Use a DNS name that resolves to QEMU's 10.0.2.2 host gateway. -DSTACK_E2E_KMS_RPC_DOMAIN=10.0.2.2.nip.io -DSTACK_E2E_GATEWAY_RPC_HOST_PORT=28000 -DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT=28001 -DSTACK_E2E_GATEWAY_PROXY_HOST_PORT=28443 -DSTACK_E2E_GATEWAY_WG_HOST_PORT=28120 +DSTACK_E2E_ARTIFACT_PORT=38081 +DSTACK_E2E_KMS_OLD_HOST_PORT=28082 +DSTACK_E2E_KMS_LATEST_HOST_PORT=28083 +# KMS 0.5.8 encodes this value as a DNS SAN. It must resolve to the test host +# from BOTH the host-side deployment client and each CVM. If omitted, the +# renderer derives .nip.io (for example below). +# DSTACK_E2E_KMS_RPC_DOMAIN=192.168.1.90.nip.io +DSTACK_E2E_GATEWAY1_RPC_HOST_PORT=28000 +DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT=28001 +DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT=28443 +DSTACK_E2E_GATEWAY1_WG_HOST_PORT=28120 +DSTACK_E2E_GATEWAY2_RPC_HOST_PORT=28100 +DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT=28101 +DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT=28543 +DSTACK_E2E_GATEWAY2_WG_HOST_PORT=28121 DSTACK_E2E_KEY_PROVIDER_PORT=13443 DSTACK_E2E_HOST_API_PORT=20011 DSTACK_E2E_MOCK_CF_HTTP_PORT=38080 DSTACK_E2E_PEBBLE_HTTP_PORT=34000 DSTACK_E2E_PEBBLE_MGMT_PORT=35000 -# Dev-compose gateway policy. "any" matches the tdxlab host-side gateway mode: -# app CVMs skip gateway app-id pinning but still use KMS authorization for apps. -DSTACK_E2E_GATEWAY_APP_ID=any -DSTACK_E2E_GATEWAY_WG_INTERFACE=wg-ds-e2e -DSTACK_E2E_GATEWAY_WG_IP=10.8.0.1/16 -DSTACK_E2E_GATEWAY_WG_RESERVED_NET=10.8.0.1/32 -DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE=10.8.0.0/18 +# Optional exact 20-byte Gateway application ID. If omitted, the renderer +# deterministically derives one. "any" is intentionally rejected. +# DSTACK_E2E_GATEWAY_APP_ID=0123456789abcdef0123456789abcdef01234567 # VMM/QEMU knobs. DSTACK_E2E_CID_START=15000 DSTACK_E2E_QGS_PORT=4050 DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64 -# SGX QCNL config used by AESMD. On hosts with a local PCCS, point this at the -# host config, e.g. /etc/sgx_default_qcnl.conf. +# SGX QCNL config used by the Local-Key-Provider that seals each KMS CVM disk. DSTACK_E2E_QCNL_CONF=../../dstack/key-provider-build/sgx_default_qcnl.conf # Runner behaviour. DSTACK_E2E_CLEAN_START=true DSTACK_E2E_CLEANUP_AFTER=false - -# Upgrade-driver behaviour. Keep the completed stack by default for inspection. DSTACK_E2E_UPGRADE_CLEAN_STATE=true DSTACK_E2E_KEEP_STACK=true + +# Set only when the current x86_64-musl KMS/Gateway binaries were already built. +DSTACK_E2E_SKIP_CURRENT_BUILD=false diff --git a/test-suites/full-stack-compose/README.md b/test-suites/full-stack-compose/README.md index 993fed942..6329cb042 100644 --- a/test-suites/full-stack-compose/README.md +++ b/test-suites/full-stack-compose/README.md @@ -3,173 +3,138 @@ SPDX-FileCopyrightText: © 2026 Phala Network SPDX-License-Identifier: Apache-2.0 --> -# dstack full-stack compose E2E suite +# Production-compatible KMS/Gateway upgrade E2E -This directory contains a Docker Compose based test suite that mirrors the -`tdxlab`/`dstack-k8s deploy/compose` shape: host-side services run as normal -processes/containers, and QEMU is used only for the real app CVM under test. +This suite runs the stateful services and applications in **real TDX CVMs**. +Docker Compose is used only for host infrastructure (VMM, authorization, +Local-Key-Provider, image server, and mock ACME/DNS): ```text -docker compose - ├─ mock-cf-dns-api # tiny Cloudflare API + TXT DNS mock - ├─ pebble # ACME test server (HTTP-capable custom Pebble image) - ├─ aesmd + local-keyprovider - ├─ dstack-auth # KMS auth webhook backed by auth-allowlist.json - ├─ dstack-kms # host-side dev KMS, not a CVM - ├─ dstack-gateway # host-side dev Gateway, not a CVM - ├─ dstack-vmm # launches real TDX/SNP app CVMs via QEMU - └─ runner - ├─ configures Gateway certbot to use mock CF + Pebble - ├─ deploys legacy and lite TDX nginx CVMs - ├─ verifies KMS key provisioning and HTTPS Gateway -> app routing - └─ snapshots durable KMS/Gateway state for upgrade assertions +host Docker Compose + ├─ dstack-vmm + dstack-auth + ├─ AESMD + SGX Local-Key-Provider + ├─ verified OS/image artifact server + └─ Pebble + mock Cloudflare DNS + +real TDX CVMs + ├─ KMS 0.5.8 (Local-Key-Provider, quote-enabled onboarding) + ├─ current KMS (new CVM, quote-attested replication from 0.5.8) + ├─ two-node Gateway 0.5.8 cluster, rolled one node at a time to current + ├─ forced-legacy nginx app CVM + └─ forced-lite nginx app CVM ``` -It is intended for a **real TDX/SGX host** (or SEV-SNP host after setting the -platform/image knobs). It is not a pure unit-test environment; QEMU still starts -a confidential VM for the app. +The upgrade scenario intentionally rejects the development trust settings that +would make a passing result irrelevant to production: + +- KMS 0.5.8 uses `quote_enabled = true`. +- Both KMS manifests explicitly set `enforce_self_authorization = true`. +- KMS OS image verification is enabled and downloads a checksum-verified image + archive from an initially empty cache. +- KMS root keys and Gateway TLS identity are created inside TDX CVMs; the suite + never generates or injects them on the host. +- The authorization webhook pins one OS digest, exact KMS aggregate + measurements, the quote-derived physical TDX device ID, exact app compose + hashes, and a concrete 20-byte Gateway app ID. `allowAnyDevice = true` and + `gatewayAppId = "any"` are rejected. +- Container references in measured app manifests are content-addressed Docker + image IDs. Tags are used only as the Docker Hub/build inputs that are pulled, + inspected, saved, and imported before boot. ## Prerequisites -1. From this directory, build the host binaries from the dstack checkout: +1. Build the host-side control binaries: ```bash - cargo build --release \ - --manifest-path ../../dstack/Cargo.toml \ - -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms \ - -p dstack-gateway -p supervisor + cargo build --release --manifest-path ../../dstack/Cargo.toml \ + -p dstack-cli -p dstack-auth -p dstack-vmm -p supervisor ``` -2. Make sure the host has the required devices/services: + The driver builds current KMS and Gateway binaries for + `x86_64-unknown-linux-musl` and places them in test-only container images. + The static binaries are layered on the released 0.5.8 production runtime; + the KMS production builder still uses the same Debian/QEMU revision, and the + current Gateway entrypoint is copied from this checkout. The built binaries + and in-CVM version endpoints must report the current workspace version and + Git revision, so Docker cache or an old binary cannot silently turn the + upgrade into a no-op. - - `/dev/kvm` - - Intel TDX support and QGS for TDX (`DSTACK_E2E_QGS_PORT`, default `4050`) - - `/dev/sgx_enclave` and `/dev/sgx_provision` for the local key provider - - an SGX QCNL config that works on the host. By default the suite uses - `../../dstack/key-provider-build/sgx_default_qcnl.conf`; on hosts with a local - PCCS, set `DSTACK_E2E_QCNL_CONF=/etc/sgx_default_qcnl.conf`. - - `/dev/vhost-vsock` for guest↔host vsock services - - a TDX-capable `qemu-system-x86_64` available inside the runtime container. - The default runtime image follows `../dstack-k8s/deploy/compose` and - installs QEMU from `ppa:kobuk-team/tdx-release`. - - WireGuard kernel support on the host. The Gateway service runs privileged - with host networking and creates `DSTACK_E2E_GATEWAY_WG_INTERFACE`. +2. Provide a TDX/SGX host with: -3. Provide an unpacked guest image store. From `meta-dstack` this is usually - `../../../build/images`; otherwise copy `.env.example` and set: + - `/dev/kvm`, `/dev/vhost-vsock`, Intel TDX, and QGS (default port `4050`) + - `/dev/sgx_enclave` and `/dev/sgx_provision` + - a working SGX QCNL/PCCS configuration + - an IPv4/DNS name for the host that is reachable from both the host-side + deployment client and QEMU user-networked CVMs; the suite derives + `.nip.io`, or accepts `DSTACK_E2E_KMS_RPC_DOMAIN` + - Docker and WireGuard kernel support - ```bash - cp .env.example .env - $EDITOR .env - ``` - -The normal latest-only run needs no published KMS/Gateway image: -`dstack-gateway` and `dstack-kms` run from the local `target/release/` binaries. -The upgrade run additionally pulls its old KMS/Gateway images from Docker Hub. +3. Provide an unpacked dstack image directory containing `digest.txt` and + `sha256sum.txt`. Copy `.env.example` to `.env` when paths or ports differ. ## Run -From this directory, run: - -```bash -DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build --abort-on-container-exit runner -``` - -On TDX this latest-only path deploys two otherwise identical CVMs. Their v3 -manifest requirements force `tdx_measure_acpi_tables=true` (legacy) and -`false` (lite), respectively. Both must reach `boot_progress=done`, which is -after the boot-time `GetAppKey` request, and both must serve nginx through the -Gateway. - -## Run the upgrade scenario - -The host-side driver is required because the runner container cannot replace -its own KMS/Gateway dependencies: - ```bash DOCKER_BUILDKIT=0 ./run-upgrade-e2e.sh ``` -Defaults: - -- old KMS: `dstacktee/dstack-kms:0.5.7` (pulled from Docker Hub) -- old Gateway: `dstacktee/dstack-gateway:0.5.8` (pulled from Docker Hub) -- latest KMS, Gateway, VMM, guest image: current checkout/build - -The driver deliberately fails during the image-pull preflight if the exact old -image is unavailable. Override images only for intentional matrix runs: - -```bash -DSTACK_E2E_OLD_KMS_IMAGE=dstacktee/dstack-kms:0.5.6 \ -DSTACK_E2E_OLD_GATEWAY_IMAGE=dstacktee/dstack-gateway:0.5.8 \ - ./run-upgrade-e2e.sh -``` - -The upgrade sequence is: - -1. Boot a forced-legacy CVM against the old KMS and old Gateway. -2. Record the KMS CA, root k256 public key, and that app's derived environment - encryption public key. -3. Replace only KMS with the current binary while retaining its cert/key dir. -4. Require the recorded identities to be byte-for-byte stable; force-reboot the - legacy CVM so it must provision keys from the new KMS; then boot a forced-lite - CVM from scratch against the new KMS. -5. Flush and record Gateway WaveKV state: node UUID, CVM registrations/IPs, - certbot config, DNS credential, ZT domain, and wildcard cert fingerprint. -6. Start rapid direct HTTP probes to both CVM WireGuard IPs, replace only the - Gateway process, and require **zero failed probe cycles**. -7. Require all recorded Gateway state and the cert fingerprint to survive and - verify both SNI routes through the new Gateway. - -The zero-downtime assertion is specifically the CVM WireGuard data plane. The -Gateway process owns the public TLS listener, so a single-node stop/start does -not claim that the external listener itself has zero downtime. External routing -is asserted immediately before and after the upgrade. - -For iterative debugging, keep the stack running: - -```bash -DOCKER_BUILDKIT=0 docker compose --env-file .env -f compose.yml up --build -``` - -The runner leaves app VMs running by default for inspection. Set -`DSTACK_E2E_CLEANUP_AFTER=true` to remove suite VMs at the end. Stale VMs whose -names start with `DSTACK_E2E_NAME_PREFIX` are removed at the start when -`DSTACK_E2E_CLEAN_START=true`. - -## What is verified - -The runner checks: - -1. Host-side KMS is reachable over TLS. -2. Host-side Gateway Admin API accepts configuration. -3. Gateway obtains a wildcard certificate for `*.DSTACK_E2E_BASE_DOMAIN` from - Pebble using the mock Cloudflare DNS API. -4. VMM API becomes reachable. -5. Real nginx app CVMs boot with `kms_enabled=true` and - `gateway_enabled=true`. -6. On TDX, v3 requirements resolve one CVM to legacy attestation and one to - lite; both complete boot-time KMS key provisioning. -7. `curl -k` through each Gateway SNI route returns the nginx welcome page. -8. KMS and Gateway state can be captured in canonical, diffable artifacts. - -`run-upgrade-e2e.sh` additionally checks KMS key continuity, Gateway persistence, -old-to-new config/storage compatibility, and zero-failure CVM WireGuard traffic -during the Gateway replacement. - -Artifacts are written under `state/work/`. - -## Notes / flexibility gaps - -- This is a dev compose KMS/Gateway setup. KMS and Gateway key material is - generated under `state/`; the allowlist returns `gatewayAppId = "any"`, which - matches the host-side tdxlab-style deployment and avoids requiring Gateway to - be a CVM. -- `dstackup install` is systemd-oriented; this suite renders `vmm.toml`, - `kms.toml`, `gateway.toml`, and `auth-allowlist.json` itself. -- Running VMM in a container depends on the container image having a QEMU build - that supports the target TEE platform. The runtime Dockerfile uses Intel's - kobuk-team TDX PPA for QEMU, matching the dstack-k8s compose deployment. -- Host API is vsock-only by design. The suite therefore uses privileged host - networking and a separate `DSTACK_E2E_HOST_API_PORT` to avoid conflicts with - other VMM instances. +The defaults pull these released images from Docker Hub: + +- `dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d` +- `dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411` + +The driver checks each released binary's `--version` output before deploying +anything. Set `DSTACK_E2E_SKIP_CURRENT_BUILD=true` only when the current musl +binaries were already built. + +## Upgrade sequence and assertions + +### KMS 0.5.8 to current + +1. Deploy KMS 0.5.8 as a Local-Key-Provider TDX CVM with no pre-created keys. +2. Call `Onboard.GetAttestationInfo`, authorize its exact `mrAggregated`, then + bootstrap it. The bootstrap response must contain non-empty attestation. +3. Boot the forced-legacy app through KMS 0.5.8 and record the KMS CA SPKI, + root k256 public key, and the app's derived environment-encryption public + key. (The expected onboarding path reissues the self-signed CA certificate, + so its DER bytes/expiry are not a stable identity; its private key/SPKI is.) +4. Deploy current KMS in a new Local-Key-Provider TDX CVM, authorize its exact + measurement, and call `Onboard.Onboard` against KMS 0.5.8. This exercises the + production RA-TLS root-key replication path; it is not a directory copy. +5. Require all recorded identities/derived keys to be byte-for-byte equal, + stop KMS 0.5.8, and force the legacy app to boot against only current KMS. +6. Boot a fresh forced-lite app against current KMS. Both modes must reach + `boot_progress=done`, which occurs after boot-time `GetAppKey` succeeds. + +The artifact server journal must contain two successful GETs for the exact +measured OS archive (one from each fresh KMS data disk). VM logs are also +rejected if they say image verification or self-authorization is disabled. + +### Gateway 0.5.8 to current + +1. Deploy two Gateway 0.5.8 TDX CVMs under one pinned Gateway app ID. Both get + app keys and RA-TLS certificates from KMS; their environment is encrypted by + VMM/KMS. +2. Form a WaveKV cluster, configure Pebble/Cloudflare mock DNS once, and require + the second node to receive the configuration and wildcard certificate. +3. Verify the legacy and lite app SNI routes through **both** Gateway nodes. +4. Snapshot each node's UUID, CVM registrations/IPs, certbot configuration, DNS + credential metadata, ZT domains, and wildcard-certificate fingerprint. +5. Run a rapid HA probe that alternates the preferred physical Gateway for + every legacy/lite request and falls back to the other node in the same cycle. +6. Stop, update the measured app compose, and restart node 1 on current KMS; + require it to carry both apps and retain all durable state. Repeat for node 2. +7. Require zero failed HA cycles and unchanged durable state/certificates. +8. Force certificate issuance through each upgraded node against a mock DNS API + that rejects anything except the original bearer token. This proves the DNS + credential value survived even though the current admin API redacts it. + +This is a production-style **rolling two-node** zero-downtime assertion. The +suite does not claim that rebooting a single Gateway CVM can preserve that +node's listener; availability is maintained by the peer, as it must be in a +production rollout. + +Artifacts, manifests, version headers, canonical state snapshots, probe +results, attestation information, and VM logs are written to `state/work/`. +The stack is retained by default for inspection; set +`DSTACK_E2E_KEEP_STACK=false` and/or `DSTACK_E2E_CLEANUP_AFTER=true` to clean it. diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml index 47e078cda..b42560acf 100644 --- a/test-suites/full-stack-compose/compose.yml +++ b/test-suites/full-stack-compose/compose.yml @@ -15,17 +15,19 @@ x-suite-env: &suite-env DSTACK_E2E_PLATFORM: ${DSTACK_E2E_PLATFORM:-tdx} DSTACK_E2E_VMM_PORT: ${DSTACK_E2E_VMM_PORT:-29080} DSTACK_E2E_AUTH_PORT: ${DSTACK_E2E_AUTH_PORT:-28011} - DSTACK_E2E_KMS_HOST_PORT: ${DSTACK_E2E_KMS_HOST_PORT:-28082} - DSTACK_E2E_KMS_RPC_DOMAIN: ${DSTACK_E2E_KMS_RPC_DOMAIN:-10.0.2.2.nip.io} - DSTACK_E2E_GATEWAY_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} - DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} - DSTACK_E2E_GATEWAY_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} - DSTACK_E2E_GATEWAY_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY_WG_HOST_PORT:-28120} - DSTACK_E2E_GATEWAY_WG_INTERFACE: ${DSTACK_E2E_GATEWAY_WG_INTERFACE:-wg-ds-e2e} - DSTACK_E2E_GATEWAY_WG_IP: ${DSTACK_E2E_GATEWAY_WG_IP:-10.8.0.1/16} - DSTACK_E2E_GATEWAY_WG_RESERVED_NET: ${DSTACK_E2E_GATEWAY_WG_RESERVED_NET:-10.8.0.1/32} - DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE: ${DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE:-10.8.0.0/18} - DSTACK_E2E_GATEWAY_APP_ID: ${DSTACK_E2E_GATEWAY_APP_ID:-any} + DSTACK_E2E_ARTIFACT_PORT: ${DSTACK_E2E_ARTIFACT_PORT:-38081} + DSTACK_E2E_KMS_OLD_HOST_PORT: ${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} + DSTACK_E2E_KMS_LATEST_HOST_PORT: ${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} + DSTACK_E2E_KMS_RPC_DOMAIN: ${DSTACK_E2E_KMS_RPC_DOMAIN:-} + DSTACK_E2E_GATEWAY1_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} + DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} + DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} + DSTACK_E2E_GATEWAY1_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} + DSTACK_E2E_GATEWAY2_RPC_HOST_PORT: ${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} + DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT: ${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} + DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT: ${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} + DSTACK_E2E_GATEWAY2_WG_HOST_PORT: ${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} + DSTACK_E2E_GATEWAY_APP_ID: ${DSTACK_E2E_GATEWAY_APP_ID:-} DSTACK_E2E_KEY_PROVIDER_PORT: ${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} DSTACK_E2E_HOST_API_PORT: ${DSTACK_E2E_HOST_API_PORT:-20011} DSTACK_E2E_CID_START: ${DSTACK_E2E_CID_START:-15000} @@ -34,11 +36,10 @@ x-suite-env: &suite-env DSTACK_E2E_BASE_DOMAIN: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test} DSTACK_E2E_MOCK_CF_HTTP_PORT: ${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} DSTACK_E2E_PEBBLE_HTTP_PORT: ${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} - DSTACK_E2E_APP_IMAGE: ${DSTACK_E2E_APP_IMAGE:-nginx:alpine} DSTACK_E2E_APP_NAME: ${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} DSTACK_E2E_NAME_PREFIX: ${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} - DSTACK_E2E_ALLOW_UNPINNED_IMAGE: ${DSTACK_E2E_ALLOW_UNPINNED_IMAGE:-false} - DSTACK_E2E_PHASE: ${DSTACK_E2E_PHASE:-full} + DSTACK_E2E_PHASE: ${DSTACK_E2E_PHASE:-upgrade} + DSTACK_E2E_CURRENT_VERSION: ${DSTACK_E2E_CURRENT_VERSION:-0.6.0} services: init-config: @@ -49,7 +50,7 @@ services: volumes: - ./state:/suite-state - ./scripts:/suite/scripts:ro - - ${DSTACK_E2E_IMAGE_STORE:-../../../build/images}:/images:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro environment: <<: *suite-env command: ["/suite/scripts/render-config.sh"] @@ -62,11 +63,12 @@ services: e2e-net: ipv4_address: 172.31.0.10 ports: - - "127.0.0.1:${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080}:8080" + - "0.0.0.0:${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080}:8080" environment: PORT: 8080 DEBUG: ${DSTACK_E2E_MOCK_DEBUG:-false} MOCK_CF_ZONES: ${DSTACK_E2E_BASE_DOMAIN:-e2e.test},test,local + MOCK_CF_API_TOKEN: test-token healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health')"] interval: 5s @@ -82,8 +84,8 @@ services: e2e-net: ipv4_address: 172.31.0.11 ports: - - "127.0.0.1:${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000}:14000" - - "127.0.0.1:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000" + - "0.0.0.0:${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000}:14000" + - "0.0.0.0:${DSTACK_E2E_PEBBLE_MGMT_PORT:-35000}:15000" environment: PEBBLE_VA_NOSLEEP: "1" PEBBLE_VA_ALWAYS_VALID: "1" @@ -131,6 +133,15 @@ services: - aesmd-sock:/var/run/aesmd/ ports: - "127.0.0.1:${DSTACK_E2E_KEY_PROVIDER_PORT:-13443}:3443" + healthcheck: + # The provider is a raw TCP service without an HTTP health endpoint. + # Opening the socket proves that the production SGX enclave was loaded; + # the container cannot listen while AESMD/DCAP provisioning is broken. + test: ["CMD-SHELL", "timeout 2 bash -c '/dev/null | grep -q '^HTTP/1.1 200'\""] - interval: 5s - timeout: 3s - retries: 30 + - exec /workspace/target/release/dstack-auth --config /suite-state/config/auth-allowlist.json --address 0.0.0.0 --port ${DSTACK_E2E_AUTH_PORT:-28011} restart: unless-stopped - gateway: - # See kms above. The old image entrypoint is intentionally bypassed so - # both versions consume the exact same config and persistent data dir. - image: ${DSTACK_E2E_GATEWAY_IMAGE:-dstack-e2e-runtime:local} - entrypoint: ["${DSTACK_E2E_GATEWAY_BIN:-/workspace/target/release/dstack-gateway}"] - privileged: true + artifacts: + image: dstack-e2e-runtime:local network_mode: host depends_on: init-config: condition: service_completed_successfully - kms: - condition: service_healthy volumes: - ./state:/suite-state - - ../../dstack:/workspace:ro - - /lib/modules:/lib/modules:ro - environment: - RUST_LOG: ${DSTACK_E2E_GATEWAY_RUST_LOG:-info,dstack_gateway=debug,certbot=debug} - command: ["-c", "/suite-state/config/gateway.toml"] + # Keep an in-band request journal. The runner uses the exact successful + # OS archive GETs as evidence that both KMS versions exercised image + # download/verification; VM serial logs do not include container stdout. + command: + - bash + - -lc + - >- + exec python3 -u -m http.server "${DSTACK_E2E_ARTIFACT_PORT:-38081}" + --bind 0.0.0.0 --directory /suite-state/artifacts + 2> >(tee -a /suite-state/work/artifacts-access.log >&2) healthcheck: - test: ["CMD-SHELL", "timeout 3 sh -c \"printf 'GET /prpc/Info?json HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n' | openssl s_client -quiet -connect 127.0.0.1:${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} 2>/dev/null | grep -q '^HTTP/1.1 200'\""] + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${DSTACK_E2E_ARTIFACT_PORT:-38081}/"] interval: 5s timeout: 3s - retries: 30 + retries: 20 restart: unless-stopped vmm: @@ -212,16 +197,14 @@ services: condition: service_completed_successfully auth: condition: service_started - kms: - condition: service_healthy - gateway: + artifacts: condition: service_healthy local-keyprovider: - condition: service_started + condition: service_healthy volumes: - ./state:/suite-state - ../../dstack:/workspace:ro - - ${DSTACK_E2E_IMAGE_STORE:-../../../build/images}:/images:ro + - ${DSTACK_E2E_IMAGE_STORE:-../../../meta-dstack/build/images}:/images:ro - /dev:/dev - /lib/modules:/lib/modules:ro environment: @@ -240,9 +223,9 @@ services: condition: service_healthy pebble: condition: service_healthy - kms: - condition: service_healthy - gateway: + auth: + condition: service_started + artifacts: condition: service_healthy vmm: condition: service_started diff --git a/test-suites/full-stack-compose/mock-cf-dns/server.py b/test-suites/full-stack-compose/mock-cf-dns/server.py index 344522ccd..b3bc07d51 100755 --- a/test-suites/full-stack-compose/mock-cf-dns/server.py +++ b/test-suites/full-stack-compose/mock-cf-dns/server.py @@ -73,6 +73,19 @@ def _record_content(payload: dict[str, Any]) -> str: return "" +def _authorized(handler: BaseHTTPRequestHandler) -> bool: + """Require the configured Cloudflare bearer token for API operations.""" + expected = os.environ.get("MOCK_CF_API_TOKEN", "test-token") + if handler.headers.get("Authorization", "") == f"Bearer {expected}": + return True + _json( + handler, + 403, + {"success": False, "errors": [{"message": "invalid API token"}]}, + ) + return False + + class Handler(BaseHTTPRequestHandler): """Handle the mock Cloudflare HTTP API.""" @@ -95,6 +108,8 @@ def do_GET(self) -> None: # noqa: N802 with STATE_LOCK: _json(self, 200, {"records": RECORDS}) return + if path.startswith("/client/v4/") and not _authorized(self): + return if path == "/client/v4/zones": zones = _zones() _json( @@ -149,6 +164,8 @@ def do_POST(self) -> None: # noqa: N802 {"success": False, "errors": [{"message": f"not found: {path}"}]}, ) return + if not _authorized(self): + return length = int(self.headers.get("content-length", "0") or "0") payload = json.loads(self.rfile.read(length) or b"{}") with STATE_LOCK: @@ -178,6 +195,8 @@ def do_DELETE(self) -> None: # noqa: N802 {"success": False, "errors": [{"message": f"not found: {path}"}]}, ) return + if not _authorized(self): + return record_id = urllib.parse.unquote(m.group(2)) with STATE_LOCK: before = len(RECORDS) diff --git a/test-suites/full-stack-compose/run-upgrade-e2e.sh b/test-suites/full-stack-compose/run-upgrade-e2e.sh index ea17b308b..b0ee43b55 100755 --- a/test-suites/full-stack-compose/run-upgrade-e2e.sh +++ b/test-suites/full-stack-compose/run-upgrade-e2e.sh @@ -6,7 +6,6 @@ set -euo pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) REPO_DIR=$(cd -- "$SCRIPT_DIR/../.." && pwd) STATE_DIR="$SCRIPT_DIR/state" -WORK_DIR="$STATE_DIR/work" ENV_FILE=${DSTACK_E2E_ENV_FILE:-$SCRIPT_DIR/.env} setting() { @@ -32,19 +31,22 @@ setting() { printf '%s' "$fallback" } -OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE dstacktee/dstack-kms:0.5.7) -OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE dstacktee/dstack-gateway:0.5.8) -LATEST_RUNTIME_IMAGE=$(setting DSTACK_E2E_LATEST_RUNTIME_IMAGE dstack-e2e-runtime:local) -OLD_KMS_BIN=$(setting DSTACK_E2E_OLD_KMS_BIN /usr/local/bin/dstack-kms) -OLD_GATEWAY_BIN=$(setting DSTACK_E2E_OLD_GATEWAY_BIN /usr/local/bin/dstack-gateway) -LATEST_KMS_BIN=/workspace/target/release/dstack-kms -LATEST_GATEWAY_BIN=/workspace/target/release/dstack-gateway +OLD_KMS_IMAGE=$(setting DSTACK_E2E_OLD_KMS_IMAGE \ + dstacktee/dstack-kms:0.5.8@sha256:9650dcb47dad0065470f432f00e78e012912214ef1a5b1d7272918817e61a26d) +OLD_GATEWAY_IMAGE=$(setting DSTACK_E2E_OLD_GATEWAY_IMAGE \ + dstacktee/dstack-gateway:0.5.8@sha256:6eb1dc1a5000f37cc5b0322d3fdb71e7f2e31859b5e3a611634919278cee2411) +APP_IMAGE=$(setting DSTACK_E2E_APP_IMAGE nginx:alpine) KEEP_STACK=$(setting DSTACK_E2E_KEEP_STACK true) CLEAN_STATE=$(setting DSTACK_E2E_UPGRADE_CLEAN_STATE true) -KMS_HOST_PORT=$(setting DSTACK_E2E_KMS_HOST_PORT 28082) -GATEWAY_RPC_HOST_PORT=$(setting DSTACK_E2E_GATEWAY_RPC_HOST_PORT 28000) -GATEWAY_WG_INTERFACE=$(setting DSTACK_E2E_GATEWAY_WG_INTERFACE wg-ds-e2e) -probe_pid="" +SKIP_BUILD=$(setting DSTACK_E2E_SKIP_CURRENT_BUILD false) +# dstack's build metadata deliberately embeds a 20-hex abbreviated revision. +CURRENT_REV=$(git -C "$REPO_DIR" rev-parse --short=20 HEAD) +CURRENT_VERSION=$(sed -n 's/^version = "\([^"]*\)"/\1/p' \ + "$REPO_DIR/dstack/Cargo.toml" | head -n1) +[[ -n "$CURRENT_VERSION" ]] || { + echo "ERROR: could not read current workspace version" >&2 + exit 1 +} COMPOSE=(docker compose -f "$SCRIPT_DIR/compose.yml") if [[ -f "$ENV_FILE" ]]; then @@ -53,76 +55,148 @@ fi log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } die() { log "ERROR: $*" >&2; exit 1; } - -compose() { - "${COMPOSE[@]}" "$@" -} +compose() { "${COMPOSE[@]}" "$@"; } need_bin() { - [[ -x "$1" ]] || die "missing current binary $1; build the release binaries documented in README.md" + [[ -x "$1" ]] || die "missing executable $1" } pull_released_image() { local image=$1 component=$2 - log "pulling old $component image from Docker Hub: $image" - if ! docker pull "$image"; then - die "cannot pull $image from Docker Hub; set DSTACK_E2E_OLD_${component^^}_IMAGE only if an exact replacement was intentionally published" - fi + log "pulling released $component image from Docker Hub: $image" + docker pull "$image" || die "cannot pull released $component image $image" } -wait_http() { - local name=$1 url=$2 deadline=$((SECONDS + 180)) - while (( SECONDS < deadline )); do - if curl -kfsS "$url" >/dev/null 2>&1; then - log "$name ready" - return 0 - fi - sleep 2 - done - compose logs --tail=200 "$name" >&2 || true - die "$name did not become ready: $url" +reset_state() { + log "resetting Compose stack and E2E state" + compose down --remove-orphans >/dev/null 2>&1 || true + docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ + 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' } -container_version() { - local service=$1 binary=$2 stage=$3 id image version - id=$(compose ps -q "$service") - [[ -n "$id" ]] || die "$service container not found" - image=$(docker inspect -f '{{.Config.Image}}' "$id") - version=$(docker exec "$id" "$binary" --version 2>&1) - jq -n --arg stage "$stage" --arg service "$service" --arg image "$image" \ - --arg image_id "$(docker inspect -f '{{.Image}}' "$id")" --arg version "$version" \ - '{stage:$stage, service:$service, image:$image, image_id:$image_id, version:$version}' \ - > "$WORK_DIR/${service}-${stage}.version.json" - log "$service $stage: $version ($image)" +build_current_binaries() { + if [[ "$SKIP_BUILD" == true ]]; then + log "using prebuilt current musl KMS/Gateway binaries" + else + log "building current KMS/Gateway as production-style static musl binaries" + cargo build --manifest-path "$REPO_DIR/dstack/Cargo.toml" \ + --release --target x86_64-unknown-linux-musl \ + -p dstack-kms -p dstack-gateway + fi + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" + need_bin "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway" } -run_phase() { - local phase=$1 - log "running runner phase: $phase" - DSTACK_E2E_PHASE="$phase" compose run --rm --no-deps \ - -e DSTACK_E2E_PHASE="$phase" runner +prepare_container_artifacts() { + local artifact_dir="$STATE_DIR/artifacts/images" + local context_dir="$STATE_DIR/image-build" + local rev current_kms_image current_gateway_image + local old_kms_id old_gateway_id current_kms_id current_gateway_id app_id + rev=$(git -C "$REPO_DIR" rev-parse --short=16 HEAD) + current_kms_image="dstack-e2e-kms-current:${rev}" + current_gateway_image="dstack-e2e-gateway-current:${rev}" + mkdir -p "$artifact_dir" "$context_dir/kms" "$context_dir/gateway" + + cp "$REPO_DIR/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms" \ + "$context_dir/kms/dstack-kms" + cat > "$context_dir/kms/Dockerfile" < "$context_dir/gateway/Dockerfile" < "$STATE_DIR/artifacts/images.env" </dev/null \ + || die "$OLD_KMS_IMAGE is not KMS 0.5.8" + grep -E '^dstack-gateway v0\.5\.8 \(git:' "$STATE_DIR/work/gateway-old.version.txt" >/dev/null \ + || die "$OLD_GATEWAY_IMAGE is not Gateway 0.5.8" + grep -E "^dstack-kms v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "locally built KMS is not current v$CURRENT_VERSION" + grep -E "^dstack-gateway v${CURRENT_VERSION//./\\.} \\(git:" \ + "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "locally built Gateway is not current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$STATE_DIR/work/kms-current.version.txt" >/dev/null \ + || die "KMS binary was not built from current revision $CURRENT_REV" + grep -F "$CURRENT_REV" "$STATE_DIR/work/gateway-current.version.txt" >/dev/null \ + || die "Gateway binary was not built from current revision $CURRENT_REV" } -reset_state() { - log "resetting Compose stack and E2E state" - compose down --remove-orphans >/dev/null 2>&1 || true - docker run --rm --privileged --network host alpine:3.22 sh -c \ - 'ip link delete "$1" 2>/dev/null || true' -- "$GATEWAY_WG_INTERFACE" - docker run --rm -v "$STATE_DIR:/state" alpine:3.22 sh -c \ - 'find /state -mindepth 1 ! -name .gitkeep -exec rm -rf {} +' +wait_local_key_provider() { + local port deadline status + port=$(setting DSTACK_E2E_KEY_PROVIDER_PORT 13443) + deadline=$((SECONDS + 120)) + log "waiting for production SGX Local-Key-Provider on 127.0.0.1:${port}" + while (( SECONDS < deadline )); do + status=$(compose ps --format json local-keyprovider 2>/dev/null \ + | jq -rs 'map(select(.Service == "local-keyprovider"))[0].Health // ""' 2>/dev/null \ + || true) + if [[ "$status" == healthy ]]; then + log "Local-Key-Provider enclave is healthy" + return 0 + fi + sleep 2 + done + compose logs --tail=200 aesmd local-keyprovider >&2 || true + die "Local-Key-Provider did not become healthy; fix SGX/DCAP/PCCS provisioning rather than disabling attestation" } on_exit() { local rc=$? - if [[ -n "$probe_pid" ]] && kill -0 "$probe_pid" 2>/dev/null; then - touch "$WORK_DIR/network-probe.stop" 2>/dev/null || true - wait "$probe_pid" 2>/dev/null || true - fi if (( rc != 0 )); then - log "upgrade E2E failed; recent service logs follow" - compose logs --tail=250 kms gateway vmm runner >&2 || true + log "upgrade E2E failed; recent infrastructure logs follow" + compose logs --tail=250 auth artifacts vmm runner >&2 || true fi - if [[ "$KEEP_STACK" != "true" ]]; then + if [[ "$KEEP_STACK" != true ]]; then compose down --remove-orphans >/dev/null 2>&1 || true else log "leaving stack running for inspection (DSTACK_E2E_KEEP_STACK=true)" @@ -135,75 +209,36 @@ main() { need_bin "$REPO_DIR/dstack/target/release/dstack" need_bin "$REPO_DIR/dstack/target/release/dstack-auth" need_bin "$REPO_DIR/dstack/target/release/dstack-vmm" - need_bin "$REPO_DIR/dstack/target/release/dstack-kms" - need_bin "$REPO_DIR/dstack/target/release/dstack-gateway" need_bin "$REPO_DIR/dstack/target/release/supervisor" pull_released_image "$OLD_KMS_IMAGE" kms pull_released_image "$OLD_GATEWAY_IMAGE" gateway - [[ "$CLEAN_STATE" == "true" ]] && reset_state + pull_released_image "$APP_IMAGE" application + build_current_binaries + [[ "$CLEAN_STATE" == true ]] && reset_state - log "building support containers" + log "building E2E infrastructure" compose build init-config mock-cf-dns-api aesmd local-keyprovider - compose up init-config - compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth - - log "starting old KMS and Gateway" - DSTACK_E2E_KMS_IMAGE="$OLD_KMS_IMAGE" DSTACK_E2E_KMS_BIN="$OLD_KMS_BIN" \ - compose up -d --no-deps --force-recreate kms - wait_http kms "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" - DSTACK_E2E_GATEWAY_IMAGE="$OLD_GATEWAY_IMAGE" DSTACK_E2E_GATEWAY_BIN="$OLD_GATEWAY_BIN" \ - compose up -d --no-deps --force-recreate gateway - wait_http gateway "https://127.0.0.1:${GATEWAY_RPC_HOST_PORT}/prpc/Info?json" - compose up -d --no-deps vmm - mkdir -p "$WORK_DIR" - container_version kms "$OLD_KMS_BIN" old - container_version gateway "$OLD_GATEWAY_BIN" old - run_phase prepare-old - - log "upgrading KMS in place to the current checkout" - compose stop -t 30 kms - DSTACK_E2E_KMS_IMAGE="$LATEST_RUNTIME_IMAGE" DSTACK_E2E_KMS_BIN="$LATEST_KMS_BIN" \ - compose up -d --no-deps --force-recreate kms - wait_http kms "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" - container_version kms "$LATEST_KMS_BIN" latest - run_phase after-kms-upgrade - - log "starting strict direct-WireGuard availability probe" - rm -f "$WORK_DIR/network-probe.ready" "$WORK_DIR/network-probe.stop" - compose run --rm --no-deps runner /suite/scripts/network-probe.sh \ - >"$WORK_DIR/network-probe.log" 2>&1 & - probe_pid=$! - for _ in $(seq 1 120); do - if [[ -e "$WORK_DIR/network-probe.ready" ]]; then - break - fi - kill -0 "$probe_pid" 2>/dev/null || { - cat "$WORK_DIR/network-probe.log" >&2 - die "network probe exited during warmup" - } - sleep 1 - done - [[ -e "$WORK_DIR/network-probe.ready" ]] || die "network probe did not become ready" - - log "upgrading Gateway in place while CVM traffic remains active" - compose stop -t 30 gateway - DSTACK_E2E_GATEWAY_IMAGE="$LATEST_RUNTIME_IMAGE" DSTACK_E2E_GATEWAY_BIN="$LATEST_GATEWAY_BIN" \ - compose up -d --no-deps --force-recreate gateway - wait_http gateway "https://127.0.0.1:${GATEWAY_RPC_HOST_PORT}/prpc/Info?json" - container_version gateway "$LATEST_GATEWAY_BIN" latest - sleep 3 - touch "$WORK_DIR/network-probe.stop" - if ! wait "$probe_pid"; then - cat "$WORK_DIR/network-probe.log" >&2 - die "CVM WireGuard data plane was interrupted during Gateway upgrade" - fi - probe_pid="" - cat "$WORK_DIR/network-probe.log" - run_phase after-gateway-upgrade + compose run --rm init-config + prepare_container_artifacts + + log "starting authorization, artifact, attestation, ACME and VMM infrastructure" + compose up -d mock-cf-dns-api pebble aesmd local-keyprovider auth artifacts + wait_local_key_provider + compose up -d vmm + + log "running production-compatible KMS/Gateway rolling-upgrade E2E" + # Keep the complete runner transcript in state/work even though the runner is + # an ephemeral Compose container. This is especially important for failures + # during deployment, before a per-VM log file exists. + DSTACK_E2E_PHASE=upgrade compose run --rm --no-deps \ + -e DSTACK_E2E_PHASE=upgrade \ + -e DSTACK_E2E_CURRENT_VERSION="$CURRENT_VERSION" \ + -e DSTACK_E2E_CURRENT_REV="$CURRENT_REV" runner \ + 2>&1 | tee "$STATE_DIR/work/runner.log" log "upgrade E2E success" - log "artifacts: $WORK_DIR" + log "artifacts: $STATE_DIR/work" } main "$@" diff --git a/test-suites/full-stack-compose/runtime/Dockerfile b/test-suites/full-stack-compose/runtime/Dockerfile index 12cd5f737..3d6fc1ae2 100644 --- a/test-suites/full-stack-compose/runtime/Dockerfile +++ b/test-suites/full-stack-compose/runtime/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update && \ openssl \ procps \ python3 \ + python3-venv \ qemu-system-x86 \ qemu-utils \ socat \ @@ -31,6 +32,15 @@ RUN apt-get update && \ xz-utils \ && rm -rf /var/lib/apt/lists/* +# vmm-cli encrypts deployment environment variables with the KMS key and +# verifies the KMS signature before a production CVM is created. +COPY requirements.txt /tmp/vmm-requirements.txt +RUN python3 -m venv /opt/dstack-venv && \ + /opt/dstack-venv/bin/pip install --no-cache-dir \ + -r /tmp/vmm-requirements.txt && \ + rm /tmp/vmm-requirements.txt +ENV PATH="/opt/dstack-venv/bin:${PATH}" + WORKDIR /suite ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["bash"] diff --git a/test-suites/full-stack-compose/runtime/requirements.txt b/test-suites/full-stack-compose/runtime/requirements.txt new file mode 100644 index 000000000..4c016c51d --- /dev/null +++ b/test-suites/full-stack-compose/runtime/requirements.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +# Keep the runtime subset synchronized with dstack/vmm/requirements.txt. +cryptography==46.0.7 +eth-hash[pycryptodome]==0.7.1 +eth-keys==0.7.0 +eth-utils==5.3.0 diff --git a/test-suites/full-stack-compose/scripts/allowlist.py b/test-suites/full-stack-compose/scripts/allowlist.py index 8f599169b..d67865cdf 100755 --- a/test-suites/full-stack-compose/scripts/allowlist.py +++ b/test-suites/full-stack-compose/scripts/allowlist.py @@ -20,6 +20,14 @@ def norm_hex(value: str) -> str: return value.lower() +def checked_hex(value: str, length: int, label: str) -> str: + """Normalize and validate a fixed-width hexadecimal authorization value.""" + value = norm_hex(value) + if len(value) != length or any(ch not in "0123456789abcdef" for ch in value): + raise ValueError(f"invalid {label}: expected {length} hexadecimal characters") + return value + + def load(path: Path) -> dict: """Load an allowlist or return an empty default policy.""" if path.exists(): @@ -27,7 +35,7 @@ def load(path: Path) -> dict: return { "osImages": [], "gatewayAppId": "", - "kms": {"mrAggregated": [], "devices": [], "allowAnyDevice": True}, + "kms": {"mrAggregated": [], "devices": [], "allowAnyDevice": False}, "apps": {}, } @@ -44,18 +52,21 @@ def add_app(args: argparse.Namespace) -> None: path = Path(args.path) data = load(path) apps = data.setdefault("apps", {}) - app_id = norm_hex(args.app_id) - compose_hash = norm_hex(args.compose_hash) + app_id = checked_hex(args.app_id, 40, "app ID") + compose_hash = checked_hex(args.compose_hash, 64, "compose hash") + device_id = checked_hex(args.device_id, 64, "device ID") entry = apps.setdefault( - app_id, {"composeHashes": [], "devices": [], "allowAnyDevice": True} + app_id, {"composeHashes": [], "devices": [], "allowAnyDevice": False} ) hashes = entry.setdefault("composeHashes", []) if not any(norm_hex(h) == compose_hash for h in hashes): hashes.append(compose_hash) - entry.setdefault("devices", []) - entry["allowAnyDevice"] = True + devices = entry.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + entry["allowAnyDevice"] = False if args.gateway_app_id is not None: - data["gatewayAppId"] = norm_hex(args.gateway_app_id) + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") save(path, data) print(json.dumps({"appId": app_id, "composeHash": compose_hash, "path": str(path)})) @@ -64,11 +75,31 @@ def set_gateway(args: argparse.Namespace) -> None: """Set the allowlisted Gateway application ID.""" path = Path(args.path) data = load(path) - data["gatewayAppId"] = norm_hex(args.gateway_app_id) + data["gatewayAppId"] = checked_hex(args.gateway_app_id, 40, "Gateway app ID") save(path, data) print(json.dumps({"gatewayAppId": data["gatewayAppId"], "path": str(path)})) +def add_kms(args: argparse.Namespace) -> None: + """Authorize one exact KMS aggregate measurement.""" + path = Path(args.path) + data = load(path) + kms = data.setdefault( + "kms", {"mrAggregated": [], "devices": [], "allowAnyDevice": False} + ) + measurement = checked_hex(args.mr_aggregated, 64, "KMS mrAggregated") + device_id = checked_hex(args.device_id, 64, "device ID") + measurements = kms.setdefault("mrAggregated", []) + if not any(norm_hex(item) == measurement for item in measurements): + measurements.append(measurement) + devices = kms.setdefault("devices", []) + if not any(norm_hex(item) == device_id for item in devices): + devices.append(device_id) + kms["allowAnyDevice"] = False + save(path, data) + print(json.dumps({"mrAggregated": measurement, "path": str(path)})) + + def main() -> None: """Parse command-line arguments and update the allowlist.""" parser = argparse.ArgumentParser() @@ -77,12 +108,18 @@ def main() -> None: p.add_argument("--path", required=True) p.add_argument("--app-id", required=True) p.add_argument("--compose-hash", required=True) + p.add_argument("--device-id", required=True) p.add_argument("--gateway-app-id") p.set_defaults(func=add_app) p = sub.add_parser("set-gateway") p.add_argument("--path", required=True) p.add_argument("--gateway-app-id", required=True) p.set_defaults(func=set_gateway) + p = sub.add_parser("add-kms") + p.add_argument("--path", required=True) + p.add_argument("--mr-aggregated", required=True) + p.add_argument("--device-id", required=True) + p.set_defaults(func=add_kms) args = parser.parse_args() args.func(args) diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py index c5be78c57..6a8cd579a 100755 --- a/test-suites/full-stack-compose/scripts/app_compose.py +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -"""Render app-compose manifests used by the full-stack E2E suite.""" +"""Render measured app-compose manifests used by the full-stack E2E suite.""" from __future__ import annotations @@ -12,39 +12,199 @@ from pathlib import Path +def image_loader(artifact_port: int, archive: str) -> str: + """Return a pre-launch script that imports one content-addressed image.""" + return f"""set -eu +archive=/tmp/{archive} +curl -fL --retry 5 --retry-delay 1 \\ + http://10.0.2.2:{artifact_port}/images/{archive} -o "$archive" +docker load -i "$archive" +rm -f "$archive" +""" + + def write_manifest(path: Path, manifest: dict) -> None: - """Write a manifest and print its derived identifiers.""" + """Write a manifest and print its byte-exact derived identifiers.""" body = json.dumps(manifest, indent=2, sort_keys=True) + "\n" path.write_text(body) digest = hashlib.sha256(body.encode()).hexdigest() print(json.dumps({"path": str(path), "composeHash": digest, "appId": digest[:40]})) -def nginx(args: argparse.Namespace) -> None: - """Render an nginx app-compose manifest.""" - docker_compose = f"""services: - web: - image: {args.app_image} - ports: - - "80:80" -""" - manifest = { +def common_manifest(name: str, docker_compose: str, pre_launch_script: str) -> dict: + """Return the production CVM manifest baseline used by service apps.""" + return { "docker_compose_file": docker_compose, - "gateway_enabled": True, - "kms_enabled": True, + "gateway_enabled": False, + "kms_enabled": False, "local_key_provider_enabled": False, "manifest_version": 2, - "name": args.name, - "no_instance_id": False, + "name": name, + "no_instance_id": True, + "pre_launch_script": pre_launch_script, "public_logs": True, "public_sysinfo": True, "runner": "docker-compose", - "secure_time": False, + "secure_time": True, + "storage_fs": "ext4", } + + +def kms(args: argparse.Namespace) -> None: + """Render a KMS CVM using the production onboarding and verifier path.""" + config = f"""[rpc] +address = "0.0.0.0" +port = 8000 + +[rpc.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + +[rpc.tls.mutual] +ca_certs = "/kms/certs/tmp-ca.crt" +mandatory = false + +[core] +cert_dir = "/kms/certs" +admin_token_hash = "" +pccs_url = "" +enforce_self_authorization = true + +[core.image] +verify = true +cache_dir = "/kms/images" +download_url = "http://10.0.2.2:{args.artifact_port}/os/mr_{{OS_IMAGE_HASH}}.tar.gz" +download_timeout = "2m" + +[core.metrics] +enabled = true + +[core.auth_api] +type = "webhook" + +[core.auth_api.webhook] +url = "http://10.0.2.2:{args.auth_port}" + +[core.onboard] +enabled = true +auto_bootstrap_domain = "" +# Required by KMS 0.5.8. Current KMS ignores this retired field. +quote_enabled = true +address = "0.0.0.0" +port = 8000 +""" + docker_compose = f"""services: + kms: + image: {args.image_ref} + volumes: + - kms-volume:/kms + - /var/run/dstack.sock:/var/run/dstack.sock + ports: + - "8000:8000" + restart: unless-stopped + configs: + - source: kms_config + target: /kms/kms.toml + command: sh -c 'mkdir -p /kms/certs /kms/images && exec dstack-kms -c /kms/kms.toml' + +volumes: + kms-volume: + +configs: + kms_config: + content: | +""" + "".join(f" {line}\n" for line in config.splitlines()) + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["local_key_provider_enabled"] = True + write_manifest(Path(args.output), manifest) + + +def gateway(args: argparse.Namespace) -> None: + """Render one production Gateway CVM node.""" + docker_compose = f"""services: + gateway: + image: {args.image_ref} + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - /dstack:/dstack + - data:/data + network_mode: host + privileged: true + environment: + - WG_ENDPOINT=${{WG_ENDPOINT}} + - MY_URL=${{MY_URL}} + - BOOTNODE_URL=${{BOOTNODE_URL}} + - WG_IP=${{WG_IP}} + - WG_RESERVED_NET=${{WG_RESERVED_NET}} + - WG_CLIENT_RANGE=${{WG_CLIENT_RANGE}} + - NODE_ID=${{NODE_ID}} + - RUST_LOG=info,certbot=debug + - PCCS_URL=${{PCCS_URL}} + - RPC_DOMAIN=${{RPC_DOMAIN}} + - PROXY_LISTEN_PORT=443 + - PROXY_WORKERS=4 + - SYNC_INTERVAL=1s + - SYNC_TIMEOUT=10s + - SYNC_PERSIST_INTERVAL=1s + - SYNC_CONNECTIONS_ENABLED=true + - SYNC_CONNECTIONS_INTERVAL=1s + - ADMIN_LISTEN_ADDR=0.0.0.0 + - ADMIN_LISTEN_PORT=8001 + - ADMIN_API_TOKEN=${{ADMIN_API_TOKEN}} + restart: always + +volumes: + data: +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest["kms_enabled"] = True + manifest["allowed_envs"] = [ + "ADMIN_API_TOKEN", + "BOOTNODE_URL", + "MY_URL", + "NODE_ID", + "PCCS_URL", + "RPC_DOMAIN", + "WG_CLIENT_RANGE", + "WG_ENDPOINT", + "WG_IP", + "WG_RESERVED_NET", + ] + write_manifest(Path(args.output), manifest) + + +def nginx(args: argparse.Namespace) -> None: + """Render a digest-pinned nginx application CVM.""" + docker_compose = f"""services: + web: + image: {args.image_ref} + ports: + - "80:80" +""" + manifest = common_manifest( + args.name, + docker_compose, + image_loader(args.artifact_port, args.image_archive), + ) + manifest.update( + { + "gateway_enabled": True, + "kms_enabled": True, + "no_instance_id": False, + "secure_time": False, + } + ) if args.attestation_mode != "auto": # A string v3 manifest makes old guests fail closed instead of silently - # ignoring the requirement. The current VMM resolves this field into - # the corresponding legacy/lite VM attestation config. + # ignoring the requested TDX quote mode. manifest["manifest_version"] = "3" manifest["requirements"] = { "tdx_measure_acpi_tables": args.attestation_mode == "legacy" @@ -52,15 +212,31 @@ def nginx(args: argparse.Namespace) -> None: write_manifest(Path(args.output), manifest) +def add_image_args(parser: argparse.ArgumentParser) -> None: + """Add the content-addressed container-image arguments shared by apps.""" + parser.add_argument("--image-ref", required=True) + parser.add_argument("--image-archive", required=True) + parser.add_argument("--artifact-port", required=True, type=int) + parser.add_argument("--name", required=True) + parser.add_argument("--output", required=True) + + def main() -> None: """Parse command-line arguments and render a manifest.""" parser = argparse.ArgumentParser() sub = parser.add_subparsers(required=True) + p = sub.add_parser("kms") + add_image_args(p) + p.add_argument("--auth-port", required=True, type=int) + p.set_defaults(func=kms) + + p = sub.add_parser("gateway") + add_image_args(p) + p.set_defaults(func=gateway) + p = sub.add_parser("nginx") - p.add_argument("--name", required=True) - p.add_argument("--app-image", required=True) - p.add_argument("--output", required=True) + add_image_args(p) p.add_argument( "--attestation-mode", choices=("auto", "legacy", "lite"), diff --git a/test-suites/full-stack-compose/scripts/network-probe.sh b/test-suites/full-stack-compose/scripts/network-probe.sh index a8de95028..9e0387bc7 100755 --- a/test-suites/full-stack-compose/scripts/network-probe.sh +++ b/test-suites/full-stack-compose/scripts/network-probe.sh @@ -5,70 +5,87 @@ set -euo pipefail STATE_DIR=${DSTACK_E2E_STATE_DIR:-/suite-state} WORK_DIR=${DSTACK_E2E_WORK_DIR:-$STATE_DIR/work} -TARGETS=${DSTACK_E2E_NETWORK_TARGETS:-$WORK_DIR/network-targets.tsv} -READY=$WORK_DIR/network-probe.ready -STOP=$WORK_DIR/network-probe.stop -RESULT=$WORK_DIR/network-probe.result.json -FAILURES=$WORK_DIR/network-probe.failures.log -INTERVAL=${DSTACK_E2E_NETWORK_PROBE_INTERVAL:-0.05} -REQUEST_TIMEOUT=${DSTACK_E2E_NETWORK_PROBE_TIMEOUT:-1} -WARMUP_SUCCESSES=${DSTACK_E2E_NETWORK_PROBE_WARMUP:-20} +# shellcheck disable=SC1091 +source "$STATE_DIR/state.env" -log() { printf '[%(%H:%M:%S)T] %s\n' -1 "$*"; } -die() { log "ERROR: $*" >&2; exit 1; } - -[[ -s "$TARGETS" ]] || die "missing network targets: $TARGETS" -rm -f "$READY" "$STOP" "$RESULT" "$FAILURES" +ready="$WORK_DIR/network-probe.ready" +stop="$WORK_DIR/network-probe.stop" +result="$WORK_DIR/network-probe.result.json" +attempts=0 +failures=0 +failovers=0 +cycle=0 +started_ns=$(date +%s%N) -probe_all() { - local count=0 label ip body - while IFS=$'\t' read -r label ip; do - [[ -n "$label" && -n "$ip" ]] || continue - body=$(curl -fsS --max-time "$REQUEST_TIMEOUT" "http://${ip}:80/" 2>&1) \ - && grep -qi 'welcome to nginx' <<<"$body" \ - || return 1 - count=$((count + 1)) - done < "$TARGETS" - (( count > 0 )) +probe_endpoint() { + local label=$1 instance=$2 port=$3 sni + sni="${instance}-80.${BASE_DOMAIN}" + curl -kfsS --connect-timeout 1 --max-time 2 \ + --connect-to "${sni}:${port}:127.0.0.1:${port}" \ + "https://${sni}:${port}/" 2>/dev/null \ + | grep -qi 'welcome to nginx' } -log "warming up direct CVM WireGuard probes" -successes=0 -deadline=$((SECONDS + 120)) -while (( successes < WARMUP_SUCCESSES )); do - (( SECONDS < deadline )) || die "network probe warmup timed out" - if probe_all; then - successes=$((successes + 1)) +probe_ha() { + local label=$1 instance=$2 first second + if (( cycle % 2 == 0 )); then + first=$GATEWAY1_PROXY_HOST_PORT + second=$GATEWAY2_PROXY_HOST_PORT else - successes=0 + first=$GATEWAY2_PROXY_HOST_PORT + second=$GATEWAY1_PROXY_HOST_PORT fi - sleep "$INTERVAL" + attempts=$((attempts + 1)) + if probe_endpoint "$label" "$instance" "$first"; then + return 0 + fi + if probe_endpoint "$label" "$instance" "$second"; then + failovers=$((failovers + 1)) + return 0 + fi + failures=$((failures + 1)) + printf '%(%FT%T%z)T route unavailable label=%s preferred=%s fallback=%s\n' \ + -1 "$label" "$first" "$second" >&2 + return 1 +} + +declare -a labels=(legacy lite) +declare -A instances +for label in "${labels[@]}"; do + instances[$label]=$(jq -r '.instance_id // ""' "$WORK_DIR/${label}.info.json") + [[ -n "${instances[$label]}" ]] || { + echo "missing instance id for $label" >&2 + exit 1 + } done -started_at=$(date +%s%3N) -printf 'ready\n' > "$READY" -log "probe ready; strict zero-failure window started" +# Warm up against both physical Gateway nodes before declaring the HA probe live. +for label in "${labels[@]}"; do + for port in "$GATEWAY1_PROXY_HOST_PORT" "$GATEWAY2_PROXY_HOST_PORT"; do + probe_endpoint "$label" "${instances[$label]}" "$port" || { + echo "warmup failed: $label via Gateway proxy port $port" >&2 + exit 1 + } + done +done +touch "$ready" -attempts=0 -failures=0 -while [[ ! -e "$STOP" ]]; do - attempts=$((attempts + 1)) - if ! probe_all; then - failures=$((failures + 1)) - printf '%s\tattempt=%s\n' "$(date --iso-8601=ns)" "$attempts" >> "$FAILURES" - fi - sleep "$INTERVAL" +while [[ ! -e "$stop" ]]; do + cycle=$((cycle + 1)) + for label in "${labels[@]}"; do + probe_ha "$label" "${instances[$label]}" || true + done + sleep 0.05 done -finished_at=$(date +%s%3N) +ended_ns=$(date +%s%N) +duration_ms=$(((ended_ns - started_ns) / 1000000)) jq -n \ - --argjson started_at_ms "$started_at" \ - --argjson finished_at_ms "$finished_at" \ + --argjson duration_ms "$duration_ms" \ --argjson attempts "$attempts" \ --argjson failures "$failures" \ - '{started_at_ms:$started_at_ms, finished_at_ms:$finished_at_ms, duration_ms:($finished_at_ms-$started_at_ms), attempts:$attempts, failures:$failures}' \ - | tee "$RESULT" + --argjson failovers "$failovers" \ + '{duration_ms:$duration_ms,attempts:$attempts,failures:$failures,successful_failovers:$failovers}' \ + | tee "$result" -(( attempts > 0 )) || die "network probe recorded no attempts" -(( failures == 0 )) || die "CVM WireGuard data plane had $failures failed probe cycle(s)" -log "zero-downtime CVM network assertion passed" +(( failures == 0 )) diff --git a/test-suites/full-stack-compose/scripts/render-config.sh b/test-suites/full-stack-compose/scripts/render-config.sh index 168b19945..500b61c4b 100755 --- a/test-suites/full-stack-compose/scripts/render-config.sh +++ b/test-suites/full-stack-compose/scripts/render-config.sh @@ -8,23 +8,26 @@ CONFIG_DIR=${DSTACK_E2E_CONFIG_DIR:-$STATE_DIR/config} RUN_DIR=${DSTACK_E2E_RUN_DIR:-$STATE_DIR/run} VM_DIR=${DSTACK_E2E_VM_DIR:-$STATE_DIR/vm} VOLUMES_DIR=${DSTACK_E2E_VOLUMES_DIR:-$STATE_DIR/volumes} +ARTIFACT_DIR=${DSTACK_E2E_ARTIFACT_DIR:-$STATE_DIR/artifacts} IMAGE_ROOT=${DSTACK_E2E_IMAGE_ROOT:-/images} IMAGE_NAME=${DSTACK_E2E_IMAGE_NAME:-dstack-0.6.0} PLATFORM=${DSTACK_E2E_PLATFORM:-tdx} VMM_PORT=${DSTACK_E2E_VMM_PORT:-29080} AUTH_PORT=${DSTACK_E2E_AUTH_PORT:-28011} -KMS_HOST_PORT=${DSTACK_E2E_KMS_HOST_PORT:-28082} -KMS_RPC_DOMAIN=${DSTACK_E2E_KMS_RPC_DOMAIN:-10.0.2.2.nip.io} -GATEWAY_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY_RPC_HOST_PORT:-28000} -GATEWAY_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY_ADMIN_HOST_PORT:-28001} -GATEWAY_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY_PROXY_HOST_PORT:-28443} -GATEWAY_WG_HOST_PORT=${DSTACK_E2E_GATEWAY_WG_HOST_PORT:-28120} -GATEWAY_WG_INTERFACE=${DSTACK_E2E_GATEWAY_WG_INTERFACE:-wg-ds-e2e} -GATEWAY_WG_IP=${DSTACK_E2E_GATEWAY_WG_IP:-10.8.0.1/16} -GATEWAY_WG_RESERVED_NET=${DSTACK_E2E_GATEWAY_WG_RESERVED_NET:-10.8.0.1/32} -GATEWAY_WG_CLIENT_RANGE=${DSTACK_E2E_GATEWAY_WG_CLIENT_RANGE:-10.8.0.0/18} -GATEWAY_APP_ID=${DSTACK_E2E_GATEWAY_APP_ID:-any} +ARTIFACT_PORT=${DSTACK_E2E_ARTIFACT_PORT:-38081} +KMS_OLD_HOST_PORT=${DSTACK_E2E_KMS_OLD_HOST_PORT:-28082} +KMS_LATEST_HOST_PORT=${DSTACK_E2E_KMS_LATEST_HOST_PORT:-28083} +KMS_RPC_DOMAIN=${DSTACK_E2E_KMS_RPC_DOMAIN:-} +GATEWAY1_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY1_RPC_HOST_PORT:-28000} +GATEWAY1_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY1_ADMIN_HOST_PORT:-28001} +GATEWAY1_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY1_PROXY_HOST_PORT:-28443} +GATEWAY1_WG_HOST_PORT=${DSTACK_E2E_GATEWAY1_WG_HOST_PORT:-28120} +GATEWAY2_RPC_HOST_PORT=${DSTACK_E2E_GATEWAY2_RPC_HOST_PORT:-28100} +GATEWAY2_ADMIN_HOST_PORT=${DSTACK_E2E_GATEWAY2_ADMIN_HOST_PORT:-28101} +GATEWAY2_PROXY_HOST_PORT=${DSTACK_E2E_GATEWAY2_PROXY_HOST_PORT:-28543} +GATEWAY2_WG_HOST_PORT=${DSTACK_E2E_GATEWAY2_WG_HOST_PORT:-28121} +GATEWAY_APP_ID=${DSTACK_E2E_GATEWAY_APP_ID:-$(printf dstack-e2e-gateway | sha256sum | cut -c1-40)} KEY_PROVIDER_PORT=${DSTACK_E2E_KEY_PROVIDER_PORT:-13443} HOST_API_PORT=${DSTACK_E2E_HOST_API_PORT:-20011} CID_START=${DSTACK_E2E_CID_START:-15000} @@ -33,20 +36,34 @@ QEMU_PATH=${DSTACK_E2E_QEMU_PATH:-/usr/bin/qemu-system-x86_64} BASE_DOMAIN=${DSTACK_E2E_BASE_DOMAIN:-e2e.test} MOCK_CF_HTTP_PORT=${DSTACK_E2E_MOCK_CF_HTTP_PORT:-38080} PEBBLE_HTTP_PORT=${DSTACK_E2E_PEBBLE_HTTP_PORT:-34000} -APP_IMAGE=${DSTACK_E2E_APP_IMAGE:-nginx:alpine} APP_NAME=${DSTACK_E2E_APP_NAME:-dstack-e2e-nginx} SUITE_PREFIX=${DSTACK_E2E_NAME_PREFIX:-dstack-e2e} -KMS_CERT_DIR=${DSTACK_E2E_KMS_CERT_DIR:-$STATE_DIR/kms-certs} -GATEWAY_CERT_DIR=${DSTACK_E2E_GATEWAY_CERT_DIR:-$STATE_DIR/gateway-certs} -GATEWAY_DATA_DIR=${DSTACK_E2E_GATEWAY_DATA_DIR:-$STATE_DIR/gateway-data} - mkdir -p \ - "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" "$STATE_DIR/work" \ - "$KMS_CERT_DIR" "$GATEWAY_CERT_DIR" "$GATEWAY_DATA_DIR" -# The host-side upgrade driver writes orchestration metadata and probe markers -# here while service containers also write artifacts as root. -chmod 0777 "$STATE_DIR/work" + "$CONFIG_DIR" "$RUN_DIR" "$VM_DIR" "$VOLUMES_DIR" \ + "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/os" "$ARTIFACT_DIR/control" \ + "$STATE_DIR/work" +chmod 0777 "$STATE_DIR/work" "$ARTIFACT_DIR" "$ARTIFACT_DIR/images" "$ARTIFACT_DIR/control" + +if [[ ! "$GATEWAY_APP_ID" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: DSTACK_E2E_GATEWAY_APP_ID must be an exact 20-byte hex app id" >&2 + exit 1 +fi +GATEWAY_APP_ID=${GATEWAY_APP_ID,,} + +if [[ -z "$KMS_RPC_DOMAIN" ]]; then + host_ip=$(ip -4 route get 1.1.1.1 2>/dev/null \ + | sed -n 's/.* src \([^ ]*\).*/\1/p' | head -n1) + if [[ ! "$host_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: unable to derive a host IPv4 address; set DSTACK_E2E_KMS_RPC_DOMAIN" >&2 + exit 1 + fi + KMS_RPC_DOMAIN="${host_ip}.nip.io" +fi +if ! getent ahostsv4 "$KMS_RPC_DOMAIN" >/dev/null 2>&1; then + echo "ERROR: DSTACK_E2E_KMS_RPC_DOMAIN does not resolve: $KMS_RPC_DOMAIN" >&2 + exit 1 +fi image_dir="$IMAGE_ROOT/$IMAGE_NAME" if [[ ! -d "$image_dir" ]]; then @@ -55,321 +72,62 @@ if [[ ! -d "$image_dir" ]]; then exit 1 fi -digest_file="digest.txt" +digest_file=digest.txt if [[ "$PLATFORM" == "amd-sev-snp" || "$PLATFORM" == "sev-snp" || "$PLATFORM" == "snp" ]]; then - PLATFORM="amd-sev-snp" - digest_file="digest.sev.txt" -elif [[ "$PLATFORM" != "tdx" ]]; then + PLATFORM=amd-sev-snp + digest_file=digest.sev.txt +elif [[ "$PLATFORM" != tdx ]]; then echo "ERROR: unsupported DSTACK_E2E_PLATFORM=$PLATFORM (expected tdx or amd-sev-snp)" >&2 exit 1 fi -OS_IMAGE_HASH="" -if [[ -s "$image_dir/$digest_file" ]]; then - OS_IMAGE_HASH=$(tr -d '[:space:]' < "$image_dir/$digest_file") -elif [[ "${DSTACK_E2E_ALLOW_UNPINNED_IMAGE:-false}" == "true" ]]; then - echo "WARN: missing $image_dir/$digest_file; auth allowlist will not pin OS image" >&2 -else - echo "ERROR: missing $image_dir/$digest_file" >&2 - echo "Set DSTACK_E2E_ALLOW_UNPINNED_IMAGE=true only for local experiments." >&2 +if [[ ! -s "$image_dir/$digest_file" ]]; then + echo "ERROR: missing $image_dir/$digest_file; production-compatible E2E never permits an unpinned OS image" >&2 exit 1 fi - -ADMIN_TOKEN_FILE="$STATE_DIR/gateway-admin-token" -if [[ ! -s "$ADMIN_TOKEN_FILE" ]]; then - openssl rand -hex 24 > "$ADMIN_TOKEN_FILE" - chmod 0600 "$ADMIN_TOKEN_FILE" || true -fi -GATEWAY_ADMIN_TOKEN=$(cat "$ADMIN_TOKEN_FILE") - -openssl_gen_ec_key() { - local path=$1 - openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:prime256v1 -out "$path" >/dev/null 2>&1 - chmod 0600 "$path" || true -} - -openssl_gen_ca() { - local key=$1 cert=$2 subject=$3 - openssl_gen_ec_key "$key" - openssl req -x509 -new -key "$key" -sha256 -days 3650 -out "$cert" \ - -subj "$subject" \ - -addext "basicConstraints=critical,CA:TRUE,pathlen:1" \ - -addext "keyUsage=critical,keyCertSign,cRLSign" >/dev/null 2>&1 -} - -openssl_gen_kms_rpc_cert() { - local dir=$1 tmp csr - tmp=$(mktemp) - csr="$dir/rpc.csr" - cat > "$tmp" </dev/null 2>&1 - openssl x509 -req -in "$csr" -CA "$dir/root-ca.crt" -CAkey "$dir/root-ca.key" \ - -CAcreateserial -out "$dir/rpc.crt" -days 3650 -sha256 \ - -extfile "$tmp" -extensions v3_req >/dev/null 2>&1 - rm -f "$tmp" "$csr" -} - -openssl_gen_gateway_rpc_cert() { - local dir=$1 tmp - tmp=$(mktemp) - cat > "$tmp" <<'OPENSSL_CONF' -[req] -prompt = no -distinguished_name = dn -x509_extensions = v3_req - -[dn] -CN = 10.0.2.2 - -[v3_req] -basicConstraints = critical,CA:FALSE -keyUsage = critical,digitalSignature -extendedKeyUsage = serverAuth -subjectAltName = @alt_names - -[alt_names] -IP.1 = 10.0.2.2 -IP.2 = 127.0.0.1 -DNS.1 = localhost -OPENSSL_CONF - openssl_gen_ec_key "$dir/gateway-rpc.key" - openssl req -x509 -new -key "$dir/gateway-rpc.key" -sha256 -days 3650 \ - -out "$dir/gateway-rpc.crt" -config "$tmp" -extensions v3_req >/dev/null 2>&1 - rm -f "$tmp" -} - -write_k256_key() { - local path=$1 - python3 - "$path" <<'PY' -import os -import sys -# secp256k1 curve order -N = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 -path = sys.argv[1] -while True: - b = os.urandom(32) - x = int.from_bytes(b, "big") - if 1 <= x < N: - with open(path, "wb") as f: - f.write(b) - break -PY - chmod 0600 "$path" || true -} - -if [[ ! -s "$KMS_CERT_DIR/root-ca.key" || ! -s "$KMS_CERT_DIR/root-ca.crt" || \ - ! -s "$KMS_CERT_DIR/tmp-ca.key" || ! -s "$KMS_CERT_DIR/tmp-ca.crt" || \ - ! -s "$KMS_CERT_DIR/rpc.key" || ! -s "$KMS_CERT_DIR/rpc.crt" || \ - ! -s "$KMS_CERT_DIR/root-k256.key" ]]; then - echo "Generating dev KMS key material in $KMS_CERT_DIR" - rm -f "$KMS_CERT_DIR"/{root-ca.key,root-ca.crt,root-ca.srl,tmp-ca.key,tmp-ca.crt,rpc.key,rpc.crt,rpc.csr,root-k256.key,rpc-domain} - openssl_gen_ca "$KMS_CERT_DIR/root-ca.key" "$KMS_CERT_DIR/root-ca.crt" "/O=Dstack/CN=Dstack KMS CA" - openssl_gen_ca "$KMS_CERT_DIR/tmp-ca.key" "$KMS_CERT_DIR/tmp-ca.crt" "/O=Dstack/CN=Dstack Client Temp CA" - openssl_gen_kms_rpc_cert "$KMS_CERT_DIR" - write_k256_key "$KMS_CERT_DIR/root-k256.key" - printf '%s' "$KMS_RPC_DOMAIN" > "$KMS_CERT_DIR/rpc-domain" -fi - -if [[ ! -s "$GATEWAY_CERT_DIR/gateway-rpc.key" || ! -s "$GATEWAY_CERT_DIR/gateway-rpc.crt" ]]; then - echo "Generating dev Gateway RPC certificate in $GATEWAY_CERT_DIR" - rm -f "$GATEWAY_CERT_DIR"/{gateway-rpc.key,gateway-rpc.crt} - openssl_gen_gateway_rpc_cert "$GATEWAY_CERT_DIR" +OS_IMAGE_HASH=$(tr -d '[:space:]' < "$image_dir/$digest_file") +if [[ ! "$OS_IMAGE_HASH" =~ ^[0-9a-f]{64}$ ]]; then + echo "ERROR: invalid OS image digest: $OS_IMAGE_HASH" >&2 + exit 1 fi -WG_KEY_FILE="$GATEWAY_DATA_DIR/wg.key" -if [[ ! -s "$WG_KEY_FILE" ]]; then - (umask 077; wg genkey > "$WG_KEY_FILE") - chmod 0600 "$WG_KEY_FILE" || true +# Build the exact flat archive consumed by the KMS verifier. Verify the source +# first and include only the measured files named by sha256sum.txt. +( + cd "$image_dir" + sha256sum -c sha256sum.txt + mapfile -t measured_files < <(awk '{print $2}' sha256sum.txt) + for file in "${measured_files[@]}"; do + [[ "$file" != */* && -f "$file" ]] || { + echo "ERROR: unsafe or missing measured image file: $file" >&2 + exit 1 + } + done + tar -czf "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + sha256sum.txt "${measured_files[@]}" +) +mv "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz.tmp" \ + "$ARTIFACT_DIR/os/mr_${OS_IMAGE_HASH}.tar.gz" + +token_file="$STATE_DIR/gateway-admin-token" +if [[ ! -s "$token_file" ]]; then + (umask 077; openssl rand -hex 24 > "$token_file") fi -GATEWAY_WG_PRIVATE_KEY=$(tr -d '[:space:]' < "$WG_KEY_FILE") -GATEWAY_WG_PUBLIC_KEY=$(printf '%s' "$GATEWAY_WG_PRIVATE_KEY" | wg pubkey) +GATEWAY_ADMIN_TOKEN=$(tr -d '[:space:]' < "$token_file") cat > "$CONFIG_DIR/auth-allowlist.json" < "$CONFIG_DIR/kms.toml" < "$CONFIG_DIR/gateway.toml" < "$CONFIG_DIR/vmm.toml" <&2; exit 1; } need_bin() { - [[ -x "$1" ]] || die "missing executable $1; run: cargo build --release -p dstack-cli -p dstack-auth -p dstack-vmm -p dstack-kms -p dstack-gateway -p supervisor" -} - -need_current_bins() { - need_bin /workspace/target/release/dstack - need_bin /workspace/target/release/dstack-auth - need_bin /workspace/target/release/dstack-vmm - need_bin /workspace/target/release/dstack-kms - need_bin /workspace/target/release/dstack-gateway - need_bin /workspace/target/release/supervisor -} - -require_tdx() { - [[ "$PLATFORM" == "tdx" ]] || die "$PHASE requires DSTACK_E2E_PLATFORM=tdx" + [[ -x "$1" ]] || die "missing executable $1" } wait_vmm() { log "waiting for VMM at $VMM_URL" - for _ in $(seq 1 90); do + for _ in $(seq 1 120); do if "${DSTACK_CLI[@]}" apps -j >/dev/null 2>&1; then log "VMM ready" - return 0 + return fi sleep 2 done @@ -51,7 +48,8 @@ wait_vmm() { vm_ids_by_prefix() { local prefix=$1 - "${VMM_CLI[@]}" lsvm --json 2>/dev/null | jq -r --arg p "$prefix" '.[] | select(.name | startswith($p)) | .id' + "${VMM_CLI[@]}" lsvm --json 2>/dev/null \ + | jq -r --arg p "$prefix" '.[] | select(.name | startswith($p)) | .id' } remove_vm() { @@ -62,14 +60,11 @@ remove_vm() { } clean_start() { - if [[ "${DSTACK_E2E_CLEAN_START:-true}" != "true" ]]; then - return 0 - fi - log "cleaning stale VMs with prefix ${SUITE_PREFIX}" + [[ "${DSTACK_E2E_CLEAN_START:-true}" == true ]] || return 0 while read -r id; do remove_vm "$id" done < <(vm_ids_by_prefix "$SUITE_PREFIX") - for _ in $(seq 1 60); do + for _ in $(seq 1 90); do [[ -z "$(vm_ids_by_prefix "$SUITE_PREFIX")" ]] && return 0 sleep 2 done @@ -77,454 +72,819 @@ clean_start() { } print_vm_info_safe() { - jq '{ - id, - name, - status, - uptime, - app_id, - instance_id, - boot_progress, - boot_error, - shutdown_progress, - image_version, - events - }' <<<"$1" + jq '{id,name,status,uptime,app_id,instance_id,boot_progress,boot_error,image_version,events}' <<<"$1" } wait_boot_done() { - local id=$1 label=$2 timeout=${3:-480} - local deadline=$((SECONDS + timeout)) - local last="" + local id=$1 label=$2 timeout=${3:-720} + local deadline=$((SECONDS + timeout)) last="" while (( SECONDS < deadline )); do - local info status progress error instance + local info status progress error line info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) if [[ -n "$info" ]]; then status=$(jq -r '.status // ""' <<<"$info") progress=$(jq -r '.boot_progress // ""' <<<"$info") error=$(jq -r '.boot_error // ""' <<<"$info") - instance=$(jq -r '.instance_id // ""' <<<"$info") - local line="status=$status progress=${progress:-none} instance=${instance:-none} error=${error:-none}" + line="status=$status progress=${progress:-none} error=${error:-none}" if [[ "$line" != "$last" ]]; then log "$label: $line" last=$line fi - if [[ -n "$error" && "$error" != "null" ]]; then + if [[ -n "$error" && "$error" != null ]]; then print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true die "$label boot failed: $error" fi - if [[ "$status" == "running" && "$progress" == "done" ]]; then + if [[ "$status" == running && "$progress" == "done" ]]; then printf '%s' "$info" > "$WORK_DIR/${label}.info.json" - return 0 + return fi - if [[ "$status" == "exited" || "$status" == "stopped" ]]; then + if [[ "$status" == exited || "$status" == stopped ]]; then print_vm_info_safe "$info" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 240 >&2 || true die "$label exited before boot finished" fi fi sleep 5 done - print_vm_info_safe "$("${VMM_CLI[@]}" info "$id" --json)" >&2 || true + "${VMM_CLI[@]}" logs "$id" -n 300 >&2 || true die "timed out waiting for $label" } wait_vm_stopped() { - local id=$1 deadline=$((SECONDS + 120)) + local id=$1 deadline=$((SECONDS + 180)) status while (( SECONDS < deadline )); do - local status status=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null | jq -r '.status // ""' || true) - if [[ "$status" != "running" && "$status" != "starting" ]]; then - return 0 - fi + [[ "$status" != running && "$status" != starting ]] && return sleep 2 done die "VM $id did not stop" } -wait_vm_running() { - local id=$1 label=$2 deadline=$((SECONDS + 90)) last="" +stop_vm() { + local id=$1 label=$2 + log "stopping $label ($id)" + "${VMM_CLI[@]}" stop "$id" --force >/dev/null + wait_vm_stopped "$id" +} + +start_vm() { + local id=$1 label=$2 + log "starting $label ($id)" + "${VMM_CLI[@]}" start "$id" >/dev/null + wait_boot_done "$id" "$label" +} + +register_app() { + local app_id=$1 hash=$2 + [[ -n "$ATTESTED_DEVICE_ID" ]] || die "cannot authorize app before pinning the attested TDX device" + /suite/scripts/allowlist.py add-app --path "$ALLOWLIST" \ + --app-id "$app_id" --compose-hash "$hash" \ + --device-id "$ATTESTED_DEVICE_ID" >/dev/null +} + +register_kms_measurement() { + local measurement=$1 device_id=$2 + /suite/scripts/allowlist.py add-kms --path "$ALLOWLIST" \ + --mr-aggregated "$measurement" --device-id "$device_id" >/dev/null +} + +render_kms() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py kms \ + --name "dstack-e2e-kms-${stage}" \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --auth-port "$AUTH_PORT" \ + --output "$WORK_DIR/kms-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/kms-${stage}.manifest-meta.json" +} + +deploy_kms_onboard() { + local stage=$1 port=$2 out vm_id + log "deploying $stage KMS CVM in Local-Key-Provider mode" + if ! out=$("${VMM_CLI[@]}" deploy \ + --name "${SUITE_PREFIX}-kms-${stage}" \ + --image "$IMAGE_NAME" \ + --compose "$WORK_DIR/kms-${stage}.app-compose.json" \ + --port "tcp:0.0.0.0:${port}:8000" \ + --vcpu "${DSTACK_E2E_KMS_VCPU:-4}" \ + --memory "${DSTACK_E2E_KMS_MEMORY:-4096}" \ + --disk "${DSTACK_E2E_KMS_DISK:-30}" 2>&1); then + die "failed to deploy $stage KMS CVM: $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/kms-${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse $stage KMS VM id" + printf '%s' "$vm_id" > "$WORK_DIR/kms-${stage}.vm_id" + wait_boot_done "$vm_id" "kms-${stage}" +} + +wait_onboard() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) url + url="http://127.0.0.1:${port}/prpc/Onboard.GetAttestationInfo?json" + log "waiting for $stage KMS quote-enabled onboarding endpoint" while (( SECONDS < deadline )); do - local info status error - info=$("${VMM_CLI[@]}" info "$id" --json 2>/dev/null || true) - if [[ -n "$info" ]]; then - status=$(jq -r '.status // ""' <<<"$info") - error=$(jq -r '.boot_error // ""' <<<"$info") - if [[ "$status" != "$last" ]]; then - log "$label restart: status=${status:-unknown}" - last=$status - fi - if [[ -n "$error" && "$error" != "null" ]]; then - print_vm_info_safe "$info" >&2 || true - die "$label restart failed: $error" - fi - [[ "$status" == "running" ]] && return 0 + if curl -fsS "$url" -o "$WORK_DIR/kms-${stage}.attestation-info.json"; then + jq -e '((.mr_aggregated // .mrAggregated // "") | length) > 0 and ((.device_id // .deviceId // "") | length) > 0' \ + "$WORK_DIR/kms-${stage}.attestation-info.json" >/dev/null && return fi - sleep 1 + sleep 3 done - die "$label did not enter running state after restart" + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS onboarding endpoint not ready" +} + +authorize_kms_from_attestation() { + local stage=$1 measurement device_id + measurement=$(jq -r '.mr_aggregated // .mrAggregated' "$WORK_DIR/kms-${stage}.attestation-info.json") + device_id=$(jq -r '.device_id // .deviceId' "$WORK_DIR/kms-${stage}.attestation-info.json") + [[ "$measurement" =~ ^[0-9a-fA-F]+$ ]] || die "invalid $stage KMS mrAggregated" + [[ "$device_id" =~ ^[0-9a-fA-F]{64}$ ]] || die "invalid $stage KMS device ID" + if [[ -n "$ATTESTED_DEVICE_ID" && "$device_id" != "$ATTESTED_DEVICE_ID" ]]; then + die "$stage KMS quote came from unexpected TDX device $device_id" + fi + ATTESTED_DEVICE_ID=$device_id + register_kms_measurement "$measurement" "$device_id" + log "authorized exact $stage KMS mrAggregated=${measurement} on device=${device_id}" } -register_app() { - local app_id=$1 hash=$2 gateway_id=${3:-} - local args=(/suite/scripts/allowlist.py add-app --path "$ALLOWLIST" --app-id "$app_id" --compose-hash "$hash") - if [[ -n "$gateway_id" ]]; then - args+=(--gateway-app-id "$gateway_id") - fi - python3 "${args[@]}" >/dev/null +onboard_rpc() { + local port=$1 method=$2 data=$3 out=$4 + curl -fsS -X POST -H 'Content-Type: application/json' \ + "http://127.0.0.1:${port}/prpc/Onboard.${method}?json" \ + --data-raw "$data" -o "$out" + jq -e 'has("error") | not' "$out" >/dev/null \ + || die "Onboard.${method} returned an error: $(cat "$out")" + log "Onboard.${method} completed" } -wait_kms() { - log "waiting for KMS at https://127.0.0.1:${KMS_HOST_PORT}" - for _ in $(seq 1 90); do - if curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/prpc/GetMeta?json" >/dev/null 2>&1; then - log "KMS ready" - return 0 +finish_onboarding() { + local stage=$1 port=$2 + curl -fsS "http://127.0.0.1:${port}/finish" > "$WORK_DIR/kms-${stage}.finish.txt" + wait_kms_tls "$stage" "$port" +} + +wait_kms_tls() { + local stage=$1 port=$2 deadline=$((SECONDS + 300)) + log "waiting for $stage KMS TLS service" + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${port}/prpc/GetMeta?json" \ + -o "$WORK_DIR/kms-${stage}.ready-meta.json"; then + log "$stage KMS TLS service ready" + return fi - sleep 2 + sleep 3 done - die "KMS not ready" + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/kms-${stage}.vm_id")" -n 300 >&2 || true + die "$stage KMS TLS service not ready" } kms_rpc_get() { - local method=$1 - curl -kfsS "https://127.0.0.1:${KMS_HOST_PORT}/prpc/KMS.${method}?json" + local port=$1 method=$2 + curl -kfsS "https://127.0.0.1:${port}/prpc/${method}?json" } kms_rpc_post() { - local method=$1 data=$2 + local port=$1 method=$2 data=$3 curl -kfsS -X POST -H 'Content-Type: application/json' \ - "https://127.0.0.1:${KMS_HOST_PORT}/prpc/KMS.${method}?json" \ - --data-raw "$data" + "https://127.0.0.1:${port}/prpc/${method}?json" --data-raw "$data" } capture_kms_identity() { - local stage=$1 app_id - app_id=$(cat "$WORK_DIR/legacy.app_id") - log "capturing KMS identity ($stage) for app_id=$app_id" - kms_rpc_get GetMeta | tee "$WORK_DIR/kms-${stage}.meta.raw.json" \ + local stage=$1 port=$2 app_id=$3 ca_pem ca_spki k256_pubkey + ca_pem="$WORK_DIR/kms-${stage}.ca.pem" + kms_rpc_get "$port" GetMeta | tee "$WORK_DIR/kms-${stage}.meta.raw.json" \ | jq -S '{ca_cert, k256_pubkey}' > "$WORK_DIR/kms-${stage}.meta.json" - kms_rpc_post GetAppEnvEncryptPubKey "$(jq -cn --arg id "$app_id" '{app_id:$id}')" \ + jq -er '.ca_cert' "$WORK_DIR/kms-${stage}.meta.json" > "$ca_pem" + openssl x509 -in "$ca_pem" -noout -checkend 0 >/dev/null + ca_spki=$(openssl x509 -in "$ca_pem" -pubkey -noout \ + | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 \ + | awk '{print $NF}') + k256_pubkey=$(jq -er '.k256_pubkey' "$WORK_DIR/kms-${stage}.meta.json") + jq -nS --arg ca_spki_sha256 "$ca_spki" --arg k256_pubkey "$k256_pubkey" \ + '{ca_spki_sha256:$ca_spki_sha256,k256_pubkey:$k256_pubkey}' \ + > "$WORK_DIR/kms-${stage}.identity.json" + kms_rpc_post "$port" GetAppEnvEncryptPubKey \ + "$(jq -cn --arg id "$app_id" '{app_id:$id}')" \ | tee "$WORK_DIR/kms-${stage}.app-key.raw.json" \ | jq -S '{public_key}' > "$WORK_DIR/kms-${stage}.app-key.json" - jq -e '.ca_cert != "" and .k256_pubkey != ""' "$WORK_DIR/kms-${stage}.meta.json" >/dev/null + jq -e '.ca_spki_sha256 != "" and .k256_pubkey != ""' \ + "$WORK_DIR/kms-${stage}.identity.json" >/dev/null jq -e '.public_key != ""' "$WORK_DIR/kms-${stage}.app-key.json" >/dev/null } assert_kms_identity_unchanged() { - capture_kms_identity after - diff -u "$WORK_DIR/kms-before.meta.json" "$WORK_DIR/kms-after.meta.json" \ - || die "KMS CA or root k256 public key changed across upgrade" - diff -u "$WORK_DIR/kms-before.app-key.json" "$WORK_DIR/kms-after.app-key.json" \ - || die "per-app environment encryption key changed across KMS upgrade" - log "KMS persistent identity and per-app key are unchanged" + # Onboarding deliberately reissues the self-signed CA certificate with a new + # validity end time. The transferable identity is its private key/SPKI, not + # the DER bytes of that freshly issued certificate. + diff -u "$WORK_DIR/kms-old.identity.json" "$WORK_DIR/kms-latest.identity.json" \ + || die "KMS CA key or root k256 key changed during 0.5.8 -> current onboarding" + diff -u "$WORK_DIR/kms-old.app-key.json" "$WORK_DIR/kms-latest.app-key.json" \ + || die "per-app environment encryption key changed during KMS upgrade" + log "KMS CA SPKI, root k256 key and per-app environment key are unchanged" +} + +render_gateway_manifests() { + local stage=$1 image_ref=$2 archive=$3 meta + meta=$(/suite/scripts/app_compose.py gateway \ + --name dstack-e2e-gateway \ + --image-ref "$image_ref" \ + --image-archive "$archive" \ + --artifact-port "$ARTIFACT_PORT" \ + --output "$WORK_DIR/gateway-${stage}.app-compose.json") + printf '%s\n' "$meta" > "$WORK_DIR/gateway-${stage}.manifest-meta.json" + register_app "$GATEWAY_APP_ID" "$(jq -r .composeHash <<<"$meta")" +} + +write_gateway_env() { + local node=$1 rpc_port wg_port third_octet bootnode + if [[ "$node" == 1 ]]; then + rpc_port=$GATEWAY1_RPC_HOST_PORT + wg_port=$GATEWAY1_WG_HOST_PORT + third_octet=0 + bootnode="" + else + rpc_port=$GATEWAY2_RPC_HOST_PORT + wg_port=$GATEWAY2_WG_HOST_PORT + third_octet=64 + bootnode="$GATEWAY1_URL" + fi + cat > "$WORK_DIR/gateway${node}.env" <&1); then + die "failed to deploy Gateway node $node ($stage): $out" + fi + printf '%s\n' "$out" | tee "$WORK_DIR/gateway${node}.${stage}.deploy.log" + vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) + [[ -n "$vm_id" ]] || die "failed to parse Gateway node $node VM id" + printf '%s' "$vm_id" > "$WORK_DIR/gateway${node}.vm_id" + wait_boot_done "$vm_id" "gateway${node}-${stage}" + wait_gateway "$node" +} + +gateway_version() { + local node=$1 stage=$2 rpc _ + read -r rpc _ < <(gateway_ports "$node") + curl -kfsS -D "$WORK_DIR/gateway${node}-${stage}.headers" -o /dev/null \ + "https://127.0.0.1:${rpc}/prpc/Info?json" + tr -d '\r' < "$WORK_DIR/gateway${node}-${stage}.headers" \ + | awk 'tolower($1)=="x-app-version:" {$1=""; sub(/^ /,""); print}' \ + > "$WORK_DIR/gateway${node}-${stage}.version.txt" + [[ -s "$WORK_DIR/gateway${node}-${stage}.version.txt" ]] \ + || die "Gateway node $node did not return X-App-Version" + if [[ "$stage" == old ]]; then + grep -E '^v0\.5\.8 \(git:' "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run released v0.5.8" + else + grep -E "^v${CURRENT_VERSION//./\\.} \\(git:" \ + "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current v$CURRENT_VERSION" + grep -F "$CURRENT_REV" "$WORK_DIR/gateway${node}-${stage}.version.txt" >/dev/null \ + || die "Gateway node $node did not run current revision $CURRENT_REV" + fi + log "Gateway node $node $stage: $(cat "$WORK_DIR/gateway${node}-${stage}.version.txt")" } admin_curl() { - local method=$1 - local data=${2:-'{}'} - local out code + local node=$1 method=$2 data=${3:-'{}'} admin out code + read -r _ admin _ < <(gateway_ports "$node") out=$(mktemp) code=$(curl -sS -o "$out" -w '%{http_code}' -X POST \ -H "Authorization: Bearer ${GATEWAY_ADMIN_TOKEN}" \ -H 'Content-Type: application/json' \ - "http://127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}/prpc/Admin.${method}?json" \ + "http://127.0.0.1:${admin}/prpc/Admin.${method}?json" \ --data-raw "$data" || true) if [[ "$code" =~ ^2 ]]; then cat "$out" rm -f "$out" - return 0 + return fi - log "Admin.${method} failed HTTP ${code}: $(cat "$out")" >&2 + log "Gateway $node Admin.${method} failed HTTP $code: $(cat "$out")" >&2 rm -f "$out" return 1 } -wait_gateway_admin() { - log "waiting for Gateway admin API on 127.0.0.1:${GATEWAY_ADMIN_HOST_PORT}" - for _ in $(seq 1 90); do - if admin_curl Status >/dev/null 2>&1; then - log "Gateway admin ready" - return 0 +wait_gateway() { + local node=$1 rpc deadline=$((SECONDS + 300)) + read -r rpc _ < <(gateway_ports "$node") + while (( SECONDS < deadline )); do + if curl -kfsS "https://127.0.0.1:${rpc}/prpc/Info?json" >/dev/null 2>&1 \ + && admin_curl "$node" Status >/dev/null 2>&1; then + log "Gateway node $node ready" + return fi - sleep 2 + sleep 3 done - die "Gateway admin API not ready" -} - -bootstrap_gateway_certbot() { - local acme_url="http://127.0.0.1:${PEBBLE_HTTP_PORT}/dir" - local cf_url="http://127.0.0.1:${MOCK_CF_HTTP_PORT}/client/v4" - log "configuring Gateway certbot: acme=$acme_url cf=$cf_url domain=$BASE_DOMAIN" - # Pebble's test certificates are short-lived. Keep the renewal threshold - # below their lifetime so a Gateway restart tests certificate restoration - # instead of intentionally rotating the certificate during startup. - admin_curl SetCertbotConfig "$(jq -cn --arg u "$acme_url" '{acme_url:$u, renew_before_expiration_secs:3600}')" >/dev/null - admin_curl CreateDnsCredential "$(jq -cn --arg u "$cf_url" '{name:"mock-cloudflare", provider_type:"cloudflare", cf_api_token:"test-token", cf_api_url:$u, set_as_default:true, dns_txt_ttl:1, max_dns_wait:0}')" >/dev/null || true - admin_curl AddZtDomain "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, port:443, priority:100}')" >/dev/null || true - log "requesting wildcard cert for *.${BASE_DOMAIN}" - admin_curl RenewZtDomainCert "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d, force:true}')" \ - | tee "$WORK_DIR/renew-cert.json" + "${VMM_CLI[@]}" logs "$(cat "$WORK_DIR/gateway${node}.vm_id")" -n 300 >&2 || true + die "Gateway node $node not ready" +} + +bootstrap_gateway() { + log "configuring Gateway cluster through node 1" + admin_curl 1 SetCertbotConfig \ + "$(jq -cn --arg u "http://10.0.2.2:${PEBBLE_HTTP_PORT}/dir" \ + '{acme_url:$u, renew_before_expiration_secs:3600}')" >/dev/null + admin_curl 1 CreateDnsCredential \ + "$(jq -cn --arg u "http://10.0.2.2:${MOCK_CF_HTTP_PORT}/client/v4" \ + '{name:"mock-cloudflare",provider_type:"cloudflare",cf_api_token:"test-token",cf_api_url:$u,set_as_default:true,dns_txt_ttl:1,max_dns_wait:0}')" \ + >/dev/null + admin_curl 1 AddZtDomain \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,port:443,priority:100}')" \ + >/dev/null + admin_curl 1 RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')" \ + | tee "$WORK_DIR/gateway-renew-cert.json" + wait_gateway_cert 1 + wait_gateway_cluster_sync + wait_gateway_cert 2 +} + +wait_gateway_cluster_sync() { + local deadline=$((SECONDS + 240)) domains certbot + log "waiting for Gateway WaveKV config/certificate sync to node 2" + while (( SECONDS < deadline )); do + domains=$(admin_curl 2 ListZtDomains 2>/dev/null || true) + certbot=$(admin_curl 2 GetCertbotConfig 2>/dev/null || true) + if jq -e --arg d "$BASE_DOMAIN" '.domains[]? | (.domain // .config.domain) == $d' \ + <<<"$domains" >/dev/null 2>&1 \ + && jq -e '.acme_url != ""' <<<"$certbot" >/dev/null 2>&1; then + log "Gateway cluster state synced" + return + fi + sleep 3 + done + die "Gateway node 2 did not receive synced cluster state" } wait_gateway_cert() { - local sni="gateway.${BASE_DOMAIN}" deadline=$((SECONDS + ${DSTACK_E2E_CERT_TIMEOUT:-240})) - log "waiting for Gateway TLS certificate with SNI $sni" + local node=$1 proxy deadline=$((SECONDS + 300)) sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") while (( SECONDS < deadline )); do - if echo | timeout 8 openssl s_client \ - -connect "127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ - -servername "$sni" 2>/dev/null \ + if echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ | openssl x509 -noout -ext subjectAltName 2>/dev/null \ | grep -Fq "*.${BASE_DOMAIN}"; then - log "Gateway wildcard certificate is active" - return 0 + log "Gateway node $node wildcard certificate active" + return fi - sleep 5 + sleep 3 done - die "timed out waiting for Gateway certificate" + die "Gateway node $node wildcard certificate not active" } gateway_cert_fingerprint() { - local sni="gateway.${BASE_DOMAIN}" - echo | timeout 8 openssl s_client \ - -connect "127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ - -servername "$sni" 2>/dev/null \ + local node=$1 proxy sni="gateway.${BASE_DOMAIN}" + read -r _ _ proxy _ < <(gateway_ports "$node") + echo | timeout 8 openssl s_client -connect "127.0.0.1:${proxy}" -servername "$sni" 2>/dev/null \ | openssl x509 -noout -fingerprint -sha256 2>/dev/null } wait_gateway_persisted() { - log "waiting for Gateway WaveKV persistence" - local deadline=$((SECONDS + 90)) status + local node=$1 deadline=$((SECONDS + 120)) status while (( SECONDS < deadline )); do - status=$(admin_curl WaveKvStatus 2>/dev/null || true) - if [[ -n "$status" ]] \ - && jq -e '.persistent.dirty == false and .persistent.n_keys > 0' <<<"$status" >/dev/null 2>&1; then - log "Gateway persistent store is clean ($(jq -r '.persistent.n_keys' <<<"$status") keys)" - return 0 + status=$(admin_curl "$node" WaveKvStatus 2>/dev/null || true) + if jq -e '.persistent.dirty == false and .persistent.n_keys > 0' <<<"$status" >/dev/null 2>&1; then + return fi - sleep 1 + sleep 2 done - die "Gateway persistent store did not flush" + die "Gateway node $node persistent store did not flush" } capture_gateway_state() { - local stage=$1 - log "capturing Gateway durable state ($stage)" - admin_curl Status | tee "$WORK_DIR/gateway-${stage}.status.raw.json" \ - | jq -S '{uuid, hosts: ([.hosts[] | {instance_id, ip, app_id, base_domain}] | sort_by(.instance_id))}' \ - > "$WORK_DIR/gateway-${stage}.status.json" - admin_curl GetCertbotConfig | tee "$WORK_DIR/gateway-${stage}.certbot.raw.json" \ - | jq -S '{acme_url, renew_interval_secs, renew_before_expiration_secs, renew_timeout_secs}' \ - > "$WORK_DIR/gateway-${stage}.certbot.json" - admin_curl ListDnsCredentials | tee "$WORK_DIR/gateway-${stage}.dns.raw.json" \ - | jq -S '{default_id, credentials: ([.credentials[] | {id, name, provider_type, cf_zone_id, cf_api_url, dns_txt_ttl, max_dns_wait}] | sort_by(.id))}' \ - > "$WORK_DIR/gateway-${stage}.dns.json" - admin_curl ListZtDomains | tee "$WORK_DIR/gateway-${stage}.domains.raw.json" \ - | jq -S '{domains: ([.domains[] | .config] | sort_by(.domain))}' \ - > "$WORK_DIR/gateway-${stage}.domains.json" - gateway_cert_fingerprint > "$WORK_DIR/gateway-${stage}.cert-fingerprint.txt" + local node=$1 stage=$2 + wait_gateway_persisted "$node" + admin_curl "$node" Status | tee "$WORK_DIR/gateway${node}-${stage}.status.raw.json" \ + | jq -S '. as $status | { + id: $status.id, + uuid: ([$status.nodes[] | select(.id == $status.id)][0].uuid), + hosts: ([$status.hosts[] | {instance_id,ip,app_id,base_domain}] | sort_by(.instance_id)) + }' \ + > "$WORK_DIR/gateway${node}-${stage}.status.json" + admin_curl "$node" GetCertbotConfig | jq -S \ + '{acme_url,renew_interval_secs,renew_before_expiration_secs,renew_timeout_secs}' \ + > "$WORK_DIR/gateway${node}-${stage}.certbot.json" + admin_curl "$node" ListDnsCredentials | jq -S \ + '{default_id,credentials:([.credentials[]|{ + id,name,provider_type, + cf_api_token_set: ((.cf_api_token // "") | length > 0), + cf_zone_id,cf_api_url,dns_txt_ttl,max_dns_wait + }]|sort_by(.id))}' \ + > "$WORK_DIR/gateway${node}-${stage}.dns.json" + admin_curl "$node" ListZtDomains | jq -S \ + '{domains:([.domains[]|(.config // .)]|sort_by(.domain))}' \ + > "$WORK_DIR/gateway${node}-${stage}.domains.json" + gateway_cert_fingerprint "$node" > "$WORK_DIR/gateway${node}-${stage}.cert.txt" } assert_gateway_state_unchanged() { - capture_gateway_state after - local file + local node=$1 file + capture_gateway_state "$node" after for file in status certbot dns domains; do - diff -u "$WORK_DIR/gateway-before.${file}.json" "$WORK_DIR/gateway-after.${file}.json" \ - || die "Gateway $file state changed across upgrade" + diff -u "$WORK_DIR/gateway${node}-before.${file}.json" \ + "$WORK_DIR/gateway${node}-after.${file}.json" \ + || die "Gateway node $node $file state changed across upgrade" done - diff -u "$WORK_DIR/gateway-before.cert-fingerprint.txt" "$WORK_DIR/gateway-after.cert-fingerprint.txt" \ - || die "Gateway wildcard certificate changed across upgrade" - log "Gateway registrations, identity, certbot config, DNS/domain config and certificate are unchanged" + diff -u "$WORK_DIR/gateway${node}-before.cert.txt" "$WORK_DIR/gateway${node}-after.cert.txt" \ + || die "Gateway node $node wildcard certificate changed across upgrade" + log "Gateway node $node durable state is unchanged" } -deploy_app() { - local label=$1 attestation_mode=$2 - local out meta app_id hash vm_id - log "rendering $label nginx app-compose (TDX $attestation_mode)" - meta=$(python3 /suite/scripts/app_compose.py nginx \ +assert_gateway_dns_credential_usable() { + local node=$1 result deadline=$((SECONDS + 90)) + log "proving Gateway node $node retained the Cloudflare credential" + while (( SECONDS < deadline )); do + result=$(admin_curl "$node" RenewZtDomainCert \ + "$(jq -cn --arg d "$BASE_DOMAIN" '{domain:$d,force:true}')") + printf '%s\n' "$result" > "$WORK_DIR/gateway${node}-post-upgrade-renew.json" + if jq -e '.renewed == true and .not_after > 0' <<<"$result" >/dev/null; then + wait_gateway_cert "$node" + return + fi + # WaveKV's distributed certificate lock may still be held by the peer's + # immediately preceding issuance. + sleep 3 + done + die "Gateway node $node could not issue with its persisted DNS credential" +} + +render_app() { + local label=$1 mode=$2 meta app_id hash + meta=$(/suite/scripts/app_compose.py nginx \ --name "${APP_NAME}-${label}" \ - --app-image "$APP_IMAGE" \ - --attestation-mode "$attestation_mode" \ + --image-ref "$APP_IMAGE_ID" \ + --image-archive app.tar \ + --artifact-port "$ARTIFACT_PORT" \ + --attestation-mode "$mode" \ --output "$WORK_DIR/${label}.app-compose.json") - read -r app_id hash < <(jq -r '"\(.appId) \(.composeHash)"' <<<"$meta") + printf '%s\n' "$meta" > "$WORK_DIR/${label}.manifest-meta.json" + app_id=$(jq -r .appId <<<"$meta") + hash=$(jq -r .composeHash <<<"$meta") printf '%s' "$app_id" > "$WORK_DIR/${label}.app_id" - local gateway_id - gateway_id=$(jq -r '.gatewayAppId' "$ALLOWLIST") - log "registering $label app in auth allowlist app_id=$app_id gateway_app_id=$gateway_id" - register_app "$app_id" "$hash" "$gateway_id" - log "deploying $label nginx app CVM" - out=$("${VMM_CLI[@]}" deploy \ + register_app "$app_id" "$hash" +} + +deploy_app() { + local label=$1 mode=$2 kms_url=$3 out vm_id + render_app "$label" "$mode" + log "deploying $label app CVM (forced TDX $mode)" + if ! out=$("${VMM_CLI[@]}" deploy \ --name "${SUITE_PREFIX}-${label}" \ --image "$IMAGE_NAME" \ --compose "$WORK_DIR/${label}.app-compose.json" \ + --kms-url "$kms_url" \ + --gateway-url "$GATEWAY1_URL" \ + --gateway-url "$GATEWAY2_URL" \ --vcpu "${DSTACK_E2E_APP_VCPU:-2}" \ --memory "${DSTACK_E2E_APP_MEMORY:-2048}" \ - --disk "${DSTACK_E2E_APP_DISK:-20}") + --disk "${DSTACK_E2E_APP_DISK:-20}" 2>&1); then + die "failed to deploy $label app CVM: $out" + fi printf '%s\n' "$out" | tee "$WORK_DIR/${label}.deploy.log" vm_id=$(sed -n 's/^Created VM with ID: //p' <<<"$out" | tail -n1) - [[ -n "$vm_id" ]] || die "failed to parse $label app VM id" + [[ -n "$vm_id" ]] || die "failed to parse $label VM id" printf '%s' "$vm_id" > "$WORK_DIR/${label}.vm_id" - wait_boot_done "$vm_id" "$label" "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" - assert_attestation_mode "$label" "$attestation_mode" + wait_boot_done "$vm_id" "$label" + assert_attestation_mode "$label" "$mode" } assert_attestation_mode() { local label=$1 expected=$2 id sys_config actual id=$(cat "$WORK_DIR/${label}.vm_id") sys_config="$VM_DIR/$id/shared/.sys-config.json" - [[ -s "$sys_config" ]] || die "missing $label sys config: $sys_config" + [[ -s "$sys_config" ]] || die "missing $label sys config" actual=$(jq -r '.vm_config | fromjson | .tdx_attestation_variant // "legacy"' "$sys_config") - [[ "$actual" == "$expected" ]] \ - || die "$label resolved attestation mode is $actual, expected $expected" - jq -e '.kms_urls | length > 0' "$sys_config" >/dev/null \ - || die "$label has no KMS URL" - log "$label booted with resolved TDX mode=$actual and completed KMS key provisioning" -} - -restart_app_after_kms_upgrade() { - local label=$1 id - id=$(cat "$WORK_DIR/${label}.vm_id") - log "force-stopping $label CVM to require a fresh boot-time KMS key request" - "${VMM_CLI[@]}" stop "$id" --force >/dev/null - wait_vm_stopped "$id" - log "restarting $label CVM against upgraded KMS" - "${VMM_CLI[@]}" start "$id" >/dev/null - # The Start RPC returns before the supervisor updates the persisted VM state. - # Avoid treating that short, expected `exited` window as a failed boot. - wait_vm_running "$id" "$label" - wait_boot_done "$id" "$label" "${DSTACK_E2E_APP_BOOT_TIMEOUT:-600}" - assert_attestation_mode "$label" legacy + [[ "$actual" == "$expected" ]] || die "$label mode=$actual, expected $expected" + jq -e '(.kms_urls | length) > 0 and (.gateway_urls | length) == 2' "$sys_config" >/dev/null + log "$label completed boot-time KMS key provisioning in TDX $actual mode" } verify_app_via_gateway() { - local label=$1 app_info instance_id sni url deadline - app_info=$(cat "$WORK_DIR/${label}.info.json") - instance_id=$(jq -r '.instance_id // ""' <<<"$app_info") - [[ -n "$instance_id" && "$instance_id" != "null" ]] || die "$label app has no instance_id" - sni="${instance_id}-80.${BASE_DOMAIN}" - url="https://${sni}:${GATEWAY_PROXY_HOST_PORT}/" - log "verifying $label app through Gateway: $url" - deadline=$((SECONDS + ${DSTACK_E2E_APP_HTTP_TIMEOUT:-240})) + local label=$1 node=$2 instance sni proxy url deadline + instance=$(jq -r '.instance_id' "$WORK_DIR/${label}.info.json") + read -r _ _ proxy _ < <(gateway_ports "$node") + sni="${instance}-80.${BASE_DOMAIN}" + url="https://${sni}:${proxy}/" + deadline=$((SECONDS + 300)) while (( SECONDS < deadline )); do - if curl -fsS -k --connect-to "${sni}:${GATEWAY_PROXY_HOST_PORT}:127.0.0.1:${GATEWAY_PROXY_HOST_PORT}" \ - "$url" > "$WORK_DIR/${label}.http.out" 2>"$WORK_DIR/${label}.http.err"; then - if grep -qi 'welcome to nginx' "$WORK_DIR/${label}.http.out"; then - log "$label Gateway -> app HTTP check passed" - return 0 - fi + if curl -kfsS --max-time 5 \ + --connect-to "${sni}:${proxy}:127.0.0.1:${proxy}" "$url" \ + | grep -qi 'welcome to nginx'; then + log "$label reachable through Gateway node $node" + return fi - sleep 5 + sleep 3 done - cat "$WORK_DIR/${label}.http.err" >&2 || true - die "timed out verifying $label app through Gateway" + die "$label not reachable through Gateway node $node" } -write_network_targets() { - local status label instance ip - status=$(admin_curl Status) - : > "$WORK_DIR/network-targets.tsv" +verify_all_routes() { + local label node for label in legacy lite; do - instance=$(jq -r '.instance_id' "$WORK_DIR/${label}.info.json") - ip=$(jq -r --arg id "$instance" '.hosts[] | select(.instance_id == $id) | .ip' <<<"$status") - [[ -n "$ip" && "$ip" != "null" ]] || die "Gateway has no WireGuard IP for $label ($instance)" - printf '%s\t%s\n' "$label" "$ip" >> "$WORK_DIR/network-targets.tsv" - curl -fsS --max-time 3 "http://${ip}:80/" | grep -qi 'welcome to nginx' \ - || die "cannot reach $label directly over WireGuard at $ip" + for node in 1 2; do + verify_app_via_gateway "$label" "$node" + done done - log "CVM WireGuard targets: $(tr '\n' ' ' < "$WORK_DIR/network-targets.tsv")" } -cleanup_after() { - if [[ "${DSTACK_E2E_CLEANUP_AFTER:-false}" != "true" ]]; then - return 0 - fi - log "DSTACK_E2E_CLEANUP_AFTER=true: removing suite VMs" - local f - for f in "$WORK_DIR"/*.vm_id; do - [[ -s "$f" ]] && remove_vm "$(cat "$f")" +restart_legacy_on_latest_kms() { + local id + id=$(cat "$WORK_DIR/legacy.vm_id") + stop_vm "$id" legacy + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" legacy + assert_attestation_mode legacy legacy +} + +upgrade_gateway_node() { + local node=$1 id + id=$(cat "$WORK_DIR/gateway${node}.vm_id") + stop_vm "$id" "Gateway node $node" + "${VMM_CLI[@]}" update-app-compose "$id" "$WORK_DIR/gateway-latest.app-compose.json" >/dev/null + "${VMM_CLI[@]}" update "$id" --kms-url "$KMS_LATEST_URL" >/dev/null + start_vm "$id" "gateway${node}-latest" + wait_gateway "$node" + wait_gateway_cert "$node" + gateway_version "$node" latest + # The newly restarted node must carry traffic before its peer is touched. + verify_app_via_gateway legacy "$node" + verify_app_via_gateway lite "$node" + assert_gateway_state_unchanged "$node" +} + +start_network_probe() { + rm -f "$WORK_DIR/network-probe."{ready,stop,result.json,log} + /suite/scripts/network-probe.sh > "$WORK_DIR/network-probe.log" 2>&1 & + probe_pid=$! + for _ in $(seq 1 120); do + [[ -e "$WORK_DIR/network-probe.ready" ]] && return + kill -0 "$probe_pid" 2>/dev/null || { + cat "$WORK_DIR/network-probe.log" >&2 + die "HA network probe exited during warmup" + } + sleep 1 done + die "HA network probe did not become ready" } -prepare_common() { - wait_vmm - clean_start - wait_kms - wait_gateway_admin - bootstrap_gateway_certbot - wait_gateway_cert -} - -phase_full() { - prepare_common - if [[ "$PLATFORM" == "tdx" ]]; then - deploy_app legacy legacy - deploy_app lite lite - verify_app_via_gateway legacy - verify_app_via_gateway lite - capture_kms_identity full - wait_gateway_persisted - capture_gateway_state full - write_network_targets - else - deploy_app app auto - verify_app_via_gateway app +stop_network_probe() { + touch "$WORK_DIR/network-probe.stop" + if ! wait "$probe_pid"; then + cat "$WORK_DIR/network-probe.log" >&2 + die "CVM traffic had downtime during rolling Gateway upgrade" fi - log "E2E success" - cleanup_after + probe_pid="" + cat "$WORK_DIR/network-probe.log" + jq -e '.attempts > 0 and .failures == 0 and .successful_failovers > 0' \ + "$WORK_DIR/network-probe.result.json" >/dev/null \ + || die "HA probe reported downtime" +} + +assert_no_insecure_shortcuts() { + log "auditing rendered manifests for production trust settings" + local manifest + local forbidden='quote_enabled[[:space:]]*=[[:space:]]*false|enforce_self_authorization[[:space:]]*=[[:space:]]*false|verify[[:space:]]*=[[:space:]]*false' + if grep -R -E "$forbidden" "$WORK_DIR"/*.app-compose.json; then + die "rendered app manifest contains a forbidden development trust setting" + fi + + for manifest in "$WORK_DIR/kms-old.app-compose.json" \ + "$WORK_DIR/kms-latest.app-compose.json"; do + if ! python3 - "$manifest" <<'PY' +import json +import sys +import tomllib + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +assert manifest["local_key_provider_enabled"] is True +assert manifest["kms_enabled"] is False + +# Parse the actual kms.toml embedded in the rendered Compose YAML. Checking +# TOML values instead of matching adjacent strings keeps this guard strict +# while allowing comments and blank lines in the production-style config. +marker = "configs:\n kms_config:\n content: |\n" +compose = manifest["docker_compose_file"] +assert compose.count(marker) == 1 +indented_config = compose.split(marker, 1)[1] +config_lines = [] +for line in indented_config.splitlines(): + if line and not line.startswith(" "): + break + config_lines.append(line[6:] if line else "") +config = tomllib.loads("\n".join(config_lines)) + +core = config["core"] +assert core["enforce_self_authorization"] is True +assert core["image"]["verify"] is True +assert core["auth_api"]["type"] == "webhook" +assert core["onboard"]["enabled"] is True +assert core["onboard"]["quote_enabled"] is True +PY + then + die "$manifest does not use quote-attested production KMS settings" + fi + done + + for manifest in "$WORK_DIR/gateway-old.app-compose.json" \ + "$WORK_DIR/gateway-latest.app-compose.json"; do + jq -e '.kms_enabled == true and .local_key_provider_enabled == false' \ + "$manifest" >/dev/null \ + || die "$manifest does not obtain its keys from KMS" + done + + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == true + ' "$WORK_DIR/legacy.app-compose.json" >/dev/null \ + || die "legacy app manifest did not force the legacy TDX verifier" + jq -e ' + .kms_enabled == true and .gateway_enabled == true + and .manifest_version == "3" + and .requirements.tdx_measure_acpi_tables == false + ' "$WORK_DIR/lite.app-compose.json" >/dev/null \ + || die "lite app manifest did not force the lite TDX verifier" + + jq -e --arg id "$GATEWAY_APP_ID" \ + --arg device "$ATTESTED_DEVICE_ID" \ + '.gatewayAppId == $id and .gatewayAppId != "any" + and (.osImages | length) == 1 + and (.kms.mrAggregated | length) == 2 + and (.kms.allowAnyDevice == false) + and (.kms.devices == [$device]) + and (.apps | length) == 3 + and ([.apps[].allowAnyDevice] | all(. == false)) + and ([.apps[].devices] | all(. == [$device]))' \ + "$ALLOWLIST" >/dev/null +} + +save_vm_logs() { + local name file id + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] || continue + name=$(basename "$file" .vm_id) + id=$(cat "$file") + "${VMM_CLI[@]}" logs "$id" -n 2000 > "$WORK_DIR/${name}.vm.log" 2>&1 || true + done } -phase_prepare_old() { - require_tdx - prepare_common - deploy_app legacy legacy - verify_app_via_gateway legacy - capture_kms_identity before - printf 'ok\n' > "$WORK_DIR/prepare-old.done" - log "old KMS/Gateway phase complete" +cleanup_after() { + [[ "${DSTACK_E2E_CLEANUP_AFTER:-false}" == true ]] || return 0 + local file + for file in "$WORK_DIR"/*.vm_id; do + [[ -s "$file" ]] && remove_vm "$(cat "$file")" + done } -phase_after_kms_upgrade() { - require_tdx +phase_upgrade() { + [[ "$PLATFORM" == tdx ]] || die "upgrade phase requires TDX" wait_vmm - wait_kms - wait_gateway_admin + clean_start + + render_kms old "$OLD_KMS_IMAGE_ID" kms-0.5.8.tar + deploy_kms_onboard old "$KMS_OLD_HOST_PORT" + wait_onboard old "$KMS_OLD_HOST_PORT" + authorize_kms_from_attestation old + onboard_rpc "$KMS_OLD_HOST_PORT" Bootstrap \ + "$(jq -cn --arg d "$KMS_RPC_DOMAIN" '{domain:$d}')" \ + "$WORK_DIR/kms-old.bootstrap.json" + jq -e '(.attestation // "") | length > 0' "$WORK_DIR/kms-old.bootstrap.json" >/dev/null \ + || die "KMS 0.5.8 bootstrap did not return quote-bound attestation" + log "KMS 0.5.8 bootstrap returned quote-bound attestation" + finish_onboarding old "$KMS_OLD_HOST_PORT" + + render_gateway_manifests old "$OLD_GATEWAY_IMAGE_ID" gateway-0.5.8.tar + render_gateway_manifests latest "$CURRENT_GATEWAY_IMAGE_ID" gateway-current.tar + write_gateway_env 1 + write_gateway_env 2 + deploy_gateway 1 old "$KMS_OLD_URL" + gateway_version 1 old + deploy_gateway 2 old "$KMS_OLD_URL" + gateway_version 2 old + bootstrap_gateway + + deploy_app legacy legacy "$KMS_OLD_URL" + verify_app_via_gateway legacy 1 + verify_app_via_gateway legacy 2 + capture_kms_identity old "$KMS_OLD_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" + + # Production KMS upgrades are rolling, quote-attested replication into a new + # Local-Key-Provider CVM, not an unmeasured binary swap in the old CVM. + render_kms latest "$CURRENT_KMS_IMAGE_ID" kms-current.tar + deploy_kms_onboard latest "$KMS_LATEST_HOST_PORT" + wait_onboard latest "$KMS_LATEST_HOST_PORT" + authorize_kms_from_attestation latest + onboard_rpc "$KMS_LATEST_HOST_PORT" Onboard \ + "$(jq -cn --arg s "$KMS_OLD_URL" --arg d "$KMS_RPC_DOMAIN" '{source_url:$s,domain:$d}')" \ + "$WORK_DIR/kms-latest.onboard.json" + # Current Onboard returns only the replicated public key. Quote evidence is + # carried by the mutual RA-TLS connection and authorized above through the + # target's exact quote-derived measurement and physical TDX device ID. + finish_onboarding latest "$KMS_LATEST_HOST_PORT" + capture_kms_identity latest "$KMS_LATEST_HOST_PORT" "$(cat "$WORK_DIR/legacy.app_id")" assert_kms_identity_unchanged - restart_app_after_kms_upgrade legacy - verify_app_via_gateway legacy - deploy_app lite lite - verify_app_via_gateway lite - wait_gateway_persisted - capture_gateway_state before - write_network_targets - printf 'ok\n' > "$WORK_DIR/after-kms-upgrade.done" - log "KMS upgrade and latest lite/legacy key-provisioning phase complete" -} - -phase_after_gateway_upgrade() { - require_tdx - wait_vmm - wait_kms - wait_gateway_admin - wait_gateway_cert - assert_gateway_state_unchanged - verify_app_via_gateway legacy - verify_app_via_gateway lite - write_network_targets - printf 'ok\n' > "$WORK_DIR/after-gateway-upgrade.done" - log "Gateway upgrade compatibility phase complete" + + stop_vm "$(cat "$WORK_DIR/kms-old.vm_id")" "KMS 0.5.8 after cutover" + restart_legacy_on_latest_kms + deploy_app lite lite "$KMS_LATEST_URL" + verify_all_routes + + capture_gateway_state 1 before + capture_gateway_state 2 before + start_network_probe + upgrade_gateway_node 1 + upgrade_gateway_node 2 + stop_network_probe + verify_all_routes + # The current API redacts stored tokens, so metadata equality alone cannot + # prove the secret survived. The mock rejects any token other than the exact + # original value; a forced issuance through each upgraded node proves it. + assert_gateway_dns_credential_usable 1 + assert_gateway_dns_credential_usable 2 + + assert_no_insecure_shortcuts + save_vm_logs + # dstack-kms runs in an inner container, whose stdout is not part of the CVM + # serial log returned by VMM. Each KMS has a fresh data disk, so two 200 GETs + # for the exact measured archive prove that both verifiers downloaded it. + local os_archive_gets + os_archive_gets=$(grep -Fc \ + "GET /os/mr_${OS_IMAGE_HASH}.tar.gz HTTP/1.1\" 200" \ + "$WORK_DIR/artifacts-access.log" || true) + (( os_archive_gets >= 2 )) \ + || die "expected verified OS image downloads by both KMS versions, saw ${os_archive_gets}" + if grep -E 'Image verification is disabled|self-authorization is disabled' \ + "$WORK_DIR/kms-"*.vm.log; then + die "KMS logs contain a forbidden disabled verification path" + fi + + log "production-compatible KMS/Gateway rolling-upgrade E2E success" cleanup_after } +on_exit() { + local rc=$? + if [[ -n "$probe_pid" ]] && kill -0 "$probe_pid" 2>/dev/null; then + touch "$WORK_DIR/network-probe.stop" 2>/dev/null || true + wait "$probe_pid" 2>/dev/null || true + fi + if (( rc != 0 )); then + save_vm_logs || true + fi + exit "$rc" +} +trap on_exit EXIT + main() { - need_current_bins - log "running E2E phase=$PHASE" + need_bin /workspace/target/release/dstack + [[ -s "$STATE_DIR/artifacts/images.env" ]] || die "missing prepared image metadata" case "$PHASE" in - full) phase_full ;; - prepare-old) phase_prepare_old ;; - after-kms-upgrade) phase_after_kms_upgrade ;; - after-gateway-upgrade) phase_after_gateway_upgrade ;; + upgrade) phase_upgrade ;; *) die "unknown DSTACK_E2E_PHASE=$PHASE" ;; esac - log "VMM dashboard: $VMM_URL" log "Work artifacts: $WORK_DIR" } From 4d014d3b5ad580e4edc643c0d91227f92667e3fc Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 20 Jul 2026 00:45:57 -0700 Subject: [PATCH 4/4] Fix E2E key provider integration after workspace migration --- .../gateway/dstack-app/bootstrap-cluster.sh | 34 +++++-------------- dstack/gateway/dstack-app/deploy-to-vmm.sh | 6 ---- test-suites/full-stack-compose/.env.example | 2 +- test-suites/full-stack-compose/compose.yml | 16 +++++---- 4 files changed, 20 insertions(+), 38 deletions(-) diff --git a/dstack/gateway/dstack-app/bootstrap-cluster.sh b/dstack/gateway/dstack-app/bootstrap-cluster.sh index 90e7ed477..f1ad6b48b 100755 --- a/dstack/gateway/dstack-app/bootstrap-cluster.sh +++ b/dstack/gateway/dstack-app/bootstrap-cluster.sh @@ -33,7 +33,7 @@ echo "Waiting for gateway admin API at $ADMIN_ADDR..." max_retries=60 retry=0 while [ $retry -lt $max_retries ]; do - if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.Status" >/dev/null 2>&1; then + if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Status" >/dev/null 2>&1; then break fi retry=$((retry + 1)) @@ -48,45 +48,29 @@ fi echo "Admin API ready, bootstrapping configuration..." -# Set ACME URL. ACME_URL is useful for local E2E suites (for example Pebble). -if [ -n "${ACME_URL:-}" ]; then - : -elif [ "$ACME_STAGING" = "yes" ]; then +# Set ACME URL +if [ "$ACME_STAGING" = "yes" ]; then ACME_URL="https://acme-staging-v02.api.letsencrypt.org/directory" else ACME_URL="https://acme-v02.api.letsencrypt.org/directory" fi echo "Setting certbot config (ACME URL: $ACME_URL)..." -curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.SetCertbotConfig" \ +curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \ -H "Content-Type: application/json" \ -d '{"acme_url":"'"$ACME_URL"'","renew_interval_secs":3600,"renew_before_expiration_secs":864000,"renew_timeout_secs":300}' >/dev/null \ && echo " Certbot config set" || echo " WARN: failed to set certbot config" # Create DNS credential if CF_API_TOKEN is provided and no credentials exist yet if [ -n "$CF_API_TOKEN" ]; then - existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListDnsCredentials" 2>/dev/null) + existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListDnsCredentials" 2>/dev/null) cred_count=$(echo "$existing" | jq -r '.credentials | length' 2>/dev/null || echo "0") if [ "$cred_count" = "0" ]; then echo "Creating default DNS credential..." - dns_payload=$(jq -cn \ - --arg token "$CF_API_TOKEN" \ - --arg api_url "${CF_API_URL:-}" \ - --argjson ttl "${DNS_TXT_TTL:-60}" \ - --argjson max_wait "${MAX_DNS_WAIT:-300}" \ - '{ - name: "cloudflare", - provider_type: "cloudflare", - cf_api_token: $token, - set_as_default: true, - dns_txt_ttl: $ttl, - max_dns_wait: $max_wait - } - + (if $api_url == "" then {} else {cf_api_url: $api_url} end)') - curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.CreateDnsCredential" \ + curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \ -H "Content-Type: application/json" \ - -d "$dns_payload" >/dev/null \ + -d '{"name":"cloudflare","provider_type":"cloudflare","cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' >/dev/null \ && echo " DNS credential created" || echo " WARN: failed to create DNS credential" else echo " DNS credentials already exist ($cred_count), skipping" @@ -97,12 +81,12 @@ fi # Add ZT-Domain if SRV_DOMAIN is provided and domain doesn't exist yet if [ -n "$SRV_DOMAIN" ]; then - existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.ListZtDomains" 2>/dev/null) + existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" 2>/dev/null) has_domain=$(echo "$existing" | jq -r '.domains[]? | select(.domain=="'"$SRV_DOMAIN"'") | .domain' 2>/dev/null) if [ -z "$has_domain" ]; then echo "Adding ZT-Domain: $SRV_DOMAIN..." - curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Admin.AddZtDomain" \ + curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \ -H "Content-Type: application/json" \ -d '{"domain":"'"$SRV_DOMAIN"'","port":443,"priority":100}' >/dev/null \ && echo " ZT-Domain added" || echo " WARN: failed to add ZT-Domain" diff --git a/dstack/gateway/dstack-app/deploy-to-vmm.sh b/dstack/gateway/dstack-app/deploy-to-vmm.sh index 8e363f52b..5ae76a87c 100755 --- a/dstack/gateway/dstack-app/deploy-to-vmm.sh +++ b/dstack/gateway/dstack-app/deploy-to-vmm.sh @@ -64,12 +64,6 @@ NODE_ID=1 # Whether to use ACME staging (yes/no) ACME_STAGING=no -# Optional ACME directory override for local E2E (for example Pebble). -# ACME_URL=http://10.0.2.2:14000/dir -# Optional Cloudflare API override for local E2E mock. -# CF_API_URL=http://10.0.2.2:18080/client/v4 -# DNS_TXT_TTL=60 -# MAX_DNS_WAIT=300 # Networking mode: bridge or user (default: user) # NET_MODE=bridge diff --git a/test-suites/full-stack-compose/.env.example b/test-suites/full-stack-compose/.env.example index 28c113a2e..ad86f18ec 100644 --- a/test-suites/full-stack-compose/.env.example +++ b/test-suites/full-stack-compose/.env.example @@ -51,7 +51,7 @@ DSTACK_E2E_QGS_PORT=4050 DSTACK_E2E_QEMU_PATH=/usr/bin/qemu-system-x86_64 # SGX QCNL config used by the Local-Key-Provider that seals each KMS CVM disk. -DSTACK_E2E_QCNL_CONF=../../dstack/key-provider-build/sgx_default_qcnl.conf +DSTACK_E2E_QCNL_CONF=../../dstack/local-key-provider/build/sgx_default_qcnl.conf # Runner behaviour. DSTACK_E2E_CLEAN_START=true diff --git a/test-suites/full-stack-compose/compose.yml b/test-suites/full-stack-compose/compose.yml index b42560acf..f2fb877ad 100644 --- a/test-suites/full-stack-compose/compose.yml +++ b/test-suites/full-stack-compose/compose.yml @@ -100,8 +100,8 @@ services: aesmd: build: - context: ../../dstack/key-provider-build - dockerfile: Dockerfile.aesmd + context: ../../dstack/local-key-provider + dockerfile: build/Dockerfile.aesmd image: dstack-e2e-aesmd:local privileged: true network_mode: host @@ -109,19 +109,20 @@ services: - "/dev/sgx_enclave:/dev/sgx_enclave" - "/dev/sgx_provision:/dev/sgx_provision" volumes: - - ${DSTACK_E2E_QCNL_CONF:-../../dstack/key-provider-build/sgx_default_qcnl.conf}:/etc/sgx_default_qcnl.conf:ro + - ${DSTACK_E2E_QCNL_CONF:-../../dstack/local-key-provider/build/sgx_default_qcnl.conf}:/etc/sgx_default_qcnl.conf:ro - aesmd-sock:/var/run/aesmd/ restart: unless-stopped local-keyprovider: build: - context: ../../dstack/key-provider-build - dockerfile: Dockerfile.key-provider + context: ../../dstack + dockerfile: local-key-provider/build/Dockerfile.key-provider args: APT_SNAPSHOT: ${APT_SNAPSHOT:-20260423T000000Z} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1776902400} RUSTUP_VERSION: ${RUSTUP_VERSION:-1.28.2} RUSTUP_INIT_SHA256: ${RUSTUP_INIT_SHA256:-20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c} - RUST_TOOLCHAIN: ${RUST_TOOLCHAIN:-1.85.1} + RUST_TOOLCHAIN: ${RUST_TOOLCHAIN:-1.92.0} image: dstack-e2e-local-keyprovider:local privileged: true devices: @@ -129,6 +130,9 @@ services: - "/dev/sgx_provision:/dev/sgx_provision" depends_on: - aesmd + environment: + PCCS_URL: ${PCCS_URL:-https://pccs.phala.network} + LISTEN_ADDR: 0.0.0.0:3443 volumes: - aesmd-sock:/var/run/aesmd/ ports: