From bfa7c86972024b8e3c8dfd0bf5d888f262c5f647 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 04:10:06 -0700 Subject: [PATCH 1/4] feat: add nerdctl compose snapshotter runner --- docs/nerdctl-compose.md | 63 +++++++ dstack/crates/dstack-cli-core/src/compose.rs | 47 ++++- dstack/crates/dstack-cli/src/main.rs | 56 +++++- dstack/dstack-types/src/lib.rs | 31 ++++ dstack/dstack-util/src/system_setup.rs | 29 +++ dstack/guest-agent/src/rpc_service.rs | 1 + os/common/rootfs/app-compose.service | 6 +- os/common/rootfs/app-compose.sh | 167 ++++++++++++++---- os/common/rootfs/dstack-prepare.sh | 24 ++- .../files/containerd-stargz-grpc.service | 16 ++ .../stargz-snapshotter_0.18.2.bb | 29 +++ .../images/dstack-rootfs-base.inc | 3 + sdk/go/dstack/compose_hash.go | 3 +- sdk/js/src/get-compose-hash.ts | 3 +- sdk/python/src/dstack_sdk/get_compose_hash.py | 4 +- 15 files changed, 440 insertions(+), 42 deletions(-) create mode 100644 docs/nerdctl-compose.md create mode 100644 os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/files/containerd-stargz-grpc.service create mode 100644 os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb diff --git a/docs/nerdctl-compose.md b/docs/nerdctl-compose.md new file mode 100644 index 000000000..83ecf256b --- /dev/null +++ b/docs/nerdctl-compose.md @@ -0,0 +1,63 @@ +# nerdctl Compose and lazy image pulling + +dstack supports two independent Compose runners. The runner is part of +`app-compose.json`, so it is included in the compose hash and attestation. + +| `runner` | Image manager | `snapshotter` support | +|---|---|---| +| `docker-compose` | Docker Engine | none (Docker's overlayfs store) | +| `nerdctl-compose` | containerd | `overlayfs` or `stargz` | + +## Deploy with stargz + +Build and push an eStargz image before deployment. For example, with Buildx: + +```bash +docker buildx build -t registry.example.com/example/app:estargz \ + --output type=registry,oci-mediatypes=true,compression=estargz,force-compression=true \ + . +``` + +Reference that image from `docker-compose.yaml`, then deploy it with: + +```bash +dstack deploy -c docker-compose.yaml \ + --runner nerdctl-compose \ + --snapshotter stargz +``` + +The resulting application manifest contains: + +```json +{ + "manifest_version": "3", + "runner": "nerdctl-compose", + "snapshotter": "stargz" +} +``` + +Use containerd without lazy pulling by selecting overlayfs: + +```bash +dstack deploy -c docker-compose.yaml \ + --runner nerdctl-compose \ + --snapshotter overlayfs +``` + +If `snapshotter` is omitted for `nerdctl-compose`, it defaults to `overlayfs`. +Setting `snapshotter` with `docker-compose` is rejected rather than silently +ignored. + +## Compatibility + +`nerdctl compose` implements the commonly used Docker Compose features, but it +is not a drop-in implementation of every Docker-specific extension. Test +applications that use Docker socket mounts, custom runtimes, or advanced +networking before switching runners. The `nerdctl-compose` runner requires +pre-built images and rejects Compose `build` sections. This avoids depending +on an in-guest BuildKit daemon and ensures lazy-pull images were converted +before deployment. + +Both backends keep their own image and container metadata. Changing the runner +recreates the application through the selected backend; it does not migrate +existing Docker containers into containerd. diff --git a/dstack/crates/dstack-cli-core/src/compose.rs b/dstack/crates/dstack-cli-core/src/compose.rs index 5c395213d..4ae252390 100644 --- a/dstack/crates/dstack-cli-core/src/compose.rs +++ b/dstack/crates/dstack-cli-core/src/compose.rs @@ -13,10 +13,28 @@ use serde_json::json; /// `kms_enabled` selects KMS mode (deterministic, upgradeable per-app keys); /// gateway and local-key-provider are off for the direct-port single-node flow. pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: bool) -> String { - let manifest = json!({ - "manifest_version": 2, + build_app_compose_with_runtime( + name, + docker_compose_yaml, + kms_enabled, + "docker-compose", + None, + ) +} + +/// Build an app-compose manifest with an explicitly selected compose frontend. +/// `snapshotter` is meaningful only for `nerdctl-compose`. +pub fn build_app_compose_with_runtime( + name: &str, + docker_compose_yaml: &str, + kms_enabled: bool, + runner: &str, + snapshotter: Option<&str>, +) -> String { + let mut manifest = json!({ + "manifest_version": if runner == "nerdctl-compose" { json!("3") } else { json!(2) }, "name": name, - "runner": "docker-compose", + "runner": runner, "docker_compose_file": docker_compose_yaml, "kms_enabled": kms_enabled, "gateway_enabled": false, @@ -31,7 +49,30 @@ pub fn build_app_compose(name: &str, docker_compose_yaml: &str, kms_enabled: boo // (NTS is also currently broken in guest images — see dstack#745.) "secure_time": false, }); + if let Some(snapshotter) = snapshotter { + manifest["snapshotter"] = json!(snapshotter); + } // pretty-print via Value's Display (`{:#}`) — infallible, and byte-identical // to serde_json::to_string_pretty (avoids an expect on an unfailable Result). format!("{manifest:#}") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn nerdctl_manifest_uses_v3_and_records_snapshotter() { + let body = build_app_compose_with_runtime( + "test", + "services: {}", + false, + "nerdctl-compose", + Some("stargz"), + ); + let value: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(value["manifest_version"], "3"); + assert_eq!(value["runner"], "nerdctl-compose"); + assert_eq!(value["snapshotter"], "stargz"); + } +} diff --git a/dstack/crates/dstack-cli/src/main.rs b/dstack/crates/dstack-cli/src/main.rs index 9ee0a662e..5f047c15a 100644 --- a/dstack/crates/dstack-cli/src/main.rs +++ b/dstack/crates/dstack-cli/src/main.rs @@ -11,7 +11,7 @@ //! `logs`, a global `-j/--json`). use anyhow::{bail, Context, Result}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use dstack_cli_core::layout::InstallLayout; use dstack_cli_core::vmm::{Vmm, DEFAULT_HOST}; use dstack_cli_core::{compose, ports, rpc}; @@ -46,6 +46,37 @@ struct Cli { command: Command, } +#[derive(Clone, Copy, Debug, Default, ValueEnum)] +enum ComposeRunner { + #[default] + DockerCompose, + NerdctlCompose, +} + +impl ComposeRunner { + fn as_str(self) -> &'static str { + match self { + Self::DockerCompose => "docker-compose", + Self::NerdctlCompose => "nerdctl-compose", + } + } +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum Snapshotter { + Overlayfs, + Stargz, +} + +impl Snapshotter { + fn as_str(self) -> &'static str { + match self { + Self::Overlayfs => "overlayfs", + Self::Stargz => "stargz", + } + } +} + #[derive(Subcommand)] enum Command { /// Deploy an app from a docker-compose file. @@ -84,6 +115,12 @@ enum Command { /// build + hash the compose and print it, without deploying. #[arg(long)] dry_run: bool, + /// compose frontend used inside the guest. + #[arg(long, value_enum, default_value = "docker-compose")] + runner: ComposeRunner, + /// containerd snapshotter (supported only with --runner nerdctl-compose). + #[arg(long, value_enum)] + snapshotter: Option, }, /// List deployed apps. Apps, @@ -144,6 +181,8 @@ async fn main() -> Result<()> { no_kms, allowlist, dry_run, + runner, + snapshotter, } => { let compose = resolve_compose_arg(compose, compose_file)?; let image = if use_local_defaults { @@ -174,6 +213,8 @@ async fn main() -> Result<()> { allowlist.as_deref(), dry_run, json, + runner, + snapshotter, ) .await } @@ -347,10 +388,21 @@ async fn cmd_deploy( allowlist: Option<&str>, dry_run: bool, json: bool, + runner: ComposeRunner, + snapshotter: Option, ) -> Result<()> { + if matches!(runner, ComposeRunner::DockerCompose) && snapshotter.is_some() { + bail!("--snapshotter is only supported with --runner nerdctl-compose"); + } let yaml = std::fs::read_to_string(compose_path) .with_context(|| format!("reading compose file '{compose_path}'"))?; - let app_compose = compose::build_app_compose(name, &yaml, !no_kms); + let app_compose = compose::build_app_compose_with_runtime( + name, + &yaml, + !no_kms, + runner.as_str(), + snapshotter.map(Snapshotter::as_str), + ); let mut port_maps = Vec::new(); for spec in port_specs { diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 841c5fdb4..e0ee32e86 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -80,6 +80,10 @@ pub struct AppCompose { #[serde(default)] pub features: Vec, pub runner: String, + /// containerd snapshotter used by the `nerdctl-compose` runner. + /// The field is invalid for other runners. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub snapshotter: Option, #[serde(default)] pub docker_compose_file: Option, #[serde(default)] @@ -121,6 +125,13 @@ pub struct AppCompose { pub requirements: Option, } +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum ContainerSnapshotter { + Overlayfs, + Stargz, +} + /// Canonical source for the policy used when `requirements.gpu_policy` is /// absent. Both typed defaults and measurement are derived from this JSON. pub const DEFAULT_GPU_POLICY: &str = "{}"; @@ -445,6 +456,26 @@ mod app_compose_tests { assert_eq!(compose.manifest_version_u32(), Some(3)); } + #[test] + fn parses_supported_container_snapshotters() { + let compose: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "nerdctl-compose", + "snapshotter": "stargz" + })) + .unwrap(); + assert_eq!(compose.snapshotter, Some(ContainerSnapshotter::Stargz)); + + let invalid = serde_json::from_value::(serde_json::json!({ + "manifest_version": "3", + "name": "test", + "runner": "nerdctl-compose", + "snapshotter": "unknown" + })); + assert!(invalid.is_err()); + } + #[test] fn manifest_version_accepts_legacy_numeric_1_and_2() { assert_eq!( diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 7512ec29e..ae04971c6 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -839,6 +839,14 @@ fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()> "requirements requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" ); } + if app_compose.runner == "nerdctl-compose" && manifest_version < MANIFEST_VERSION_3 { + bail!( + "nerdctl-compose requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" + ); + } + if app_compose.runner != "nerdctl-compose" && app_compose.snapshotter.is_some() { + bail!("snapshotter is only supported by the nerdctl-compose runner"); + } Ok(()) } @@ -3162,6 +3170,27 @@ fn test_os_version_requirement_requires_v3_manifest() { assert!(err.to_string().contains("requires manifest_version")); } +#[test] +fn test_nerdctl_compose_requires_v3_manifest() { + let mut app_compose = test_app_compose(serde_json::json!(2), None, None); + app_compose.runner = "nerdctl-compose".to_string(); + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err.to_string().contains("nerdctl-compose requires")); + + app_compose.manifest_version = "3".to_string(); + verify_manifest_feature_requirements(&app_compose).unwrap(); +} + +#[test] +fn test_snapshotter_is_rejected_for_other_runners() { + let mut app_compose = test_app_compose(serde_json::json!("3"), None, None); + app_compose.snapshotter = Some(dstack_types::ContainerSnapshotter::Stargz); + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err + .to_string() + .contains("snapshotter is only supported by the nerdctl-compose runner")); +} + #[test] fn test_os_version_requirement_rejects_too_old_os() { let app_compose = test_app_compose(serde_json::json!("3"), Some(">=0.6.1"), None); diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e20126095..4da98f6ad 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -735,6 +735,7 @@ mod tests { name: String::new(), features: Vec::new(), runner: String::new(), + snapshotter: None, docker_compose_file: None, public_logs: false, public_sysinfo: false, diff --git a/os/common/rootfs/app-compose.service b/os/common/rootfs/app-compose.service index b92ef21e1..b3979e255 100644 --- a/os/common/rootfs/app-compose.service +++ b/os/common/rootfs/app-compose.service @@ -1,7 +1,7 @@ [Unit] Description=App Compose Service -Wants=docker.service -After=docker.service dstack-prepare.service dstack-guest-agent.service +Wants=docker.service containerd.service containerd-stargz-grpc.service +After=docker.service containerd.service containerd-stargz-grpc.service dstack-prepare.service dstack-guest-agent.service [Service] Type=oneshot @@ -9,7 +9,7 @@ RemainAfterExit=true EnvironmentFile=-/dstack/.host-shared/.decrypted-env WorkingDirectory=/dstack ExecStart=/bin/app-compose.sh -ExecStop=/bin/docker compose stop +ExecStop=/bin/app-compose.sh stop StandardOutput=journal+console StandardError=journal+console diff --git a/os/common/rootfs/app-compose.sh b/os/common/rootfs/app-compose.sh index ea5ed9de0..964c3199e 100755 --- a/os/common/rootfs/app-compose.sh +++ b/os/common/rootfs/app-compose.sh @@ -1,49 +1,156 @@ #!/bin/bash -# SPDX-FileCopyrightText: © 2024-2025 Phala Network +# SPDX-FileCopyrightText: © 2024-2026 Phala Network # # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + HOST_SHARED_DIR="/dstack/.host-shared" SYS_CONFIG_FILE="$HOST_SHARED_DIR/.sys-config.json" +APP_COMPOSE_FILE="${APP_COMPOSE_FILE:-app-compose.json}" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml}" +NERDCTL_NAMESPACE="${NERDCTL_NAMESPACE:-dstack}" +ACTION="${1:-start}" + CFG_PCCS_URL=$([ -f "$SYS_CONFIG_FILE" ] && jq -r '.pccs_url//""' "$SYS_CONFIG_FILE" || echo "") export PCCS_URL=${PCCS_URL:-$CFG_PCCS_URL} -if [ "$(jq 'has("pre_launch_script")' app-compose.json)" = true ]; then - echo "Running pre-launch script" - dstack-util notify-host -e "boot.progress" -d "pre-launch" || true - # shellcheck disable=SC1090 - source <(jq -r '.pre_launch_script' app-compose.json) -fi - -RUNNER=$(jq -r '.runner' app-compose.json) -case "$RUNNER" in -"docker-compose") - echo "Starting containers" - dstack-util notify-host -e "boot.progress" -d "starting containers" || true - if ! [ -f docker-compose.yaml ]; then - jq -r '.docker_compose_file' app-compose.json >docker-compose.yaml +runner=$(jq -r '.runner' "$APP_COMPOSE_FILE") +snapshotter=$(jq -r '.snapshotter // "overlayfs"' "$APP_COMPOSE_FILE") + +ensure_compose_file() { + if ! [ -f "$COMPOSE_FILE" ]; then + jq -r '.docker_compose_file' "$APP_COMPOSE_FILE" >"$COMPOSE_FILE" fi +} - if ! docker compose up --remove-orphans -d --build; then - dstack-util notify-host -e "boot.error" -d "failed to start containers" +validate_runner() { + case "$runner" in + docker-compose) + if jq -e 'has("snapshotter")' "$APP_COMPOSE_FILE" >/dev/null; then + echo "ERROR: snapshotter is only supported by the nerdctl-compose runner" >&2 + exit 1 + fi + ;; + nerdctl-compose) + case "$snapshotter" in + overlayfs|stargz) ;; + *) + echo "ERROR: unsupported snapshotter for nerdctl-compose: $snapshotter" >&2 + exit 1 + ;; + esac + ;; + bash) ;; + *) + echo "ERROR: unsupported runner: $runner" >&2 exit 1 + ;; + esac +} + +compose_start() { + ensure_compose_file + case "$runner" in + docker-compose) + docker compose -f "$COMPOSE_FILE" up --remove-orphans -d --build + ;; + nerdctl-compose) + if docker compose -f "$COMPOSE_FILE" config --format json | jq -e \ + 'any(.services[]; has("build"))' >/dev/null; then + echo "ERROR: nerdctl-compose requires pre-built images; Compose build sections are not supported" >&2 + return 1 + fi + if [ "$snapshotter" = stargz ]; then + # containerd's transfer-service pull path does not add the remote + # snapshot annotations required by stargz. Register every Compose + # image with ctr-remote before nerdctl creates the containers. + while IFS= read -r image; do + [ -n "$image" ] || continue + auth_args=() + registry=${image%%/*} + case "$registry" in + *.*|*:*|localhost) ;; + *) registry="docker.io" ;; + esac + if [ -r /root/.docker/config.json ]; then + auth=$(jq -r --arg registry "$registry" ' + .auths[$registry].auth // + (if $registry == "docker.io" then .auths["https://index.docker.io/v1/"].auth else empty end) // empty + ' /root/.docker/config.json) + if [ -n "$auth" ]; then + auth_args=(--user "$(printf '%s' "$auth" | base64 -d)") + fi + fi + ctr-remote --namespace "$NERDCTL_NAMESPACE" images rpull \ + --snapshotter "$snapshotter" "${auth_args[@]}" "$image" + done < <(docker compose -f "$COMPOSE_FILE" config --images) + fi + compose_args=(up --remove-orphans -d) + if [ "$snapshotter" = stargz ]; then + compose_args+=(--pull never) + fi + nerdctl --namespace "$NERDCTL_NAMESPACE" --snapshotter "$snapshotter" \ + compose -f "$COMPOSE_FILE" "${compose_args[@]}" + ;; + esac +} + +compose_stop() { + [ -f "$COMPOSE_FILE" ] || return 0 + case "$runner" in + docker-compose) + docker compose -f "$COMPOSE_FILE" stop + ;; + nerdctl-compose) + nerdctl --namespace "$NERDCTL_NAMESPACE" --snapshotter "$snapshotter" \ + compose -f "$COMPOSE_FILE" stop + ;; + esac +} + +validate_runner + +case "$ACTION" in +start) + if [ "$(jq 'has("pre_launch_script")' "$APP_COMPOSE_FILE")" = true ]; then + echo "Running pre-launch script" + dstack-util notify-host -e "boot.progress" -d "pre-launch" || true + # shellcheck disable=SC1090 + source <(jq -r '.pre_launch_script' "$APP_COMPOSE_FILE") fi - echo "Pruning unused images" - docker image prune -af - echo "Pruning unused volumes" - docker volume prune -f + + case "$runner" in + docker-compose|nerdctl-compose) + echo "Starting containers with runner=$runner snapshotter=$snapshotter" + dstack-util notify-host -e "boot.progress" -d "starting containers" || true + if ! compose_start; then + dstack-util notify-host -e "boot.error" -d "failed to start containers" + exit 1 + fi + if [ "$runner" = docker-compose ]; then + echo "Pruning unused Docker images and volumes" + docker image prune -af + docker volume prune -f + else + echo "Pruning unused containerd images" + nerdctl --namespace "$NERDCTL_NAMESPACE" --snapshotter "$snapshotter" image prune -af + fi + ;; + bash) + echo "Running main script" + dstack-util notify-host -e "boot.progress" -d "running main script" || true + jq -r '.bash_script' "$APP_COMPOSE_FILE" | bash + ;; + esac + dstack-util notify-host -e "boot.progress" -d "done" || true ;; -"bash") - echo "Running main script" - dstack-util notify-host -e "boot.progress" -d "running main script" || true - jq -r '.bash_script' app-compose.json | bash +stop) + compose_stop ;; *) - echo "ERROR: unsupported runner: $RUNNER" >&2 - dstack-util notify-host -e "boot.error" -d "unsupported runner: $RUNNER" - exit 1 + echo "Usage: $0 [start|stop]" >&2 + exit 2 ;; esac - -dstack-util notify-host -e "boot.progress" -d "done" || true diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 249bbf9cc..b1ba5c65b 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -285,15 +285,37 @@ dstack-util setup --work-dir $WORK_DIR --device "$DATA_DEVICE" --mount-point $DA log "Mounting container runtime dirs to persistent storage" mkdir -p $DATA_MNT/var/lib/docker mkdir -p $DATA_MNT/var/lib/containerd +mkdir -p $DATA_MNT/var/lib/containerd-stargz-grpc mkdir -p $DATA_MNT/var/lib/sysbox mkdir -p /var/lib/docker mkdir -p /var/lib/containerd +mkdir -p /var/lib/containerd-stargz-grpc mkdir -p /var/lib/sysbox mount --rbind $DATA_MNT/var/lib/docker /var/lib/docker mount --rbind $DATA_MNT/var/lib/containerd /var/lib/containerd +mount --rbind $DATA_MNT/var/lib/containerd-stargz-grpc /var/lib/containerd-stargz-grpc mount --rbind $DATA_MNT/var/lib/sysbox /var/lib/sysbox mount --rbind $WORK_DIR /dstack +# Register the optional remote snapshotter. docker-compose continues to use +# Docker's own overlayfs image store; nerdctl-compose selects this plugin only +# when app-compose.json requests snapshotter=stargz. +CONTAINERD_CONFIG=/etc/containerd/config.toml +mkdir -p /etc/containerd +if [ ! -s "$CONTAINERD_CONFIG" ]; then + containerd config default >"$CONTAINERD_CONFIG" +fi +if ! grep -qE '^[[:space:]]*\[proxy_plugins\.stargz\]' "$CONTAINERD_CONFIG"; then + cat >>"$CONTAINERD_CONFIG" <<'EOF' + +[proxy_plugins.stargz] + type = "snapshot" + address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + [proxy_plugins.stargz.exports] + root = "/var/lib/containerd-stargz-grpc/" +EOF +fi + echo "======== Disk usage ========" df -h echo "============================" @@ -309,7 +331,7 @@ fi RUNNER=$(jq -r '.runner' app-compose.json) case "$RUNNER" in -docker-compose) +docker-compose|nerdctl-compose) if [[ ! -f docker-compose.yaml ]]; then jq -r '.docker_compose_file' app-compose.json >docker-compose.yaml fi diff --git a/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/files/containerd-stargz-grpc.service b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/files/containerd-stargz-grpc.service new file mode 100644 index 000000000..459cf668e --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/files/containerd-stargz-grpc.service @@ -0,0 +1,16 @@ +[Unit] +Description=containerd eStargz snapshotter +Requires=dstack-prepare.service +After=network-online.target dstack-prepare.service +Before=containerd.service + +[Service] +Type=simple +ExecCondition=/usr/bin/jq -e '.runner == "nerdctl-compose" and (.snapshotter // "overlayfs") == "stargz"' /dstack/app-compose.json +ExecStart=/usr/bin/containerd-stargz-grpc --address /run/containerd-stargz-grpc/containerd-stargz-grpc.sock --root /var/lib/containerd-stargz-grpc +ExecStartPost=/bin/sh -c 'for i in $(seq 1 100); do test -S /run/containerd-stargz-grpc/containerd-stargz-grpc.sock && exit 0; sleep 0.1; done; exit 1' +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb new file mode 100644 index 000000000..32c459052 --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb @@ -0,0 +1,29 @@ +SUMMARY = "Lazy-pulling eStargz snapshotter for containerd" +HOMEPAGE = "https://github.com/containerd/stargz-snapshotter" +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10" + +SRC_URI = "https://github.com/containerd/stargz-snapshotter/releases/download/v${PV}/stargz-snapshotter-v${PV}-linux-amd64.tar.gz;subdir=${BP} \ + file://containerd-stargz-grpc.service \ +" +SRC_URI[sha256sum] = "515a3c3af0012f192ace31fb79e910597977c77227e976680aeaaef6e9ae50a9" + +S = "${UNPACKDIR}/${BP}" + +inherit systemd + +COMPATIBLE_HOST = "x86_64.*-linux" +INHIBIT_PACKAGE_DEBUG_SPLIT = "1" +INSANE_SKIP:${PN} += "already-stripped" + +SYSTEMD_SERVICE:${PN} = "containerd-stargz-grpc.service" +SYSTEMD_AUTO_ENABLE:${PN} = "enable" + +RDEPENDS:${PN} += "fuse3-utils" + +do_install() { + install -d ${D}${bindir} ${D}${systemd_system_unitdir} + install -m 0755 ${S}/containerd-stargz-grpc ${D}${bindir}/containerd-stargz-grpc + install -m 0755 ${S}/ctr-remote ${D}${bindir}/ctr-remote + install -m 0644 ${UNPACKDIR}/containerd-stargz-grpc.service ${D}${systemd_system_unitdir}/containerd-stargz-grpc.service +} diff --git a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index abe684ac6..4072aabc9 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -13,6 +13,9 @@ IMAGE_INSTALL = "\ nftables \ docker-moby \ docker-compose \ + nerdctl \ + containerd-cni \ + stargz-snapshotter \ dstack-guest \ wireguard-tools \ cryptsetup \ diff --git a/sdk/go/dstack/compose_hash.go b/sdk/go/dstack/compose_hash.go index d40e6c3db..d513967e3 100644 --- a/sdk/go/dstack/compose_hash.go +++ b/sdk/go/dstack/compose_hash.go @@ -51,6 +51,7 @@ type AppCompose struct { Name string `json:"name,omitempty"` Features []string `json:"features,omitempty"` // Deprecated Runner string `json:"runner"` + Snapshotter string `json:"snapshotter,omitempty"` DockerComposeFile string `json:"docker_compose_file,omitempty"` DockerConfig *DockerConfig `json:"docker_config,omitempty"` PublicLogs *bool `json:"public_logs,omitempty"` @@ -74,7 +75,7 @@ type AppCompose struct { func preprocessAppCompose(appCompose AppCompose) AppCompose { if appCompose.Runner == "bash" { appCompose.DockerComposeFile = "" - } else if appCompose.Runner == "docker-compose" { + } else if appCompose.Runner == "docker-compose" || appCompose.Runner == "nerdctl-compose" { appCompose.BashScript = "" } diff --git a/sdk/js/src/get-compose-hash.ts b/sdk/js/src/get-compose-hash.ts index a682c65f4..f8cf0e8f6 100644 --- a/sdk/js/src/get-compose-hash.ts +++ b/sdk/js/src/get-compose-hash.ts @@ -61,6 +61,7 @@ export interface AppCompose extends SortableObject { // Deprecated features?: string[]; runner: string; + snapshotter?: "overlayfs" | "stargz"; docker_compose_file?: string; docker_config?: DockerConfig; public_logs?: boolean; @@ -86,7 +87,7 @@ function preprocessAppCompose(dic: AppCompose): AppCompose { const obj: AppCompose = { ...dic }; if (obj.runner === "bash" && "docker_compose_file" in obj) { delete obj.docker_compose_file; - } else if (obj.runner === "docker-compose" && "bash_script" in obj) { + } else if ((obj.runner === "docker-compose" || obj.runner === "nerdctl-compose") && "bash_script" in obj) { delete obj.bash_script; } if ("pre_launch_script" in obj && !obj.pre_launch_script) { diff --git a/sdk/python/src/dstack_sdk/get_compose_hash.py b/sdk/python/src/dstack_sdk/get_compose_hash.py index 13f3ddfca..9ee2f9b48 100644 --- a/sdk/python/src/dstack_sdk/get_compose_hash.py +++ b/sdk/python/src/dstack_sdk/get_compose_hash.py @@ -102,10 +102,12 @@ def __init__( requirements: Optional[Union[Requirements, Dict[str, Any]]] = None, bash_script: Optional[str] = None, # Legacy pre_launch_script: Optional[str] = None, # Legacy + snapshotter: Optional[str] = None, **kwargs: Any, ) -> None: """Initialize a new ``AppCompose`` instance with arbitrary extra fields.""" self.runner = runner + self.snapshotter = snapshotter self.manifest_version = manifest_version self.name = name self.features = features @@ -216,7 +218,7 @@ def preprocess_app_compose(app_compose: AppCompose) -> AppCompose: if data.get("runner") == "bash" and "docker_compose_file" in data: del data["docker_compose_file"] - elif data.get("runner") == "docker-compose" and "bash_script" in data: + elif data.get("runner") in ("docker-compose", "nerdctl-compose") and "bash_script" in data: del data["bash_script"] if "pre_launch_script" in data and not data["pre_launch_script"]: From 6f16c96f3a623aeb8313b26ab030c4c08f9d51b3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 05:50:59 -0700 Subject: [PATCH 2/4] fix: package containerd config and simplify lazy pulls --- os/common/rootfs/app-compose.sh | 24 +--------- os/common/rootfs/dstack-prepare.sh | 19 -------- os/common/rootfs/nerdctl-compose-pull.sh | 48 +++++++++++++++++++ .../containerd-config_1.1.0.bb | 12 +++++ .../containerd-config/files/config.toml | 19 ++++++++ .../stargz-snapshotter_0.18.2.bb | 2 +- .../recipes-core/dstack-guest/dstack-guest.bb | 1 + sdk/python/src/dstack_sdk/get_compose_hash.py | 5 +- 8 files changed, 86 insertions(+), 44 deletions(-) create mode 100755 os/common/rootfs/nerdctl-compose-pull.sh create mode 100644 os/yocto/layers/meta-dstack/recipes-containers/containerd-config/containerd-config_1.1.0.bb create mode 100644 os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml diff --git a/os/common/rootfs/app-compose.sh b/os/common/rootfs/app-compose.sh index 964c3199e..a7d327c05 100755 --- a/os/common/rootfs/app-compose.sh +++ b/os/common/rootfs/app-compose.sh @@ -63,29 +63,7 @@ compose_start() { return 1 fi if [ "$snapshotter" = stargz ]; then - # containerd's transfer-service pull path does not add the remote - # snapshot annotations required by stargz. Register every Compose - # image with ctr-remote before nerdctl creates the containers. - while IFS= read -r image; do - [ -n "$image" ] || continue - auth_args=() - registry=${image%%/*} - case "$registry" in - *.*|*:*|localhost) ;; - *) registry="docker.io" ;; - esac - if [ -r /root/.docker/config.json ]; then - auth=$(jq -r --arg registry "$registry" ' - .auths[$registry].auth // - (if $registry == "docker.io" then .auths["https://index.docker.io/v1/"].auth else empty end) // empty - ' /root/.docker/config.json) - if [ -n "$auth" ]; then - auth_args=(--user "$(printf '%s' "$auth" | base64 -d)") - fi - fi - ctr-remote --namespace "$NERDCTL_NAMESPACE" images rpull \ - --snapshotter "$snapshotter" "${auth_args[@]}" "$image" - done < <(docker compose -f "$COMPOSE_FILE" config --images) + nerdctl-compose-pull.sh "$COMPOSE_FILE" "$NERDCTL_NAMESPACE" fi compose_args=(up --remove-orphans -d) if [ "$snapshotter" = stargz ]; then diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index b1ba5c65b..04a65d1d5 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -297,25 +297,6 @@ mount --rbind $DATA_MNT/var/lib/containerd-stargz-grpc /var/lib/containerd-starg mount --rbind $DATA_MNT/var/lib/sysbox /var/lib/sysbox mount --rbind $WORK_DIR /dstack -# Register the optional remote snapshotter. docker-compose continues to use -# Docker's own overlayfs image store; nerdctl-compose selects this plugin only -# when app-compose.json requests snapshotter=stargz. -CONTAINERD_CONFIG=/etc/containerd/config.toml -mkdir -p /etc/containerd -if [ ! -s "$CONTAINERD_CONFIG" ]; then - containerd config default >"$CONTAINERD_CONFIG" -fi -if ! grep -qE '^[[:space:]]*\[proxy_plugins\.stargz\]' "$CONTAINERD_CONFIG"; then - cat >>"$CONTAINERD_CONFIG" <<'EOF' - -[proxy_plugins.stargz] - type = "snapshot" - address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" - [proxy_plugins.stargz.exports] - root = "/var/lib/containerd-stargz-grpc/" -EOF -fi - echo "======== Disk usage ========" df -h echo "============================" diff --git a/os/common/rootfs/nerdctl-compose-pull.sh b/os/common/rootfs/nerdctl-compose-pull.sh new file mode 100755 index 000000000..cf71a6dcf --- /dev/null +++ b/os/common/rootfs/nerdctl-compose-pull.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +# Register Compose images for lazy pulling. ctr-remote adds the remote snapshot +# annotations that containerd's generic transfer service does not provide. +set -euo pipefail + +compose_file=${1:?usage: nerdctl-compose-pull.sh COMPOSE_FILE [NAMESPACE]} +namespace=${2:-dstack} +docker_config=${DOCKER_CONFIG:-/root/.docker}/config.json + +registry_for_image() { + local first_component=${1%%/*} + case "$first_component" in + *.*|*:*|localhost) printf '%s\n' "$first_component" ;; + *) printf '%s\n' docker.io ;; + esac +} + +auth_for_registry() { + local registry=$1 + [ -r "$docker_config" ] || return 0 + jq -r --arg registry "$registry" ' + .auths[$registry].auth // + (if $registry == "docker.io" then + .auths["https://index.docker.io/v1/"].auth + else empty end) // empty + ' "$docker_config" +} + +pull_image() { + local image=$1 registry auth + local -a auth_args=() + registry=$(registry_for_image "$image") + auth=$(auth_for_registry "$registry") + if [ -n "$auth" ]; then + auth_args=(--user "$(printf '%s' "$auth" | base64 -d)") + fi + ctr-remote --namespace "$namespace" images rpull \ + --snapshotter stargz "${auth_args[@]}" "$image" +} + +while IFS= read -r image; do + [ -n "$image" ] && pull_image "$image" +done < <(docker compose -f "$compose_file" config --images) diff --git a/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/containerd-config_1.1.0.bb b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/containerd-config_1.1.0.bb new file mode 100644 index 000000000..3bf2dab13 --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/containerd-config_1.1.0.bb @@ -0,0 +1,12 @@ +SUMMARY = "dstack containerd configuration" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = "file://config.toml" + +do_install() { + install -d ${D}${sysconfdir}/containerd + install -m 0644 ${UNPACKDIR}/config.toml ${D}${sysconfdir}/containerd/config.toml +} + +FILES:${PN} = "${sysconfdir}/containerd/config.toml" diff --git a/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml new file mode 100644 index 000000000..2a49e0316 --- /dev/null +++ b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml @@ -0,0 +1,19 @@ +version = 2 + +[proxy_plugins.stargz] + type = "snapshot" + address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" + [proxy_plugins.stargz.exports] + root = "/var/lib/containerd-stargz-grpc/" + +[plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia] + privileged_without_host_devices = false + runtime_engine = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" diff --git a/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb index 32c459052..20f16b875 100644 --- a/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb +++ b/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/stargz-snapshotter_0.18.2.bb @@ -19,7 +19,7 @@ INSANE_SKIP:${PN} += "already-stripped" SYSTEMD_SERVICE:${PN} = "containerd-stargz-grpc.service" SYSTEMD_AUTO_ENABLE:${PN} = "enable" -RDEPENDS:${PN} += "fuse3-utils" +RDEPENDS:${PN} += "containerd-config fuse3-utils" do_install() { install -d ${D}${bindir} ${D}${systemd_system_unitdir} diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index daa00f9ed..daa2ee136 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -70,6 +70,7 @@ do_install() { install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/app-compose.sh ${D}${bindir} + install -m 0755 ${DSTACK_ROOTFS_FILES}/nerdctl-compose-pull.sh ${D}${bindir} install -m 0644 ${DSTACK_ROOTFS_FILES}/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ diff --git a/sdk/python/src/dstack_sdk/get_compose_hash.py b/sdk/python/src/dstack_sdk/get_compose_hash.py index 9ee2f9b48..9f8aeb9e7 100644 --- a/sdk/python/src/dstack_sdk/get_compose_hash.py +++ b/sdk/python/src/dstack_sdk/get_compose_hash.py @@ -218,7 +218,10 @@ def preprocess_app_compose(app_compose: AppCompose) -> AppCompose: if data.get("runner") == "bash" and "docker_compose_file" in data: del data["docker_compose_file"] - elif data.get("runner") in ("docker-compose", "nerdctl-compose") and "bash_script" in data: + elif ( + data.get("runner") in ("docker-compose", "nerdctl-compose") + and "bash_script" in data + ): del data["bash_script"] if "pre_launch_script" in data and not data["pre_launch_script"]: From a19a4a3e1cacab342229fd95148d12f87698699a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 11:06:43 -0700 Subject: [PATCH 3/4] fix: enable native stargz transfer pulls --- os/common/rootfs/app-compose.sh | 9 +--- os/common/rootfs/nerdctl-compose-pull.sh | 48 ------------------- .../containerd-config/files/config.toml | 13 +++++ .../recipes-core/dstack-guest/dstack-guest.bb | 1 - 4 files changed, 14 insertions(+), 57 deletions(-) delete mode 100755 os/common/rootfs/nerdctl-compose-pull.sh diff --git a/os/common/rootfs/app-compose.sh b/os/common/rootfs/app-compose.sh index a7d327c05..a712bbd5e 100755 --- a/os/common/rootfs/app-compose.sh +++ b/os/common/rootfs/app-compose.sh @@ -62,15 +62,8 @@ compose_start() { echo "ERROR: nerdctl-compose requires pre-built images; Compose build sections are not supported" >&2 return 1 fi - if [ "$snapshotter" = stargz ]; then - nerdctl-compose-pull.sh "$COMPOSE_FILE" "$NERDCTL_NAMESPACE" - fi - compose_args=(up --remove-orphans -d) - if [ "$snapshotter" = stargz ]; then - compose_args+=(--pull never) - fi nerdctl --namespace "$NERDCTL_NAMESPACE" --snapshotter "$snapshotter" \ - compose -f "$COMPOSE_FILE" "${compose_args[@]}" + compose -f "$COMPOSE_FILE" up --remove-orphans -d ;; esac } diff --git a/os/common/rootfs/nerdctl-compose-pull.sh b/os/common/rootfs/nerdctl-compose-pull.sh deleted file mode 100755 index cf71a6dcf..000000000 --- a/os/common/rootfs/nerdctl-compose-pull.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: © 2026 Phala Network -# -# SPDX-License-Identifier: Apache-2.0 - -# Register Compose images for lazy pulling. ctr-remote adds the remote snapshot -# annotations that containerd's generic transfer service does not provide. -set -euo pipefail - -compose_file=${1:?usage: nerdctl-compose-pull.sh COMPOSE_FILE [NAMESPACE]} -namespace=${2:-dstack} -docker_config=${DOCKER_CONFIG:-/root/.docker}/config.json - -registry_for_image() { - local first_component=${1%%/*} - case "$first_component" in - *.*|*:*|localhost) printf '%s\n' "$first_component" ;; - *) printf '%s\n' docker.io ;; - esac -} - -auth_for_registry() { - local registry=$1 - [ -r "$docker_config" ] || return 0 - jq -r --arg registry "$registry" ' - .auths[$registry].auth // - (if $registry == "docker.io" then - .auths["https://index.docker.io/v1/"].auth - else empty end) // empty - ' "$docker_config" -} - -pull_image() { - local image=$1 registry auth - local -a auth_args=() - registry=$(registry_for_image "$image") - auth=$(auth_for_registry "$registry") - if [ -n "$auth" ]; then - auth_args=(--user "$(printf '%s' "$auth" | base64 -d)") - fi - ctr-remote --namespace "$namespace" images rpull \ - --snapshotter stargz "${auth_args[@]}" "$image" -} - -while IFS= read -r image; do - [ -n "$image" ] && pull_image "$image" -done < <(docker compose -f "$compose_file" config --images) diff --git a/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml index 2a49e0316..a769e7b17 100644 --- a/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml +++ b/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml @@ -5,6 +5,19 @@ version = 2 address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock" [proxy_plugins.stargz.exports] root = "/var/lib/containerd-stargz-grpc/" + enable_remote_snapshot_annotations = "true" + +# containerd's transfer service only registers its default snapshotter unless +# unpack_config is specified. Keep overlayfs available and add stargz so +# `nerdctl compose up` can pull directly into either snapshotter. The remote +# snapshot annotations exported above are what make stargz pulls lazy. +[plugins."io.containerd.transfer.v1.local"] + [[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux/amd64" + snapshotter = "overlayfs" + [[plugins."io.containerd.transfer.v1.local".unpack_config]] + platform = "linux/amd64" + snapshotter = "stargz" [plugins] [plugins."io.containerd.grpc.v1.cri"] diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb index daa2ee136..daa00f9ed 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-guest/dstack-guest.bb @@ -70,7 +70,6 @@ do_install() { install -m 0755 ${DSTACK_ROOTFS_FILES}/ephemeral-docker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/wg-checker.sh ${D}${bindir} install -m 0755 ${DSTACK_ROOTFS_FILES}/app-compose.sh ${D}${bindir} - install -m 0755 ${DSTACK_ROOTFS_FILES}/nerdctl-compose-pull.sh ${D}${bindir} install -m 0644 ${DSTACK_ROOTFS_FILES}/journald.conf ${D}${sysconfdir}/systemd/journald.conf.d/dstack.conf install -d ${D}${sysconfdir}/ From dceca703cef74e2f1e518195defe294d37eeb2e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 21 Jul 2026 11:18:07 -0700 Subject: [PATCH 4/4] fix: persist nerdctl compose state --- os/common/rootfs/dstack-prepare.sh | 3 +++ .../meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 + 2 files changed, 4 insertions(+) diff --git a/os/common/rootfs/dstack-prepare.sh b/os/common/rootfs/dstack-prepare.sh index 04a65d1d5..18c634e2d 100755 --- a/os/common/rootfs/dstack-prepare.sh +++ b/os/common/rootfs/dstack-prepare.sh @@ -286,14 +286,17 @@ log "Mounting container runtime dirs to persistent storage" mkdir -p $DATA_MNT/var/lib/docker mkdir -p $DATA_MNT/var/lib/containerd mkdir -p $DATA_MNT/var/lib/containerd-stargz-grpc +mkdir -p $DATA_MNT/var/lib/nerdctl mkdir -p $DATA_MNT/var/lib/sysbox mkdir -p /var/lib/docker mkdir -p /var/lib/containerd mkdir -p /var/lib/containerd-stargz-grpc +mkdir -p /var/lib/nerdctl mkdir -p /var/lib/sysbox mount --rbind $DATA_MNT/var/lib/docker /var/lib/docker mount --rbind $DATA_MNT/var/lib/containerd /var/lib/containerd mount --rbind $DATA_MNT/var/lib/containerd-stargz-grpc /var/lib/containerd-stargz-grpc +mount --rbind $DATA_MNT/var/lib/nerdctl /var/lib/nerdctl mount --rbind $DATA_MNT/var/lib/sysbox /var/lib/sysbox mount --rbind $WORK_DIR /dstack diff --git a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 4072aabc9..388dc0849 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -191,5 +191,6 @@ mkdirs() { mkdir -p ${IMAGE_ROOTFS}/etc/wireguard mkdir -p ${IMAGE_ROOTFS}/var/lib/docker mkdir -p ${IMAGE_ROOTFS}/var/lib/containerd + mkdir -p ${IMAGE_ROOTFS}/var/lib/nerdctl ln -sf dstack ${IMAGE_ROOTFS}/tapp }