diff --git a/cmd/ephemerd/main.go b/cmd/ephemerd/main.go index 6997429..b3f491c 100644 --- a/cmd/ephemerd/main.go +++ b/cmd/ephemerd/main.go @@ -360,6 +360,7 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP DindEnabled: cfg.Dind.Enabled, DindAllowPrivileged: cfg.Dind.ResolvedAllowPrivileged(), Rlimits: cfg.Runtime.Rlimits.Resolved(), + PidsLimit: cfg.Runtime.ResolvedPidsLimit(), AllowNewPrivileges: cfg.Runtime.ResolvedAllowNewPrivileges(), LinuxRuntime: cfg.Runner.Linux.ContainerdRuntime(), Network: net, @@ -675,6 +676,7 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP CacheProxyEnv: cacheProxyEnvVars, CacheProxyMounts: cacheProxyMounts, Rlimits: cfg.Runtime.Rlimits.Resolved(), + PidsLimit: cfg.Runtime.ResolvedPidsLimit(), AllowNewPrivileges: cfg.Runtime.ResolvedAllowNewPrivileges(), LinuxRuntime: cfg.Runner.Linux.ContainerdRuntime(), Network: net, diff --git a/pkg/config/config.go b/pkg/config/config.go index 75f5c27..e0ae2cb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -344,6 +344,21 @@ type ContainerdConfig struct { type RuntimeConfig struct { Rlimits RuntimeRlimits `toml:"rlimits"` + // PidsLimit is the cgroup pids.max for each runner container and each + // sibling container its job starts through dind: the most processes + // plus threads that container may hold at once. 0 (unset) means + // DefaultPidsLimit; -1 means no limit. + // + // This, not RLIMIT_NPROC, is the per-job process bound. NPROC counts + // every task owned by the UID across the whole host, and every runner + // container runs as the same UID, so concurrent jobs drew from one + // shared budget: on ducks (2026-10-22) four parallel Rust builds hit + // the old 1024 default and rustc panicked with "failed to spawn work + // thread: Resource temporarily unavailable". A cgroup limit is + // counted per container, so one job's fork bomb cannot starve the + // others and a busy neighbour cannot fail a build. + PidsLimit int64 `toml:"pids_limit"` + // AllowNewPrivileges controls whether a process in the runner // container may gain privileges through execve — i.e. whether setuid // binaries and file capabilities take effect. When true the OCI spec @@ -465,9 +480,34 @@ func (r RuntimeConfig) ResolvedAllowNewPrivileges() bool { return true } +// DefaultPidsLimit is the per-container pids.max when [runtime] +// pids_limit is unset. A -j30 Rust build peaks in the low thousands of +// threads (cargo jobs × rustc codegen threads × LLVM workers), so this +// leaves an order of magnitude of headroom while still stopping a fork +// bomb long before the host's kernel.threads-max. +const DefaultPidsLimit = 32768 + +// ResolvedPidsLimit returns the pids.max to apply, or 0 for "no limit". +func (r RuntimeConfig) ResolvedPidsLimit() int64 { + switch { + case r.PidsLimit < 0: + return 0 + case r.PidsLimit == 0: + return DefaultPidsLimit + default: + return r.PidsLimit + } +} + +// defaultNproc is RLIMIT_NPROC when [runtime.rlimits] nproc is unset. It is +// deliberately effectively unlimited: NPROC is enforced per UID across the +// host, not per container, so as a per-job limit it only ever made +// concurrent jobs fail each other. PidsLimit bounds each job instead. +const defaultNproc = 1 << 20 + // RuntimeRlimits sets POSIX resource limits (RLIMIT_*) on each runner -// container's OCI spec. Defaults match containerd's built-in OCI spec -// (nofile=1024, nproc=1024) so an empty config is a no-behavior-change. +// container's OCI spec. nofile defaults to containerd's built-in 1024; +// nproc defaults to defaultNproc (see there for why it is not 1024). // // Set higher when CI workloads need more file descriptors or processes // than containerd's defaults allow. Common case: a build tool calling @@ -482,7 +522,8 @@ type RuntimeRlimits struct { // and hard get set to this value. Default 1024 (containerd default). Nofile int64 `toml:"nofile"` // Nproc is RLIMIT_NPROC (max processes/threads for the container's - // user). Both soft and hard get set to this value. Default 1024. + // user, counted host-wide). Both soft and hard get set to this value. + // Default defaultNproc. Nproc int64 `toml:"nproc"` } @@ -494,7 +535,7 @@ func (r RuntimeRlimits) Resolved() RuntimeRlimits { r.Nofile = 1024 } if r.Nproc <= 0 { - r.Nproc = 1024 + r.Nproc = defaultNproc } return r } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d1ff2cc..bda3d60 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -2096,8 +2096,8 @@ func TestRuntimeRlimitsResolved_Defaults(t *testing.T) { if got.Nofile != 1024 { t.Errorf("Nofile = %d, want 1024 (containerd default)", got.Nofile) } - if got.Nproc != 1024 { - t.Errorf("Nproc = %d, want 1024 default", got.Nproc) + if got.Nproc != defaultNproc { + t.Errorf("Nproc = %d, want %d default (NPROC is host-wide per UID; pids.max bounds jobs)", got.Nproc, defaultNproc) } } @@ -2116,8 +2116,8 @@ func TestRuntimeRlimitsResolved_NegativeFallsBack(t *testing.T) { if got.Nofile != 1024 { t.Errorf("Nofile(-1) resolved to %d, want 1024", got.Nofile) } - if got.Nproc != 1024 { - t.Errorf("Nproc(-100) resolved to %d, want 1024", got.Nproc) + if got.Nproc != defaultNproc { + t.Errorf("Nproc(-100) resolved to %d, want %d", got.Nproc, defaultNproc) } } @@ -2128,8 +2128,8 @@ func TestRuntimeRlimitsResolved_MixedZeroAndExplicit(t *testing.T) { if got.Nofile != 65536 { t.Errorf("Nofile = %d, want 65536 (preserved)", got.Nofile) } - if got.Nproc != 1024 { - t.Errorf("Nproc = %d, want 1024 (default fill)", got.Nproc) + if got.Nproc != defaultNproc { + t.Errorf("Nproc = %d, want %d (default fill)", got.Nproc, defaultNproc) } } @@ -2180,8 +2180,8 @@ owner = "testorg" t.Errorf("Rlimits.Nofile (raw) = %d, want 0 before Resolved()", cfg.Runtime.Rlimits.Nofile) } resolved := cfg.Runtime.Rlimits.Resolved() - if resolved.Nofile != 1024 || resolved.Nproc != 1024 { - t.Errorf("Resolved() = %+v, want {1024, 1024}", resolved) + if resolved.Nofile != 1024 || resolved.Nproc != defaultNproc { + t.Errorf("Resolved() = %+v, want {1024, %d}", resolved, defaultNproc) } } @@ -2500,3 +2500,17 @@ func TestResolvedDestroyTimeout(t *testing.T) { } } } + +func TestRuntimeConfig_ResolvedPidsLimit(t *testing.T) { + for _, tc := range []struct { + in, want int64 + }{ + {0, DefaultPidsLimit}, // unset: the safe per-job default + {4096, 4096}, + {-1, 0}, // explicit opt-out: no limit + } { + if got := (RuntimeConfig{PidsLimit: tc.in}).ResolvedPidsLimit(); got != tc.want { + t.Errorf("ResolvedPidsLimit(%d) = %d, want %d", tc.in, got, tc.want) + } + } +} diff --git a/pkg/dind/containers.go b/pkg/dind/containers.go index 94ad577..9d5bdc5 100644 --- a/pkg/dind/containers.go +++ b/pkg/dind/containers.go @@ -522,6 +522,10 @@ func (s *Server) handleContainerCreate(w http.ResponseWriter, r *http.Request) { if !privileged { opts = append(opts, nonPrivilegedHardeningOpts(s.log)...) } + // Privileged or not, a sibling counts its own processes: it shares the + // host UID with every other job's containers, so RLIMIT_NPROC cannot + // bound it per job. + opts = append(opts, pidsLimitOpts(s.pidsLimit)...) if req.HostConfig != nil { // Privileged mode: all capabilities, all devices, disable seccomp/apparmor, diff --git a/pkg/dind/dind.go b/pkg/dind/dind.go index 50b7017..1aef211 100644 --- a/pkg/dind/dind.go +++ b/pkg/dind/dind.go @@ -70,6 +70,7 @@ type Server struct { buildkit *buildkit.Server // shared embedded BuildKit solver (nil → fall back to platform default) runnerNetNS string // path to runner container's net namespace; used to install DNAT rules for port bindings allowPrivileged bool // gate for docker run --privileged / --cap-add; see config.DindConfig.AllowPrivileged + pidsLimit int64 // cgroup pids.max for siblings; 0 = no limit // stager publishes every job-supplied bind source at a path under // /dind-binds// that only root can reach, so the path @@ -184,6 +185,11 @@ type Config struct { // for the threat model. AllowPrivileged bool + // PidsLimit is the cgroup pids.max applied to every sibling container, + // the same bound the runner container gets; 0 means no limit. See + // config.RuntimeConfig.PidsLimit. + PidsLimit int64 + // RegistryMirror routes this job's image pulls through a LAN // pull-through cache. Nil disables mirroring, leaving the pulls // identical to what they were before the feature existed. @@ -238,7 +244,8 @@ func New(cfg Config) (*Server, error) { buildkit: cfg.BuildKit, runnerNetNS: cfg.RunnerNetNS, allowPrivileged: cfg.AllowPrivileged, - stager: newBindStager(cfg.DataDir, cfg.JobID, cfg.Log), + pidsLimit: cfg.PidsLimit, + stager: newBindStager(cfg.DataDir, cfg.JobID, cfg.Log), mirror: cfg.RegistryMirror, log: cfg.Log.With("component", "dind", "job_id", cfg.JobID), images: make(map[string]*imageEntry), diff --git a/pkg/dind/pidslimit_linux.go b/pkg/dind/pidslimit_linux.go new file mode 100644 index 0000000..e839ca0 --- /dev/null +++ b/pkg/dind/pidslimit_linux.go @@ -0,0 +1,33 @@ +//go:build linux + +package dind + +import ( + "context" + + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/pkg/oci" + ocispec "github.com/opencontainers/runtime-spec/specs-go" +) + +// pidsLimitOpts sets a sibling's cgroup pids.max. limit <= 0 adds nothing. +// It mirrors runtime's runner-container option; runtime imports dind, so +// the two cannot share one helper. See config.RuntimeConfig.PidsLimit. +func pidsLimitOpts(limit int64) []oci.SpecOpts { + if limit <= 0 { + return nil + } + return []oci.SpecOpts{ + func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { + if s.Linux == nil { + s.Linux = &ocispec.Linux{} + } + if s.Linux.Resources == nil { + s.Linux.Resources = &ocispec.LinuxResources{} + } + l := limit + s.Linux.Resources.Pids = &ocispec.LinuxPids{Limit: &l} + return nil + }, + } +} diff --git a/pkg/dind/pidslimit_other.go b/pkg/dind/pidslimit_other.go new file mode 100644 index 0000000..51b3239 --- /dev/null +++ b/pkg/dind/pidslimit_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package dind + +import "github.com/containerd/containerd/v2/pkg/oci" + +// pidsLimitOpts is a no-op off Linux: pids.max is a Linux cgroup control, and +// dind siblings only run on Linux (see nonPrivilegedHardeningOpts). +func pidsLimitOpts(_ int64) []oci.SpecOpts { + return nil +} diff --git a/pkg/runtime/rlimits_linux.go b/pkg/runtime/rlimits_linux.go index b9fa4dc..d919fd3 100644 --- a/pkg/runtime/rlimits_linux.go +++ b/pkg/runtime/rlimits_linux.go @@ -11,6 +11,28 @@ import ( ocispec "github.com/opencontainers/runtime-spec/specs-go" ) +// pidsLimitOpts sets the container's cgroup pids.max. limit <= 0 adds +// nothing (no limit). dind applies the same limit to sibling containers +// (dind.pidsLimitOpt) — see config.RuntimeConfig.PidsLimit. +func pidsLimitOpts(limit int64) []oci.SpecOpts { + if limit <= 0 { + return nil + } + return []oci.SpecOpts{ + func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { + if s.Linux == nil { + s.Linux = &ocispec.Linux{} + } + if s.Linux.Resources == nil { + s.Linux.Resources = &ocispec.LinuxResources{} + } + l := limit + s.Linux.Resources.Pids = &ocispec.LinuxPids{Limit: &l} + return nil + }, + } +} + // rlimitsOpts sets RLIMIT_NOFILE and RLIMIT_NPROC on the container's OCI // process spec. We deliberately replace the rlimits slice (rather than // append) so the containerd default RLIMIT_NOFILE=1024 entry from diff --git a/pkg/runtime/rlimits_linux_test.go b/pkg/runtime/rlimits_linux_test.go index 9c7ce1c..3be21fc 100644 --- a/pkg/runtime/rlimits_linux_test.go +++ b/pkg/runtime/rlimits_linux_test.go @@ -37,8 +37,8 @@ func TestRlimitsOpts_AppliesConfiguredValues(t *testing.T) { } func TestRlimitsOpts_AppliesDefaultsWhenZero(t *testing.T) { - // Zero values must produce the containerd-default 1024 entries — - // emitting Rlimits with Soft=Hard=0 would cripple the container. + // Zero values must resolve to the defaults — emitting Rlimits with + // Soft=Hard=0 would cripple the container. spec := &oci.Spec{Process: &ocispec.Process{}} opts := rlimitsOpts(config.RuntimeRlimits{}) for _, opt := range opts { @@ -46,9 +46,13 @@ func TestRlimitsOpts_AppliesDefaultsWhenZero(t *testing.T) { t.Fatalf("opt: %v", err) } } + want := map[string]uint64{ + "RLIMIT_NOFILE": 1024, + "RLIMIT_NPROC": uint64(config.RuntimeRlimits{}.Resolved().Nproc), + } for _, rl := range spec.Process.Rlimits { - if rl.Soft != 1024 || rl.Hard != 1024 { - t.Errorf("%s: soft=%d hard=%d, want 1024/1024", rl.Type, rl.Soft, rl.Hard) + if rl.Soft != want[rl.Type] || rl.Hard != want[rl.Type] { + t.Errorf("%s: soft=%d hard=%d, want %d", rl.Type, rl.Soft, rl.Hard, want[rl.Type]) } } } @@ -101,3 +105,22 @@ func TestRlimitsOpts_NilProcessSpec(t *testing.T) { t.Errorf("len(Rlimits) = %d, want 2", len(spec.Process.Rlimits)) } } + +// TestPidsLimitOpts pins the per-job process bound: a positive limit lands +// in the spec's cgroup pids controller, and "no limit" adds nothing rather +// than writing pids.max=0, which would forbid every fork. +func TestPidsLimitOpts(t *testing.T) { + spec := &oci.Spec{} + for _, opt := range pidsLimitOpts(32768) { + if err := opt(context.Background(), nil, nil, spec); err != nil { + t.Fatalf("opt: %v", err) + } + } + if spec.Linux == nil || spec.Linux.Resources == nil || spec.Linux.Resources.Pids == nil || + spec.Linux.Resources.Pids.Limit == nil || *spec.Linux.Resources.Pids.Limit != 32768 { + t.Fatalf("pids limit not applied: %+v", spec.Linux) + } + if opts := pidsLimitOpts(0); len(opts) != 0 { + t.Errorf("pidsLimitOpts(0) returned %d opts, want none", len(opts)) + } +} diff --git a/pkg/runtime/rlimits_other.go b/pkg/runtime/rlimits_other.go index 0a46a6b..7e8b4de 100644 --- a/pkg/runtime/rlimits_other.go +++ b/pkg/runtime/rlimits_other.go @@ -13,3 +13,8 @@ import ( func rlimitsOpts(_ config.RuntimeRlimits) []oci.SpecOpts { return nil } + +// pidsLimitOpts is a no-op off Linux: pids.max is a Linux cgroup control. +func pidsLimitOpts(_ int64) []oci.SpecOpts { + return nil +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 65d25c4..6de9fb4 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -112,6 +112,10 @@ type Config struct { // process. Zero values fall back to the containerd default (1024). // Applies on Linux only; ignored on Windows (HCS uses a different model). Rlimits config.RuntimeRlimits + // PidsLimit is the cgroup pids.max for the runner container and every + // dind sibling of its job; 0 means no limit. A resolved value: callers + // pass cfg.Runtime.ResolvedPidsLimit(). Linux only. + PidsLimit int64 // AllowNewPrivileges permits privilege escalation via execve in the // runner container (NoNewPrivileges=false), which is what makes // `sudo` work. See config.RuntimeConfig.AllowNewPrivileges. @@ -1157,6 +1161,7 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err // apt-get install` works. See config.RuntimeConfig.AllowNewPrivileges. opts = append(opts, newPrivilegesOpts(r.cfg.AllowNewPrivileges)...) opts = append(opts, rlimitsOpts(r.cfg.Rlimits)...) + opts = append(opts, pidsLimitOpts(r.cfg.PidsLimit)...) switch { case len(cfg.Entrypoint) > 0: // Forge mode: custom entrypoint (e.g. act_runner register + daemon). @@ -1266,6 +1271,7 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err Network: r.cfg.Network, BuildKit: r.cfg.BuildKit, AllowPrivileged: r.cfg.DindAllowPrivileged, + PidsLimit: r.cfg.PidsLimit, RegistryMirror: r.cfg.RegistryMirror, Transport: resolveDindTransport(r.cfg.LinuxRuntime, goruntime.GOOS), Log: r.cfg.Log,