fix(runtime): per-job cgroup pids.max instead of a host-wide shared RLIMIT_NPROC

cmd/ephemerd/main.go

@@ -360,6 +360,7 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 			DindEnabled:         cfg.Dind.Enabled,
 			DindAllowPrivileged: cfg.Dind.ResolvedAllowPrivileged(),
 			Rlimits:             cfg.Runtime.Rlimits.Resolved(),
+			PidsLimit:           cfg.Runtime.ResolvedPidsLimit(),
 			AllowNewPrivileges:  cfg.Runtime.ResolvedAllowNewPrivileges(),
 			LinuxRuntime:        cfg.Runner.Linux.ContainerdRuntime(),
 			Network:             net,
@@ -675,6 +676,7 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 		CacheProxyEnv:       cacheProxyEnvVars,
 		CacheProxyMounts:    cacheProxyMounts,
 		Rlimits:             cfg.Runtime.Rlimits.Resolved(),
+		PidsLimit:           cfg.Runtime.ResolvedPidsLimit(),
 		AllowNewPrivileges:  cfg.Runtime.ResolvedAllowNewPrivileges(),
 		LinuxRuntime:        cfg.Runner.Linux.ContainerdRuntime(),
 		Network:             net,

pkg/config/config.go

@@ -344,6 +344,21 @@ type ContainerdConfig struct {
 type RuntimeConfig struct {
 	Rlimits RuntimeRlimits `toml:"rlimits"`
 
+	// PidsLimit is the cgroup pids.max for each runner container and each
+	// sibling container its job starts through dind: the most processes
+	// plus threads that container may hold at once. 0 (unset) means
+	// DefaultPidsLimit; -1 means no limit.
+	//
+	// This, not RLIMIT_NPROC, is the per-job process bound. NPROC counts
+	// every task owned by the UID across the whole host, and every runner
+	// container runs as the same UID, so concurrent jobs drew from one
+	// shared budget: on ducks (2026-10-22) four parallel Rust builds hit
+	// the old 1024 default and rustc panicked with "failed to spawn work
+	// thread: Resource temporarily unavailable". A cgroup limit is
+	// counted per container, so one job's fork bomb cannot starve the
+	// others and a busy neighbour cannot fail a build.
+	PidsLimit int64 `toml:"pids_limit"`
+
 	// AllowNewPrivileges controls whether a process in the runner
 	// container may gain privileges through execve — i.e. whether setuid
 	// binaries and file capabilities take effect. When true the OCI spec
@@ -465,9 +480,34 @@ func (r RuntimeConfig) ResolvedAllowNewPrivileges() bool {
 	return true
 }
 
+// DefaultPidsLimit is the per-container pids.max when [runtime]
+// pids_limit is unset. A -j30 Rust build peaks in the low thousands of
+// threads (cargo jobs × rustc codegen threads × LLVM workers), so this
+// leaves an order of magnitude of headroom while still stopping a fork
+// bomb long before the host's kernel.threads-max.
+const DefaultPidsLimit = 32768
+
+// ResolvedPidsLimit returns the pids.max to apply, or 0 for "no limit".
+func (r RuntimeConfig) ResolvedPidsLimit() int64 {
+	switch {
+	case r.PidsLimit < 0:
+		return 0
+	case r.PidsLimit == 0:
+		return DefaultPidsLimit
+	default:
+		return r.PidsLimit
+	}
+}
+
+// defaultNproc is RLIMIT_NPROC when [runtime.rlimits] nproc is unset. It is
+// deliberately effectively unlimited: NPROC is enforced per UID across the
+// host, not per container, so as a per-job limit it only ever made
+// concurrent jobs fail each other. PidsLimit bounds each job instead.
+const defaultNproc = 1 << 20
+
 // RuntimeRlimits sets POSIX resource limits (RLIMIT_*) on each runner
-// container's OCI spec. Defaults match containerd's built-in OCI spec
-// (nofile=1024, nproc=1024) so an empty config is a no-behavior-change.
+// container's OCI spec. nofile defaults to containerd's built-in 1024;
+// nproc defaults to defaultNproc (see there for why it is not 1024).
 //
 // Set higher when CI workloads need more file descriptors or processes
 // than containerd's defaults allow. Common case: a build tool calling
@@ -482,7 +522,8 @@ type RuntimeRlimits struct {
 	// and hard get set to this value. Default 1024 (containerd default).
 	Nofile int64 `toml:"nofile"`
 	// Nproc is RLIMIT_NPROC (max processes/threads for the container's
-	// user). Both soft and hard get set to this value. Default 1024.
+	// user, counted host-wide). Both soft and hard get set to this value.
+	// Default defaultNproc.
 	Nproc int64 `toml:"nproc"`
 }
 
@@ -494,7 +535,7 @@ func (r RuntimeRlimits) Resolved() RuntimeRlimits {
 		r.Nofile = 1024
 	}
 	if r.Nproc <= 0 {
-		r.Nproc = 1024
+		r.Nproc = defaultNproc
 	}
 	return r
 }

pkg/config/config_test.go

@@ -2096,8 +2096,8 @@ func TestRuntimeRlimitsResolved_Defaults(t *testing.T) {
 	if got.Nofile != 1024 {
 		t.Errorf("Nofile = %d, want 1024 (containerd default)", got.Nofile)
 	}
-	if got.Nproc != 1024 {
-		t.Errorf("Nproc = %d, want 1024 default", got.Nproc)
+	if got.Nproc != defaultNproc {
+		t.Errorf("Nproc = %d, want %d default (NPROC is host-wide per UID; pids.max bounds jobs)", got.Nproc, defaultNproc)
 	}
 }
 
@@ -2116,8 +2116,8 @@ func TestRuntimeRlimitsResolved_NegativeFallsBack(t *testing.T) {
 	if got.Nofile != 1024 {
 		t.Errorf("Nofile(-1) resolved to %d, want 1024", got.Nofile)
 	}
-	if got.Nproc != 1024 {
-		t.Errorf("Nproc(-100) resolved to %d, want 1024", got.Nproc)
+	if got.Nproc != defaultNproc {
+		t.Errorf("Nproc(-100) resolved to %d, want %d", got.Nproc, defaultNproc)
 	}
 }
 
@@ -2128,8 +2128,8 @@ func TestRuntimeRlimitsResolved_MixedZeroAndExplicit(t *testing.T) {
 	if got.Nofile != 65536 {
 		t.Errorf("Nofile = %d, want 65536 (preserved)", got.Nofile)
 	}
-	if got.Nproc != 1024 {
-		t.Errorf("Nproc = %d, want 1024 (default fill)", got.Nproc)
+	if got.Nproc != defaultNproc {
+		t.Errorf("Nproc = %d, want %d (default fill)", got.Nproc, defaultNproc)
 	}
 }
 
@@ -2180,8 +2180,8 @@ owner = "testorg"
 		t.Errorf("Rlimits.Nofile (raw) = %d, want 0 before Resolved()", cfg.Runtime.Rlimits.Nofile)
 	}
 	resolved := cfg.Runtime.Rlimits.Resolved()
-	if resolved.Nofile != 1024 || resolved.Nproc != 1024 {
-		t.Errorf("Resolved() = %+v, want {1024, 1024}", resolved)
+	if resolved.Nofile != 1024 || resolved.Nproc != defaultNproc {
+		t.Errorf("Resolved() = %+v, want {1024, %d}", resolved, defaultNproc)
 	}
 }
 
@@ -2500,3 +2500,17 @@ func TestResolvedDestroyTimeout(t *testing.T) {
 		}
 	}
 }
+
+func TestRuntimeConfig_ResolvedPidsLimit(t *testing.T) {
+	for _, tc := range []struct {
+		in, want int64
+	}{
+		{0, DefaultPidsLimit}, // unset: the safe per-job default
+		{4096, 4096},
+		{-1, 0}, // explicit opt-out: no limit
+	} {
+		if got := (RuntimeConfig{PidsLimit: tc.in}).ResolvedPidsLimit(); got != tc.want {
+			t.Errorf("ResolvedPidsLimit(%d) = %d, want %d", tc.in, got, tc.want)
+		}
+	}
+}

pkg/dind/containers.go

@@ -522,6 +522,10 @@ func (s *Server) handleContainerCreate(w http.ResponseWriter, r *http.Request) {
 	if !privileged {
 		opts = append(opts, nonPrivilegedHardeningOpts(s.log)...)
 	}
+	// Privileged or not, a sibling counts its own processes: it shares the
+	// host UID with every other job's containers, so RLIMIT_NPROC cannot
+	// bound it per job.
+	opts = append(opts, pidsLimitOpts(s.pidsLimit)...)
 
 	if req.HostConfig != nil {
 		// Privileged mode: all capabilities, all devices, disable seccomp/apparmor,

pkg/dind/dind.go

@@ -70,6 +70,7 @@ type Server struct {
 	buildkit        *buildkit.Server // shared embedded BuildKit solver (nil → fall back to platform default)
 	runnerNetNS     string           // path to runner container's net namespace; used to install DNAT rules for port bindings
 	allowPrivileged bool             // gate for docker run --privileged / --cap-add; see config.DindConfig.AllowPrivileged
+	pidsLimit       int64            // cgroup pids.max for siblings; 0 = no limit
 
 	// stager publishes every job-supplied bind source at a path under
 	// <DataDir>/dind-binds/<JobID>/ that only root can reach, so the path
@@ -184,6 +185,11 @@ type Config struct {
 	// for the threat model.
 	AllowPrivileged bool
 
+	// PidsLimit is the cgroup pids.max applied to every sibling container,
+	// the same bound the runner container gets; 0 means no limit. See
+	// config.RuntimeConfig.PidsLimit.
+	PidsLimit int64
+
 	// RegistryMirror routes this job's image pulls through a LAN
 	// pull-through cache. Nil disables mirroring, leaving the pulls
 	// identical to what they were before the feature existed.
@@ -238,7 +244,8 @@ func New(cfg Config) (*Server, error) {
 		buildkit:        cfg.BuildKit,
 		runnerNetNS:     cfg.RunnerNetNS,
 		allowPrivileged: cfg.AllowPrivileged,
-		stager:          newBindStager(cfg.DataDir, cfg.JobID, cfg.Log),
+		pidsLimit:       cfg.PidsLimit,
+		stager:         newBindStager(cfg.DataDir, cfg.JobID, cfg.Log),
 		mirror:          cfg.RegistryMirror,
 		log:             cfg.Log.With("component", "dind", "job_id", cfg.JobID),
 		images:          make(map[string]*imageEntry),

pkg/dind/pidslimit_linux.go

@@ -0,0 +1,33 @@
+//go:build linux
+
+package dind
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/v2/core/containers"
+	"github.com/containerd/containerd/v2/pkg/oci"
+	ocispec "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// pidsLimitOpts sets a sibling's cgroup pids.max. limit <= 0 adds nothing.
+// It mirrors runtime's runner-container option; runtime imports dind, so
+// the two cannot share one helper. See config.RuntimeConfig.PidsLimit.
+func pidsLimitOpts(limit int64) []oci.SpecOpts {
+	if limit <= 0 {
+		return nil
+	}
+	return []oci.SpecOpts{
+		func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+			if s.Linux == nil {
+				s.Linux = &ocispec.Linux{}
+			}
+			if s.Linux.Resources == nil {
+				s.Linux.Resources = &ocispec.LinuxResources{}
+			}
+			l := limit
+			s.Linux.Resources.Pids = &ocispec.LinuxPids{Limit: &l}
+			return nil
+		},
+	}
+}

pkg/dind/pidslimit_other.go

@@ -0,0 +1,11 @@
+//go:build !linux
+
+package dind
+
+import "github.com/containerd/containerd/v2/pkg/oci"
+
+// pidsLimitOpts is a no-op off Linux: pids.max is a Linux cgroup control, and
+// dind siblings only run on Linux (see nonPrivilegedHardeningOpts).
+func pidsLimitOpts(_ int64) []oci.SpecOpts {
+	return nil
+}

pkg/runtime/rlimits_linux.go

@@ -11,6 +11,28 @@ import (
 	ocispec "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+// pidsLimitOpts sets the container's cgroup pids.max. limit <= 0 adds
+// nothing (no limit). dind applies the same limit to sibling containers
+// (dind.pidsLimitOpt) — see config.RuntimeConfig.PidsLimit.
+func pidsLimitOpts(limit int64) []oci.SpecOpts {
+	if limit <= 0 {
+		return nil
+	}
+	return []oci.SpecOpts{
+		func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+			if s.Linux == nil {
+				s.Linux = &ocispec.Linux{}
+			}
+			if s.Linux.Resources == nil {
+				s.Linux.Resources = &ocispec.LinuxResources{}
+			}
+			l := limit
+			s.Linux.Resources.Pids = &ocispec.LinuxPids{Limit: &l}
+			return nil
+		},
+	}
+}
+
 // rlimitsOpts sets RLIMIT_NOFILE and RLIMIT_NPROC on the container's OCI
 // process spec. We deliberately replace the rlimits slice (rather than
 // append) so the containerd default RLIMIT_NOFILE=1024 entry from

pkg/runtime/rlimits_linux_test.go

@@ -37,18 +37,22 @@ func TestRlimitsOpts_AppliesConfiguredValues(t *testing.T) {
 }
 
 func TestRlimitsOpts_AppliesDefaultsWhenZero(t *testing.T) {
-	// Zero values must produce the containerd-default 1024 entries —
-	// emitting Rlimits with Soft=Hard=0 would cripple the container.
+	// Zero values must resolve to the defaults — emitting Rlimits with
+	// Soft=Hard=0 would cripple the container.
 	spec := &oci.Spec{Process: &ocispec.Process{}}
 	opts := rlimitsOpts(config.RuntimeRlimits{})
 	for _, opt := range opts {
 		if err := opt(context.Background(), nil, nil, spec); err != nil {
 			t.Fatalf("opt: %v", err)
 		}
 	}
+	want := map[string]uint64{
+		"RLIMIT_NOFILE": 1024,
+		"RLIMIT_NPROC":  uint64(config.RuntimeRlimits{}.Resolved().Nproc),
+	}
 	for _, rl := range spec.Process.Rlimits {
-		if rl.Soft != 1024 || rl.Hard != 1024 {
-			t.Errorf("%s: soft=%d hard=%d, want 1024/1024", rl.Type, rl.Soft, rl.Hard)
+		if rl.Soft != want[rl.Type] || rl.Hard != want[rl.Type] {
+			t.Errorf("%s: soft=%d hard=%d, want %d", rl.Type, rl.Soft, rl.Hard, want[rl.Type])
 		}
 	}
 }
@@ -101,3 +105,22 @@ func TestRlimitsOpts_NilProcessSpec(t *testing.T) {
 		t.Errorf("len(Rlimits) = %d, want 2", len(spec.Process.Rlimits))
 	}
 }
+
+// TestPidsLimitOpts pins the per-job process bound: a positive limit lands
+// in the spec's cgroup pids controller, and "no limit" adds nothing rather
+// than writing pids.max=0, which would forbid every fork.
+func TestPidsLimitOpts(t *testing.T) {
+	spec := &oci.Spec{}
+	for _, opt := range pidsLimitOpts(32768) {
+		if err := opt(context.Background(), nil, nil, spec); err != nil {
+			t.Fatalf("opt: %v", err)
+		}
+	}
+	if spec.Linux == nil || spec.Linux.Resources == nil || spec.Linux.Resources.Pids == nil ||
+		spec.Linux.Resources.Pids.Limit == nil || *spec.Linux.Resources.Pids.Limit != 32768 {
+		t.Fatalf("pids limit not applied: %+v", spec.Linux)
+	}
+	if opts := pidsLimitOpts(0); len(opts) != 0 {
+		t.Errorf("pidsLimitOpts(0) returned %d opts, want none", len(opts))
+	}
+}

pkg/runtime/rlimits_other.go

@@ -13,3 +13,8 @@ import (
 func rlimitsOpts(_ config.RuntimeRlimits) []oci.SpecOpts {
 	return nil
 }
+
+// pidsLimitOpts is a no-op off Linux: pids.max is a Linux cgroup control.
+func pidsLimitOpts(_ int64) []oci.SpecOpts {
+	return nil
+}

pkg/runtime/runtime.go

@@ -112,6 +112,10 @@ type Config struct {
 	// process. Zero values fall back to the containerd default (1024).
 	// Applies on Linux only; ignored on Windows (HCS uses a different model).
 	Rlimits config.RuntimeRlimits
+	// PidsLimit is the cgroup pids.max for the runner container and every
+	// dind sibling of its job; 0 means no limit. A resolved value: callers
+	// pass cfg.Runtime.ResolvedPidsLimit(). Linux only.
+	PidsLimit int64
 	// AllowNewPrivileges permits privilege escalation via execve in the
 	// runner container (NoNewPrivileges=false), which is what makes
 	// `sudo` work. See config.RuntimeConfig.AllowNewPrivileges.
@@ -1157,6 +1161,7 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err
 	// apt-get install` works. See config.RuntimeConfig.AllowNewPrivileges.
 	opts = append(opts, newPrivilegesOpts(r.cfg.AllowNewPrivileges)...)
 	opts = append(opts, rlimitsOpts(r.cfg.Rlimits)...)
+	opts = append(opts, pidsLimitOpts(r.cfg.PidsLimit)...)
 	switch {
 	case len(cfg.Entrypoint) > 0:
 		// Forge mode: custom entrypoint (e.g. act_runner register + daemon).
@@ -1266,6 +1271,7 @@ func (r *Runtime) Create(ctx context.Context, cfg CreateConfig) (*RunnerEnv, err
 			Network:         r.cfg.Network,
 			BuildKit:        r.cfg.BuildKit,
 			AllowPrivileged: r.cfg.DindAllowPrivileged,
+			PidsLimit:       r.cfg.PidsLimit,
 			RegistryMirror:  r.cfg.RegistryMirror,
 			Transport:       resolveDindTransport(r.cfg.LinuxRuntime, goruntime.GOOS),
 			Log:             r.cfg.Log,