feat(vm): bigger macOS job disks + sweep leaked macOS VM clones

cmd/ephemerd/doctor_darwin.go

@@ -4,11 +4,16 @@ package main
 
 import (
 	"fmt"
+	"io"
+	"log/slog"
 	"os"
 	"os/exec"
 	"runtime"
 	"strings"
 	"syscall"
+	"time"
+
+	"github.com/ephpm/ephemerd/pkg/vm"
 )
 
 func platformChecks(pass, warn, fail func(string)) {
@@ -125,25 +130,17 @@ func checkEmbeddedAssets(pass, warn, fail func(string)) {
 }
 
 func platformCleanup(dataDir string, pass, warn, fail func(string)) {
-	// Clean stale VM clone directories
-	clonesDir := fmt.Sprintf("%s/vm/macos/jobs", dataDir)
-	if entries, err := os.ReadDir(clonesDir); err == nil {
-		removed := 0
-		for _, e := range entries {
-			if !e.IsDir() {
-				continue
-			}
-			path := fmt.Sprintf("%s/%s", clonesDir, e.Name())
-			if err := os.RemoveAll(path); err != nil {
-				warn(fmt.Sprintf("could not remove stale VM clone %s: %v", e.Name(), err))
-			} else {
-				removed++
-			}
-		}
-		if removed > 0 {
-			pass(fmt.Sprintf("removed %d stale macOS VM clone(s)", removed))
-		} else {
-			pass("no stale macOS VM clones")
-		}
+	// Clean stale VM clones: the per-job share directories AND the <id>.img /
+	// <id>.aux disk clones next to them. Only the directories used to go,
+	// which left the multi-GB disk clones behind. Doctor only cleans with the
+	// daemon stopped, so nothing here can belong to a running VM — no age gate.
+	res := vm.SweepStaleMacOSClones(dataDir, nil, 0, time.Now(), slog.New(slog.NewTextHandler(io.Discard, nil)))
+	for _, f := range res.Failed {
+		warn("could not remove stale VM clone " + f)
+	}
+	if res.Removed > 0 {
+		pass(fmt.Sprintf("removed %d stale macOS VM clone(s), %.1f GB", res.Removed, float64(res.FreedBytes)/1e9))
+	} else if len(res.Failed) == 0 {
+		pass("no stale macOS VM clones")
 	}
 }

cmd/ephemerd/main.go

@@ -922,12 +922,13 @@ func serve(ctx context.Context, configFile, imagesDirFlag string, containerdTCPP
 				return
 			}
 			sched.SetMacOSVMConfig(&vm.MacOSVMConfig{
-				DataDir:   configDir,
-				DiskImage: files.DiskImage,
-				SSHSigner: sshSigner,
-				SSHPubKey: sshPubKey,
-				CPUs:      cfg.VM.MacOS.CPUs,
-				MemoryMB:  cfg.VM.MacOS.MemoryMB,
+				DataDir:    configDir,
+				DiskImage:  files.DiskImage,
+				SSHSigner:  sshSigner,
+				SSHPubKey:  sshPubKey,
+				CPUs:       cfg.VM.MacOS.CPUs,
+				MemoryMB:   cfg.VM.MacOS.MemoryMB,
+				DiskSizeGB: vm.ResolveMacOSDiskSize(files.DiskImage, cfg.VM.MacOS.DiskSizeGB, log),
 				// Container jobs get this env from the runtime; a macOS VM
 				// job has no container, so without it every macOS job
 				// bypassed the caches entirely.

pkg/config/config.go

@@ -1294,6 +1294,12 @@ type MacOSVMToml struct {
 	MemoryMB      uint64 `toml:"memory_mb"`      // memory per VM in MB (default: 8192)
 	MaxConcurrent int    `toml:"max_concurrent"` // max simultaneous macOS VMs (default: auto-detected from host CPUs)
 
+	// DiskSizeGB grows each job's disk to this many GiB. 0 (the default)
+	// keeps the base image's size. The clone is sparse, so host disk is only
+	// spent as the guest writes. A value not larger than the base image is
+	// ignored with a warning — the disk is never shrunk. See pkg/vm/macos_disk.go.
+	DiskSizeGB uint64 `toml:"disk_size_gb"`
+
 	// ProvisionTimeout bounds the pre-registration provisioning phase of a
 	// macOS VM job (boot + wait for the runner to become reachable). If a VM
 	// has not registered its runner within this window it is force-stopped and

pkg/config/config_test.go

@@ -1814,6 +1814,7 @@ disk_size_gb = 100
 disk_image = "/path/to/macos.img"
 cpus = 8
 memory_mb = 16384
+disk_size_gb = 100
 `), 0644); err != nil {
 		t.Fatal(err)
 	}
@@ -1847,6 +1848,9 @@ memory_mb = 16384
 	if cfg.VM.MacOS.MemoryMB != 16384 {
 		t.Errorf("VM.MacOS.MemoryMB = %d, want 16384", cfg.VM.MacOS.MemoryMB)
 	}
+	if cfg.VM.MacOS.DiskSizeGB != 100 {
+		t.Errorf("VM.MacOS.DiskSizeGB = %d, want 100", cfg.VM.MacOS.DiskSizeGB)
+	}
 }
 
 // --- Additional Providers() and validate() coverage ---

pkg/scheduler/macclones.go

@@ -0,0 +1,99 @@
+package scheduler
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/ephpm/ephemerd/pkg/vm"
+)
+
+// macCloneSweepInterval is how often the stale macOS clone sweep runs after
+// its startup pass. A leaked clone costs what its job wrote — tens of GB for
+// a heavy build — so a day of leaks must never pile up, but the pass is a
+// single directory read when there is nothing to do.
+const macCloneSweepInterval = 30 * time.Minute
+
+// macCloneSweeper owns the periodic sweep of <data>/vm/macos/jobs (see
+// pkg/vm/macos_clones.go for why clones leaked and the layers that now stop
+// it) and its running totals for status.
+type macCloneSweeper struct {
+	once sync.Once
+	kick chan struct{}
+
+	mu         sync.Mutex
+	removed    int
+	freedBytes int64
+
+	// sweep overrides vm.SweepStaleMacOSClones; only tests set it.
+	sweep func(dataDir string, live map[string]bool, grace time.Duration, now time.Time) vm.MacOSCloneSweep
+}
+
+func (m *macCloneSweeper) ch() chan struct{} {
+	m.once.Do(func() { m.kick = make(chan struct{}, 1) })
+	return m.kick
+}
+
+// kickNow requests a pass without waiting for the next tick. Non-blocking: a
+// pass already pending covers this request.
+func (m *macCloneSweeper) kickNow() {
+	select {
+	case m.ch() <- struct{}{}:
+	default:
+	}
+}
+
+// totals reports how many clones the sweep has removed since start, and the
+// disk that gave back.
+func (m *macCloneSweeper) totals() (int, int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.removed, m.freedBytes
+}
+
+// runMacCloneSweeper sweeps on every kick (SetMacOSVMConfig sends the first
+// one, which is the startup pass) and every macCloneSweepInterval. A tick
+// before macOS support is configured does nothing.
+func (s *Scheduler) runMacCloneSweeper(ctx context.Context) {
+	ticker := time.NewTicker(macCloneSweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		case <-s.macClones.ch():
+		}
+		s.sweepMacClones(time.Now())
+	}
+}
+
+// sweepMacClones runs one pass. Protected from it: every VM this process
+// still owns files for (vm.LiveMacOSVMIDs — provisioning, running, or in a
+// wedged stop, which s.running does not cover) and anything written within
+// vm.MacOSCloneGrace.
+func (s *Scheduler) sweepMacClones(now time.Time) {
+	s.mu.Lock()
+	cfg := s.cfg.MacOSVMConfig
+	s.mu.Unlock()
+	if cfg == nil || cfg.DataDir == "" {
+		return
+	}
+	sweep := s.macClones.sweep
+	if sweep == nil {
+		log := s.cfg.Log.With("component", "macos-clone-sweeper")
+		sweep = func(dataDir string, live map[string]bool, grace time.Duration, now time.Time) vm.MacOSCloneSweep {
+			return vm.SweepStaleMacOSClones(dataDir, live, grace, now, log)
+		}
+	}
+	res := sweep(cfg.DataDir, vm.LiveMacOSVMIDs(), vm.MacOSCloneGrace, now)
+	if res.Removed == 0 {
+		return
+	}
+	s.macClones.mu.Lock()
+	s.macClones.removed += res.Removed
+	s.macClones.freedBytes += res.FreedBytes
+	s.macClones.mu.Unlock()
+	s.cfg.Log.Info("swept stale macOS VM clones", "component", "macos-clone-sweeper",
+		"removed", res.Removed, "freed_bytes", res.FreedBytes, "failed", len(res.Failed))
+}

pkg/scheduler/macclones_test.go

@@ -0,0 +1,75 @@
+package scheduler
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/ephpm/ephemerd/pkg/vm"
+)
+
+// TestSweepMacClones_IdleWithoutMacOS: a node with no macOS support (every
+// Linux and Windows node, and a Mac before its base image is ready) never
+// touches the disk.
+func TestSweepMacClones_IdleWithoutMacOS(t *testing.T) {
+	s := New(Config{Log: quietLogger()})
+	called := false
+	s.macClones.sweep = func(string, map[string]bool, time.Duration, time.Time) vm.MacOSCloneSweep {
+		called = true
+		return vm.MacOSCloneSweep{}
+	}
+	s.sweepMacClones(time.Now())
+	if called {
+		t.Error("swept with no macOS VM config")
+	}
+}
+
+// TestSweepMacClones_PassesGraceAndTotals: a pass sweeps the configured data
+// dir with the production grace and the live set, and its results add up in
+// the totals status reports.
+func TestSweepMacClones_PassesGraceAndTotals(t *testing.T) {
+	s := New(Config{Log: quietLogger(), MacOSVMConfig: &vm.MacOSVMConfig{DataDir: "/var/lib/ephemerd"}})
+	var gotDir string
+	var gotGrace time.Duration
+	s.macClones.sweep = func(dir string, live map[string]bool, grace time.Duration, _ time.Time) vm.MacOSCloneSweep {
+		gotDir, gotGrace = dir, grace
+		if live == nil {
+			t.Error("sweep got a nil live set; it must always get the live VM ids")
+		}
+		return vm.MacOSCloneSweep{Removed: 3, FreedBytes: 38 << 30}
+	}
+	s.sweepMacClones(time.Now())
+	s.sweepMacClones(time.Now())
+	if gotDir != "/var/lib/ephemerd" || gotGrace != vm.MacOSCloneGrace {
+		t.Errorf("swept dir=%q grace=%s, want the data dir and vm.MacOSCloneGrace", gotDir, gotGrace)
+	}
+	if n, freed := s.macClones.totals(); n != 6 || freed != 2*(38<<30) {
+		t.Errorf("totals = (%d, %d), want (6, %d)", n, freed, 2*(38<<30))
+	}
+}
+
+// TestMacCloneSweeper_KickRunsAPass: SetMacOSVMConfig is the startup pass —
+// it must trigger a sweep promptly, not wait for the 30 minute tick.
+func TestMacCloneSweeper_KickRunsAPass(t *testing.T) {
+	s := New(Config{Log: quietLogger()})
+	passes := make(chan struct{}, 4)
+	s.macClones.sweep = func(string, map[string]bool, time.Duration, time.Time) vm.MacOSCloneSweep {
+		passes <- struct{}{}
+		return vm.MacOSCloneSweep{}
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go s.runMacCloneSweeper(ctx)
+
+	s.SetMacOSVMConfig(&vm.MacOSVMConfig{DataDir: t.TempDir()})
+	select {
+	case <-passes:
+	case <-time.After(2 * time.Second):
+		t.Fatal("SetMacOSVMConfig did not trigger a sweep")
+	}
+
+	// Kicks coalesce: a burst never blocks the caller.
+	for range 10 {
+		s.macClones.kickNow()
+	}
+}

pkg/scheduler/scheduler.go

@@ -295,6 +295,10 @@ type Scheduler struct {
 	// default; only tests set it.
 	macAbandonedCap int
 
+	// macClones runs the periodic stale-clone sweep for macOS VMs and keeps
+	// its totals for status. See macclones.go.
+	macClones macCloneSweeper
+
 	// busyProbe overrides the ground-truth "is this runner executing a
 	// job" check that vetoes orphan teardown. Nil (the default) uses
 	// probeRunnerBusy: local container/VM/process introspection, falling
@@ -324,8 +328,11 @@ const maxProvisionAttempts = 5
 // finishes.
 func (s *Scheduler) SetMacOSVMConfig(cfg *vm.MacOSVMConfig) {
 	s.mu.Lock()
-	defer s.mu.Unlock()
 	s.cfg.MacOSVMConfig = cfg
+	s.mu.Unlock()
+	// macOS jobs can start any moment now: sweep what earlier daemons left
+	// behind first. The sweep runs on its own goroutine (see Run).
+	s.macClones.kickNow()
 }
 
 // failureBackoff tracks per-repo failure counts to compute exponential backoff.
@@ -647,6 +654,9 @@ func (s *Scheduler) Run(ctx context.Context) error {
 		go s.retry.Run(ctx)
 	}
 
+	// Stale macOS VM clones: idle until macOS support is configured.
+	go s.runMacCloneSweeper(ctx)
+
 	// Clean old job logs on startup, then periodically every hour.
 	// Retention period is configurable via [log] log_retention (default 7d).
 	logDir := filepath.Join(s.cfg.DataDir, "logs")
@@ -2418,6 +2428,12 @@ func (s *Scheduler) handleHealthz(w http.ResponseWriter, r *http.Request) {
 	if n := s.AbandonedMacOSVMDetails(); len(n) > 0 {
 		status["abandoned_macos_vm_details"] = n
 	}
+	// Leaked macOS disk clones the sweeper removed since start. Non-zero means
+	// some teardown did not clean up after itself — see macclones.go.
+	if n, freed := s.macClones.totals(); n > 0 {
+		status["macos_clones_swept"] = n
+		status["macos_clone_bytes_swept"] = freed
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(status); err != nil {

pkg/vm/diskblocks_unix.go

@@ -0,0 +1,17 @@
+//go:build !windows
+
+package vm
+
+import (
+	"os"
+	"syscall"
+)
+
+// diskBlocks reports the bytes a file actually occupies. st_blocks is in
+// 512-byte units on every unix ephemerd runs on.
+func diskBlocks(fi os.FileInfo) int64 {
+	if st, ok := fi.Sys().(*syscall.Stat_t); ok {
+		return int64(st.Blocks) * 512
+	}
+	return fi.Size()
+}

pkg/vm/diskblocks_windows.go

@@ -0,0 +1,7 @@
+package vm
+
+import "os"
+
+// diskBlocks falls back to the apparent size. macOS clones never live on a
+// Windows host; this only keeps the sweep portable for tests.
+func diskBlocks(fi os.FileInfo) int64 { return fi.Size() }