Skip to content

feat(runtime): fill MicroVMRuntime lifecycle methods (RIG-2493) #719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rigel-mintaka wants to merge 1 commit into
base: compass-runner/rig-2493-u3-host-exec
Choose a base branch
from
+1,423 −134
Open

feat(runtime): fill MicroVMRuntime lifecycle methods (RIG-2493) #719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 49 additions & 5 deletions go/cmd/compass-runner/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"log/slog"
"os"
"os/signal"
"strconv"
"strings"
"syscall"

Expand Down Expand Up @@ -172,7 +173,8 @@ func run() error {
// backendFlags holds the runtime-backend selection flags, registered on the
// default flag set before flag.Parse and resolved into a runtime after it.
type backendFlags struct {
backend, vmm, virtiofsd, kernel, rootfs *string
backend, vmm, virtiofsd, kernel, rootfs, initrd, runRoot *string
cpus, memoryMB *int
}

// registerBackendFlags declares the backend-selection flags. Call before
Expand All @@ -190,19 +192,41 @@ func registerBackendFlags() backendFlags {
"Path to the guest kernel image (microvm backend). Defaults to $COMPASS_MICROVM_KERNEL."),
rootfs: flag.String("microvm-rootfs", "",
"Path to the guest rootfs image (microvm backend). Defaults to $COMPASS_MICROVM_ROOTFS."),
initrd: flag.String("microvm-initrd", "",
"Path to the guest initramfs image (microvm backend). Defaults to $COMPASS_MICROVM_INITRD."),
runRoot: flag.String("microvm-runroot", "",
"Root dir for per-session microVM runtime dirs (microvm backend). Defaults to $COMPASS_MICROVM_RUNROOT."),
cpus: flag.Int("microvm-cpus", 0,
"Default vCPU count per session guest (microvm backend); 0 leaves the VMM default. "+
"Defaults to $COMPASS_MICROVM_CPUS."),
memoryMB: flag.Int("microvm-memory-mb", 0,
"Default guest RAM in MiB per session (microvm backend); 0 leaves the VMM default. "+
"Defaults to $COMPASS_MICROVM_MEMORY_MB."),
}
}

// selectEngine resolves the configured runtime backend from the parsed flags
// and their environment fallbacks.
func (f backendFlags) selectEngine() (runtime.ContainerRuntime, error) {
cpus, err := intOrEnv(*f.cpus, "COMPASS_MICROVM_CPUS")
if err != nil {
return nil, err
}
memoryMB, err := intOrEnv(*f.memoryMB, "COMPASS_MICROVM_MEMORY_MB")
if err != nil {
return nil, err
}
return runtime.SelectBackend(runtime.BackendConfig{
Backend: orEnv(*f.backend, "COMPASS_RUNTIME_BACKEND"),
MicroVM: runtime.MicroVMConfig{
VMMPath: orEnv(*f.vmm, "COMPASS_MICROVM_VMM"),
VirtiofsdPath: orEnv(*f.virtiofsd, "COMPASS_MICROVM_VIRTIOFSD"),
KernelImage: orEnv(*f.kernel, "COMPASS_MICROVM_KERNEL"),
RootfsImage: orEnv(*f.rootfs, "COMPASS_MICROVM_ROOTFS"),
VMMPath: orEnv(*f.vmm, "COMPASS_MICROVM_VMM"),
VirtiofsdPath: orEnv(*f.virtiofsd, "COMPASS_MICROVM_VIRTIOFSD"),
KernelImage: orEnv(*f.kernel, "COMPASS_MICROVM_KERNEL"),
RootfsImage: orEnv(*f.rootfs, "COMPASS_MICROVM_ROOTFS"),
InitrdImage: orEnv(*f.initrd, "COMPASS_MICROVM_INITRD"),
RunRoot: orEnv(*f.runRoot, "COMPASS_MICROVM_RUNROOT"),
DefaultCPUs: cpus,
DefaultMemoryMB: memoryMB,
},
})
}
Expand All @@ -215,6 +239,26 @@ func orEnv(flagVal, envKey string) string {
return os.Getenv(envKey)
}

// intOrEnv returns flagVal when non-zero, else the named environment variable
// parsed as an int. An empty env var is 0 (unset — the config treats 0 as
// "leave the VMM default"); a present-but-non-numeric env var is an error
// naming the offending variable and value so a misconfiguration surfaces at
// startup rather than as a zero silently swallowing a typo.
func intOrEnv(flagVal int, envKey string) (int, error) {
if flagVal != 0 {
return flagVal, nil
}
raw := os.Getenv(envKey)
if raw == "" {
return 0, nil
}
parsed, err := strconv.Atoi(raw)
if err != nil {
return 0, fmt.Errorf("$%s=%q is not an integer: %w", envKey, raw, err)
}
return parsed, nil
}

// parseEgress parses the comma-separated allowlist into a validated EgressPolicy.
// An empty list is a valid default-deny policy (no host reachable).
func parseEgress(csv string) (runtime.EgressPolicy, error) {
Expand Down
122 changes: 46 additions & 76 deletions go/internal/runtime/microvm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
package runtime

// microvm.go is the microVM ContainerRuntime backend seam: a MicroVMRuntime
// that satisfies the same ContainerRuntime interface as PodmanCLI, plus the
// config-driven backend selection the Runner startup uses to choose between
// them. Every runtime method is a typed-error stub here — the in-guest control
// plane that boots a VMM, wires the virtiofs share, and speaks the agent
// protocol over vsock lands later, behind these frozen signatures. Selecting
// the microVM backend today therefore fails loudly at first use rather than
// silently faking container behavior.
// microvm.go is the microVM ContainerRuntime backend seam: the operator config,
// the MicroVMRuntime type + its per-session state table, and the config-driven
// backend selection the Runner startup uses to choose between the microVM and
// podman backends. The lifecycle method bodies — which boot a VMM, wire the
// virtiofs share, and speak the guest control plane over vsock — live in
// microvm_lifecycle.go behind a //go:build unix tag, because the microvm
// package they call (Launch/GuestExec/VM) is itself unix-only. This file holds
// only what backend selection needs to type-check on any platform: the config
// structs, the type declaration, and SelectBackend.

import (
"context"
"errors"
"fmt"
"strings"
"time"
"sync"
)

// MicroVMConfig is the operator-supplied wiring for the microVM backend: the
// paths to the VMM and virtiofs daemon binaries and to the guest kernel and
// rootfs images. Empty fields are tolerated at construction — the values are
// consumed when the in-guest control plane lands, and the V5 preflight names
// any missing one at startup rather than deep in a launch.
// paths to the VMM and virtiofs daemon binaries, the guest boot images, the
// per-session runtime-dir root, and the default guest sizing. Empty fields are
// tolerated at construction — the values are consumed when a session boots, and
// the V5 preflight names any missing one at startup rather than deep in a
// launch.
type MicroVMConfig struct {
// VMMPath is the path to the virtual machine monitor binary.
VMMPath string
Expand All @@ -32,6 +32,21 @@ type MicroVMConfig struct {
KernelImage string
// RootfsImage is the path to the guest root filesystem image.
RootfsImage string
// InitrdImage is the path to the guest initramfs image. Load-bearing, not
// optional: the pinned generic kernel ships its virtio/erofs/overlay drivers
// as modules, so the initrd is what loads them and mounts the root before
// switch_root (microvm-v2a §(a)).
InitrdImage string
// RunRoot is the root under which each session's runtime dir is created
// (<RunRoot>/microvm/<session>/), holding that session's AF_UNIX sockets —
// the layout V7 formalizes with pidfiles.
RunRoot string
// DefaultCPUs is the vCPU count each session guest boots with (hotplug-grown
// later per D5). Zero leaves it to the VMM's own default.
DefaultCPUs int
// DefaultMemoryMB is the RAM each session guest boots with, in MiB
// (hotplug-grown later per D5). Zero leaves it to the VMM's own default.
DefaultMemoryMB int
}

// BackendConfig selects and configures the container runtime backend. Backend
Expand All @@ -45,74 +60,29 @@ type BackendConfig struct {
MicroVM MicroVMConfig
}

// ErrMicroVMNotImplemented is returned by every MicroVMRuntime method until the
// in-guest control plane lands. The full ContainerRuntime surface is frozen on
// the type now (so backend selection can choose it and no interface change
// lands later); the VMM boot, virtiofs share, and vsock agent transport behind
// each verb are still to come, so invoking one today is a programming error the
// sentinel names explicitly rather than a silent no-op that would fake a
// container operation that never happened.
var ErrMicroVMNotImplemented = errors.New("runtime: MicroVMRuntime is not implemented until the in-guest control plane lands")

// MicroVMRuntime is a ContainerRuntime that isolates each agent in its own
// microVM instead of a rootless container. It holds the microVM wiring the
// in-guest control plane will consume; its methods are typed-error stubs until
// that lands.
// microVM instead of a rootless container. It holds the operator wiring plus a
// per-session state table (keyed by the ContainerID Create mints), guarded by
// mu against concurrent lifecycle calls. Its method bodies live in
// microvm_lifecycle.go (//go:build unix); the microvmSession type they operate
// on is declared there too.
type MicroVMRuntime struct {
config MicroVMConfig
mu sync.Mutex
// sessions maps each live ContainerID to its session state. Every read and
// write is guarded by mu. Name lookups (Exists, duplicate-name refusal) scan
// this map for a matching spec.Name — a scan is cheap at one-VM-per-session
// scale and keeps a single source of truth.
sessions map[ContainerID]*microvmSession
}

// NewMicroVMRuntime builds a MicroVMRuntime from the supplied config, mirroring
// NewPodmanCLI's shape.
// NewPodmanCLI's shape, with an empty session table ready for Create.
func NewMicroVMRuntime(cfg MicroVMConfig) *MicroVMRuntime {
return &MicroVMRuntime{config: cfg}
}

var _ ContainerRuntime = (*MicroVMRuntime)(nil)

// Create is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Create(_ context.Context, _ ContainerSpec) (ContainerID, error) {
return "", ErrMicroVMNotImplemented
}

// Start is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Start(_ context.Context, _ ContainerID) error {
return ErrMicroVMNotImplemented
}

// Exec is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Exec(_ context.Context, _ ContainerID, _ ExecSpec) (ExecOutput, error) {
return ExecOutput{}, ErrMicroVMNotImplemented
}

// ExecStreaming is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) ExecStreaming(_ context.Context, _ ContainerID, _ StreamingExecSpec) (*StreamingExec, error) {
return nil, ErrMicroVMNotImplemented
}

// Stop is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Stop(_ context.Context, _ ContainerID, _ time.Duration) error {
return ErrMicroVMNotImplemented
}

// Remove is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Remove(_ context.Context, _ ContainerID) error {
return ErrMicroVMNotImplemented
}

// Exists is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Exists(_ context.Context, _ string) (bool, error) {
return false, ErrMicroVMNotImplemented
}

// MountLabel is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) MountLabel(_ context.Context, _ ContainerID) (string, error) {
return "", ErrMicroVMNotImplemented
}

// Resize is unimplemented until the in-guest control plane lands.
func (m *MicroVMRuntime) Resize(_ context.Context, _ ContainerID, _ ResourceLimits) error {
return ErrMicroVMNotImplemented
return &MicroVMRuntime{
config: cfg,
sessions: make(map[ContainerID]*microvmSession),
}
}

// SelectBackend chooses the container runtime backend from cfg. An empty or
Expand Down
56 changes: 40 additions & 16 deletions go/internal/runtime/microvm/launch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"

Expand Down Expand Up @@ -69,7 +70,7 @@ type child struct {
name string
cmd *exec.Cmd
logPath string
waited bool // set once cmd.Wait has returned, so liveness probes and PSS skip a reaped process
waited atomic.Bool // set once cmd.Wait has returned, so liveness probes and PSS skip a reaped process
}

// VM is a running (or partially-started, on the Launch error path) guest and
Expand All @@ -82,6 +83,12 @@ type VM struct {
virtiofsd *child // nil under the net-only smoke (no --fs)
passt *child

// vmmExited is closed by the sole VMM reaper (started in launch) once the
// cloud-hypervisor process has been Wait'd, so the caller can observe a
// prompt guest self-power-off instead of a zombie-blind Signal(0) poll. Nil
// only under the hermetic fail-closed path (no VMM on PATH).
vmmExited chan struct{}

consolePath string // --serial file: the guest serial console

vsockSocket string // host end of the hybrid vsock (empty under the net-only smoke)
Expand Down Expand Up @@ -215,6 +222,16 @@ func launch(ctx context.Context, cfg BootConfig, opts launchOptions) (_ *VM, err
if startErr := startChild(vm.vmm); startErr != nil {
return nil, fmt.Errorf("microvm: starting cloud-hypervisor: %w", startErr)
}
// The sole VMM reaper owns the single cmd.Wait for cloud-hypervisor: it
// unblocks WaitVMMExit on a guest self-power-off and lets Shutdown observe
// the exit without a second Wait. The Wait error is deliberately discarded —
// a killed VMM yields an expected *exec.ExitError, mirroring waitResult.
vm.vmmExited = make(chan struct{})
go func() {
_ = vm.vmm.cmd.Wait() // discard: a killed VMM's *exec.ExitError is the expected teardown outcome (mirrors waitResult)
vm.vmm.waited.Store(true)
close(vm.vmmExited)
}()
if opts.withVsock {
vm.sockets = append(vm.sockets, cfg.VsockSocket)
}
Expand Down Expand Up @@ -333,14 +350,14 @@ func (vm *VM) Health(ctx context.Context) (*compassv1.HealthResponse, error) {
func (vm *VM) Shutdown(ctx context.Context) error {
vm.shutdownOnce.Do(func() {
var errs []error
// VMM first: kill outright, then Wait to reap.
// VMM first: kill outright, then let the sole reaper's single Wait
// complete via vmmExited (Shutdown must not Wait the VMM itself — that
// would be a second Wait on the same process).
if vm.vmm != nil && vm.vmm.cmd.Process != nil {
if killErr := vm.vmm.cmd.Process.Kill(); killErr != nil && !errors.Is(killErr, os.ErrProcessDone) {
errs = append(errs, fmt.Errorf("killing cloud-hypervisor: %w", killErr))
}
if waitErr := waitProcess(vm.vmm); waitErr != nil {
errs = append(errs, waitErr)
}
<-vm.vmmExited
}
// Then the auxiliary daemons: SIGTERM, bounded wait, SIGKILL.
for _, c := range []*child{vm.virtiofsd, vm.passt} {
Expand Down Expand Up @@ -378,24 +395,31 @@ func reap(c *child) error {
go func() { done <- c.cmd.Wait() }()
select {
case err := <-done:
c.waited = true
c.waited.Store(true)
return waitResult(c.name, err)
case <-time.After(reapGrace):
if killErr := c.cmd.Process.Kill(); killErr != nil && !errors.Is(killErr, os.ErrProcessDone) {
return fmt.Errorf("SIGKILL %s: %w", c.name, killErr)
}
c.waited = true
c.waited.Store(true)
return waitResult(c.name, <-done)
}
}

// waitProcess Wait's a process that has already been signalled to die and
// normalizes the "expected" exit (killed/exited non-zero) to nil — Shutdown
// killed it on purpose, so a non-nil ExitError is not a Shutdown failure.
func waitProcess(c *child) error {
err := c.cmd.Wait()
c.waited = true
return waitResult(c.name, err)
// WaitVMMExit reports whether the VMM process exited within timeout, observed
// via the reaper (not a zombie-blind Signal(0) poll): a guest that powers itself
// off makes the reaper's Wait return and close vmmExited promptly, so the caller
// sees the self-exit instead of burning the full grace window on a zombie.
func (vm *VM) WaitVMMExit(timeout time.Duration) bool {
if vm.vmm == nil || vm.vmmExited == nil {
return true
}
select {
case <-vm.vmmExited:
return true
case <-time.After(timeout):
return false
}
}

// waitResult swallows the ExitError a deliberately-killed process yields (a
Expand All @@ -417,7 +441,7 @@ func waitResult(name string, err error) error {
// is definitively gone; otherwise signal 0 probes liveness without affecting it.
func (vm *VM) Running(name string) bool {
c := vm.childByName(name)
if c == nil || c.cmd.Process == nil || c.waited {
if c == nil || c.cmd.Process == nil || c.waited.Load() {
return false
}
return c.cmd.Process.Signal(syscall.Signal(0)) == nil
Expand All @@ -433,7 +457,7 @@ func (vm *VM) PSS() (map[string]int64, error) {
out := make(map[string]int64)
var errs []error
for _, c := range []*child{vm.vmm, vm.virtiofsd, vm.passt} {
if c == nil || c.cmd.Process == nil || c.waited {
if c == nil || c.cmd.Process == nil || c.waited.Load() {
continue
}
pss, err := readPSS(c.cmd.Process.Pid)
Expand Down
Loading
Loading