From 08fe5d317422045cae42b9deeafe5c300d127e1c Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:00:52 +0530 Subject: [PATCH 1/2] Run VM guests on macOS with Hypervisor.framework On an Apple Silicon Mac with QEMU installed (brew install qemu), VM instances now run QEMU on the Mac with -accel hvf instead of emulated inside the Docker VM, which has no nested virtualization. The VM container stays and keeps the instance's address, passt, security groups, published ports, metadata route, disks and console log. In host mode its entrypoint prepares the disks and seed, starts passt and serves the disks over NBD (qemu-storage-daemon) on Unix sockets. A host runner (homecloud vm-host, a detached process per guest) starts QEMU and relays its network card and disks into the container through docker exec streams, copies the serial console into the container log, and ties QEMU's lifetime to the container's. The serial console, guest agent and QMP sockets are in the guest's run directory, where the terminal, run-command, metrics and power controls connect. The backend is chosen automatically (instances report virtualization hvf), with serve --vm-accel / vm_accel / HC_VM_ACCEL to force hvf, container, kvm or tcg, and falls back to the container when QEMU, its firmware or Hypervisor.framework is unavailable. The firmware is QEMU's edk2 build: Debian's AAVMF hangs under hvf. Also stop unit tests from pinning hostArch for later live tests. --- README.md | 4 +- cli/cmd/serve.go | 9 + cli/cmd/vmhost.go | 44 ++ cli/internal/core/core.go | 6 + cli/internal/runtime/self.go | 3 + cli/internal/svc/ec2/capabilities.go | 8 +- cli/internal/svc/ec2/ec2.go | 7 + cli/internal/svc/ec2/terminal.go | 39 ++ cli/internal/svc/ec2/vm.go | 59 +- cli/internal/svc/ec2/vm/hc-vm-run | 44 +- cli/internal/svc/ec2/vm/host.go | 141 +++++ cli/internal/svc/ec2/vm/host_test.go | 72 +++ cli/internal/svc/ec2/vm_access.go | 42 +- cli/internal/svc/ec2/vm_host.go | 622 ++++++++++++++++++++ cli/internal/svc/ec2/vm_host_unix.go | 72 +++ cli/internal/svc/ec2/vm_host_windows.go | 18 + cli/internal/svc/ec2/vm_live_test.go | 2 +- cli/internal/svc/ec2/vm_security_test.go | 37 +- console/components/ec2/instance-actions.tsx | 9 +- console/components/ec2/instance-detail.tsx | 6 +- console/components/ec2/launch-wizard.tsx | 8 +- console/lib/types/ec2.ts | 8 +- docs/architecture.md | 4 +- docs/install-server.md | 7 +- 24 files changed, 1221 insertions(+), 50 deletions(-) create mode 100644 cli/cmd/vmhost.go create mode 100644 cli/internal/svc/ec2/vm/host.go create mode 100644 cli/internal/svc/ec2/vm/host_test.go create mode 100644 cli/internal/svc/ec2/vm_host.go create mode 100644 cli/internal/svc/ec2/vm_host_unix.go create mode 100644 cli/internal/svc/ec2/vm_host_windows.go diff --git a/README.md b/README.md index 3f65bd95..7497393f 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Run `homecloud aws-env` to get the environment variables. For Terraform, point t | Service | AWS equivalent | Status | Notes | | --- | --- | --- | --- | -| Compute | EC2, EBS, AMIs | AWS API | Instances are **containers**; `ami-ubuntu-24-04-vm` and `ami-debian-12-vm` boot real **virtual machines** (QEMU, KVM when the host has `/dev/kvm`, emulated otherwise) on the same VPC networking, with extra volumes, snapshots, images, run-command and a serial console. Key pairs, user data, volumes, snapshots, launch templates, Elastic IPs (records only), IMDSv1/v2, browser shell | +| Compute | EC2, EBS, AMIs | AWS API | Instances are **containers**; `ami-ubuntu-24-04-vm` and `ami-debian-12-vm` boot real **virtual machines** (QEMU, KVM when the host has `/dev/kvm`, Hypervisor.framework on Apple Silicon Macs with QEMU installed, emulated otherwise) on the same VPC networking, with extra volumes, snapshots, images, run-command and a serial console. Key pairs, user data, volumes, snapshots, launch templates, Elastic IPs (records only), IMDSv1/v2, browser shell | | Auto Scaling | EC2 Auto Scaling | AWS API | Target tracking on CPU; scheduled actions and lifecycle hooks are not implemented | | Networking | VPC, security groups | AWS API | Security groups enforced inside the VPC; network ACLs recorded, not enforced; no peering; IPv4 only | | Load balancing | ELB v2 | AWS API | Application load balancers (HTTP/HTTPS, path/host rules). No network load balancers | @@ -229,7 +229,7 @@ Releases are signed (from v0.5.0): the release workflow signs `checksums.txt` an - **Shipped in [0.4.0](https://github.com/solinode/homecloud/releases/tag/v0.4.0):** VM-backed EC2 instances (QEMU, KVM when available), the container image `ghcr.io/solinode/homecloud`, a GitHub Action and testcontainers modules for Go and Python, a nightly Terraform modules compatibility suite ([17 of 17 pass](docs/compatibility.md)), and fixes from two security audits ([platform](docs/security-audit-2026-10.md), [VM instances](docs/security-audit-vm-2026-10.md)). - **Shipped in [0.5.0](https://github.com/solinode/homecloud/releases/tag/v0.5.0):** signed releases (Sigstore) with SBOMs and build provenance, verified by the installer and `homecloud upgrade`; automatic backups before upgrades and `homecloud upgrade --rollback`; restores that bring back databases, caches, load balancers and instances on a new machine; Prometheus metrics and a system status page ([docs/monitoring.md](docs/monitoring.md)); Testcontainers modules for Java and Node. -- **Next:** closing the AWS API gaps users hit most, verified CDK and Pulumi support, native VM acceleration on macOS, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms. +- **Next:** closing the AWS API gaps users hit most, verified CDK and Pulumi support, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms. - **Planned:** multi-node clusters ([#55](https://github.com/solinode/homecloud/issues/55), [design](docs/design/multi-node.md)), edge compute and hardware integrations ([#56](https://github.com/solinode/homecloud/issues/56), [design](docs/design/edge.md)). See the [open issues](https://github.com/solinode/homecloud/issues) for everything planned. diff --git a/cli/cmd/serve.go b/cli/cmd/serve.go index 659cae1a..03e11c89 100644 --- a/cli/cmd/serve.go +++ b/cli/cmd/serve.go @@ -17,6 +17,7 @@ import ( "github.com/homecloudhq/homecloud/cli/internal/core" "github.com/homecloudhq/homecloud/cli/internal/httpx" "github.com/homecloudhq/homecloud/cli/internal/server" + "github.com/homecloudhq/homecloud/cli/internal/svc/ec2" "github.com/spf13/cobra" ) @@ -47,6 +48,7 @@ and CLI credentials in the data directory. Settings passed as flags are saved to } server.Version = Version server.Commit = Commit + ec2.VMHostCommand = vmHostCommand ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() return server.Run(ctx, cfg, server.Options{ResetRootPassword: resetRoot, AllowNewerData: allowNewerData}) @@ -70,6 +72,7 @@ and CLI credentials in the data directory. Settings passed as flags are saved to f.BoolVar(&allowNewerData, "allow-newer-data", false, "start even if the data directory was last used by a newer HomeCloud (risks corrupting it; prefer homecloud upgrade --rollback)") f.StringVar(&cfg.MetricsAddr, "metrics-addr", cfg.MetricsAddr, "also serve Prometheus metrics WITHOUT authentication on this loopback address, e.g. 127.0.0.1:9464 (the API's /metrics always requires the IAM action homecloud:GetMetrics)") f.BoolVar(&cfg.MetricsAddrInsecure, "metrics-addr-insecure", false, "allow a non-loopback --metrics-addr (its metrics need no credentials: protect it with a firewall)") + f.StringVar(&cfg.VMAccel, "vm-accel", "auto", "how VM instances run: auto, hvf (QEMU on this Mac with Hypervisor.framework), container (QEMU in Docker, KVM when available), kvm or tcg") RootCmd.AddCommand(serveCmd) var p client.Profile @@ -219,6 +222,12 @@ func mergeConfig(cmd *cobra.Command, flags core.Config) (core.Config, error) { set("tls-key", func() { cfg.TLSKey = flags.TLSKey }) set("metrics-addr", func() { cfg.MetricsAddr = flags.MetricsAddr }) set("metrics-addr-insecure", func() { cfg.MetricsAddrInsecure = flags.MetricsAddrInsecure }) + set("vm-accel", func() { cfg.VMAccel = flags.VMAccel }) + switch cfg.VMAccel { + case "", "auto", "hvf", "container", "kvm", "tcg": + default: + return cfg, fmt.Errorf("--vm-accel is auto, hvf, container, kvm or tcg, got %q", cfg.VMAccel) + } if err := server.ValidateMetricsAddr(cfg.MetricsAddr, cfg.MetricsAddrInsecure); err != nil { return cfg, err } diff --git a/cli/cmd/vmhost.go b/cli/cmd/vmhost.go new file mode 100644 index 00000000..a31d2ce0 --- /dev/null +++ b/cli/cmd/vmhost.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "context" + "os" + "os/exec" + "os/signal" + "syscall" + + "github.com/homecloudhq/homecloud/cli/internal/runtime" + "github.com/homecloudhq/homecloud/cli/internal/svc/ec2" + "github.com/spf13/cobra" +) + +// vm-host is started by the server, one per VM instance whose guest runs on +// this Mac (QEMU with Hypervisor.framework): it runs QEMU and joins it to the +// instance's VM container. It is a process of its own so that guests keep +// running when the server restarts. +func init() { + RootCmd.AddCommand(&cobra.Command{ + Use: "vm-host DIR", + Short: "Run the host side of a VM instance (started by the server)", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + d, err := runtime.New() + if err != nil { + return err + } + // SIGTERM and SIGINT are ignored: the guest stops when its container does. + signal.Ignore(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + return ec2.RunVMHost(context.Background(), d, args[0]) + }, + }) +} + +// vmHostCommand is ec2.VMHostCommand for the server: this binary's vm-host. +func vmHostCommand(dir string) *exec.Cmd { + exe, err := os.Executable() + if err != nil { + exe = os.Args[0] + } + return exec.Command(exe, "vm-host", dir) +} diff --git a/cli/internal/core/core.go b/cli/internal/core/core.go index c721529f..cfaa9c29 100644 --- a/cli/internal/core/core.go +++ b/cli/internal/core/core.go @@ -74,6 +74,12 @@ type Config struct { // a loopback address unless MetricsAddrInsecure is set. MetricsAddr string `json:"metrics_addr,omitempty"` MetricsAddrInsecure bool `json:"metrics_addr_insecure,omitempty"` + // VMAccel chooses how VM instances run: "auto" (the default: on an Apple + // Silicon Mac with QEMU installed, QEMU on the Mac with Hypervisor.framework; + // otherwise in the VM container with KVM when the Docker host has /dev/kvm, + // emulated if not), "hvf" (always on the Mac), "container" (always in the + // container, KVM when available), "kvm" or "tcg". HC_VM_ACCEL overrides it. + VMAccel string `json:"vm_accel,omitempty"` } func DefaultDataDir() string { diff --git a/cli/internal/runtime/self.go b/cli/internal/runtime/self.go index b09194d4..1f769991 100644 --- a/cli/internal/runtime/self.go +++ b/cli/internal/runtime/self.go @@ -30,6 +30,9 @@ import ( // /proc/self/mountinfo (/var/lib/docker/containers//hostname and friends). var containerIDPattern = regexp.MustCompile(`/containers/([0-9a-f]{64})/`) +// InContainer reports whether this process runs in a container. +func InContainer() bool { return inContainer() } + // inContainer reports whether this process runs in a container. func inContainer() bool { if os.Getenv("HOMECLOUD_IN_CONTAINER") == "1" { diff --git a/cli/internal/svc/ec2/capabilities.go b/cli/internal/svc/ec2/capabilities.go index 22958f14..66dc15df 100644 --- a/cli/internal/svc/ec2/capabilities.go +++ b/cli/internal/svc/ec2/capabilities.go @@ -5,7 +5,8 @@ import "github.com/homecloudhq/homecloud/cli/internal/httpx" // Capabilities tells the console what this host can run, so the launch wizard // can warn before someone picks a VM image on a host without KVM. type Capabilities struct { - // VM is "kvm" when virtual machine instances run hardware-accelerated, + // VM is "hvf" when virtual machine instances run on this Mac with + // Hypervisor.framework, "kvm" when they run hardware-accelerated, // "emulated" when the Docker host has no usable /dev/kvm (QEMU emulates the // CPU, which is slow), and "unavailable" when VMs cannot run at all. VM string `json:"vm"` @@ -18,8 +19,5 @@ func (s *Service) capabilities(c *httpx.Ctx) (any, error) { if s.vmRunnerFailure() != nil { return Capabilities{VM: "unavailable"}, nil } - if s.vmKVM() { - return Capabilities{VM: "kvm"}, nil - } - return Capabilities{VM: "emulated"}, nil + return Capabilities{VM: s.vmAccel()}, nil } diff --git a/cli/internal/svc/ec2/ec2.go b/cli/internal/svc/ec2/ec2.go index b0533122..2a7df187 100644 --- a/cli/internal/svc/ec2/ec2.go +++ b/cli/internal/svc/ec2/ec2.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/netip" + goruntime "runtime" "slices" "strings" "sync" @@ -1024,6 +1025,9 @@ func (s *Service) instanceGone(i Instance) { if s.imds != nil { s.imds.forget(i.ID) } + if i.IsVM() && goruntime.GOOS == "darwin" { + go s.vmHostCleanup(i.ID) + } } func (s *Service) modify(c *httpx.Ctx) (any, error) { @@ -1460,6 +1464,9 @@ func (s *Service) dns(vpcID string) []string { // Recover settles instances whose launch or state change was interrupted by a restart. func (s *Service) Recover() { + if goruntime.GOOS == "darwin" && s.env.Docker != nil { + s.vmHostWatch() // guests on this Mac whose containers kept running + } for _, i := range store.List[Instance](s.env.Store, cInstances) { switch i.State { case "pending", "stopping", "shutting-down": diff --git a/cli/internal/svc/ec2/terminal.go b/cli/internal/svc/ec2/terminal.go index ec80bdb7..33f440dc 100644 --- a/cli/internal/svc/ec2/terminal.go +++ b/cli/internal/svc/ec2/terminal.go @@ -1,9 +1,11 @@ package ec2 import ( + "context" "encoding/json" "io" "net/http" + "time" docker "github.com/fsouza/go-dockerclient" "github.com/homecloudhq/homecloud/cli/internal/core" @@ -34,6 +36,10 @@ func (s *Service) terminal(c *httpx.Ctx) (any, error) { defer conn.Close() if i.IsVM() { _ = websocket.Message.Send(conn, "[homecloud] serial console of "+i.ID+": press Enter for a login prompt (one session at a time).\r\n") + if vmHostMode(i) { + s.hostSerial(conn, i) + return + } } ex, err := s.env.Docker.C.CreateExec(docker.CreateExecOptions{ Container: i.ContainerID, Tty: true, AttachStdin: true, AttachStdout: true, AttachStderr: true, @@ -83,6 +89,39 @@ func (s *Service) terminal(c *httpx.Ctx) (any, error) { return nil, nil } +// hostSerial joins the terminal to the serial console of a guest run on the +// host (its socket is in the guest's run directory, not in the VM container). +func (s *Service) hostSerial(conn *websocket.Conn, i Instance) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + sc, err := s.vmHostDial(ctx, i, vm.HostSerialSocket) + cancel() + if err != nil { + _ = websocket.Message.Send(conn, "\r\n[homecloud] could not open the serial console: "+err.Error()+"\r\n") + return + } + defer sc.Close() + go func() { + defer sc.Close() + for { + var raw string + if err := websocket.Message.Receive(conn, &raw); err != nil { + return + } + var m struct { + T string `json:"t"` + D string `json:"d"` + } + if json.Unmarshal([]byte(raw), &m) == nil && m.T == "i" { + if _, err := sc.Write([]byte(m.D)); err != nil { + return + } + } + } + }() + _, _ = io.Copy(wsWriter{conn}, sc) + _ = websocket.Message.Send(conn, "\r\n[homecloud] session closed\r\n") +} + type wsWriter struct{ c *websocket.Conn } func (w wsWriter) Write(p []byte) (int, error) { diff --git a/cli/internal/svc/ec2/vm.go b/cli/internal/svc/ec2/vm.go index 9af1fd06..f6f2387e 100644 --- a/cli/internal/svc/ec2/vm.go +++ b/cli/internal/svc/ec2/vm.go @@ -34,7 +34,9 @@ import ( // it. The root disk is a qcow2 overlay on the cached base image, kept in the // instance's root volume (an EBS volume record at /dev/xvda, mounted at /vm). // The guest uses hardware acceleration when the Docker host has /dev/kvm and -// is otherwise emulated. +// is otherwise emulated, except on an Apple Silicon Mac with QEMU installed, +// where QEMU runs on the Mac itself with Hypervisor.framework and the VM +// container keeps the network, disks and console (vm_host.go). // // Stage 1 covers the instance lifecycle. Extra volumes, snapshots and images // of VM disks and backup are not available for VM instances yet. The browser @@ -56,12 +58,19 @@ type vmState struct { kvmOnce sync.Once kvm bool + accelOnce sync.Once + accel string // vmAccel + qemu string // QEMU binary for host-run guests, once probed + watchOnce sync.Once + spawnMu sync.Mutex + spawned map[string]time.Time // VM container -> when its host runner was started + fetchMu sync.Mutex // one download at a time } // vmPlan is what Launch works out once for every instance of a VM launch. type vmPlan struct { - virtualization string // kvm | emulated + virtualization string // hvf | kvm | emulated user string diskGB int root VolumeSpec @@ -131,18 +140,15 @@ func (s *Service) planVM(in RunInput, img Image) (vmPlan, error) { } p.ami = img.VMDisk p.root.SizeGB = p.diskGB - p.virtualization = "emulated" - if s.vmKVM() { - p.virtualization = "kvm" - } + p.virtualization = s.vmAccel() return p, nil } // vmKVM reports whether the Docker host offers /dev/kvm. It is probed once by -// creating a container with the device (HC_VM_ACCEL=kvm|tcg overrides). +// creating a container with the device (HC_VM_ACCEL or vm_accel = kvm|tcg overrides). func (s *Service) vmKVM() bool { s.vm.kvmOnce.Do(func() { - switch os.Getenv("HC_VM_ACCEL") { + switch s.vmAccelSetting() { case "kvm": s.vm.kvm = true return @@ -316,6 +322,16 @@ func (s *Service) vmRunSpec(inst Instance, network string) runtime.RunSpec { if mach.KVM { spec.Devices = []string{"/dev/kvm"} } + if vmHostMode(inst) { + // QEMU runs on the host (vm_host.go): the container only prepares the + // disks and serves them and passt, and lives as long as the guest. + spec.Cmd = nil + spec.Env["HC_VM_HOST"] = "1" + spec.Labels[labelVMHost] = "1" + spec.NanoCPUs = int64(min(2, s.hostCPU) * 1e9) + spec.MemoryMB = 768 + s.vmHostWatch() + } if inst.VMAMI != "" { spec.Env["HC_VM_AMI"] = "1" } @@ -459,6 +475,12 @@ func (s *Service) launchVM(inst Instance, network string) { s.metadataRoute(ctx, cid, inst.VpcID) s.vpc.ProtectNow(ctx, s.member(inst, cid)) err = s.vmWaitReady(inst, since) + if err != nil && vmHostMode(inst) { + // passt did not start, which a guest on the host cannot do without: run it in the container. + if why := s.vmPasstFailure(cid, since); why != "" { + err = fmt.Errorf("%w: %s", errPasstExited, why) + } + } if errors.Is(err, errPasstExited) { // passt died while the guest booted: boot it again in user-mode networking. log.Printf("ec2: %s: %v; restarting the guest with user-mode networking", inst.ID, err) @@ -505,8 +527,17 @@ var errPasstExited = errors.New("passt exited") // vmFallbackToUser records that passt does not work for an instance and // rebuilds its container with QEMU's user-mode networking (the guest boots // again; it sits behind NAT, and security group changes rebuild it). +// +// A guest run on the host (hvf) cannot use user-mode networking with the +// instance's address: it moves into the VM container, emulated. func (s *Service) vmFallbackToUser(id string) error { - if _, err := store.Update(s.env.Store, cInstances, id, func(x *Instance) error { x.VMNetwork = "user"; return nil }); err != nil { + if _, err := store.Update(s.env.Store, cInstances, id, func(x *Instance) error { + x.VMNetwork = "user" + if x.Virtualization == virtHVF { + x.Virtualization = "emulated" + } + return nil + }); err != nil { return err } return s.recreate(id) @@ -637,7 +668,7 @@ func tail(s string, n int) string { // vmBootTimeout is how long a guest gets to finish booting before the instance // is reported running anyway (images without cloud-init never print the marker). func vmBootTimeout(inst Instance) time.Duration { - if inst.Virtualization == "kvm" { + if inst.Virtualization == "kvm" || inst.Virtualization == virtHVF { return 10 * time.Minute } return 30 * time.Minute @@ -683,7 +714,7 @@ func (s *Service) vmWaitReady(inst Instance, since time.Time) error { // vmShutdownTimeout is how long the guest gets to power off after the ACPI request. func vmShutdownTimeout(inst Instance) time.Duration { - if inst.Virtualization == "kvm" { + if inst.Virtualization == "kvm" || inst.Virtualization == virtHVF { return 90 * time.Second } return 3 * time.Minute @@ -697,7 +728,11 @@ func (s *Service) vmPowerOff(inst Instance) bool { return true } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - if _, err := s.env.Docker.Exec(ctx, inst.ContainerID, []string{"/usr/local/bin/hc-vm-ctl", "system_powerdown"}, nil); err != nil { + if vmHostMode(inst) { + if err := s.vmQMP(ctx, inst, "system_powerdown"); err != nil { + log.Printf("ec2: power down %s: %v", inst.ID, err) + } + } else if _, err := s.env.Docker.Exec(ctx, inst.ContainerID, []string{"/usr/local/bin/hc-vm-ctl", "system_powerdown"}, nil); err != nil { log.Printf("ec2: power down %s: %v", inst.ID, err) } cancel() diff --git a/cli/internal/svc/ec2/vm/hc-vm-run b/cli/internal/svc/ec2/vm/hc-vm-run index b46ad6f2..bc7a2ee9 100644 --- a/cli/internal/svc/ec2/vm/hc-vm-run +++ b/cli/internal/svc/ec2/vm/hc-vm-run @@ -8,8 +8,9 @@ # # Environment: HC_VM_BASE (file in /images), HC_VM_DISK_GB, HC_VM_ARCH # (aarch64|x86_64), HC_VM_DNS (the resolver address the guest is given), -# HC_VM_NET=user and HC_VM_HOSTFWD (user-mode networking, see below). -# The arguments are the QEMU command line. +# HC_VM_NET=user and HC_VM_HOSTFWD (user-mode networking, see below), +# HC_VM_HOST=1 (QEMU runs on the Docker host's host, see host.go in the vm package). +# The arguments are the QEMU command line (none in host mode). set -eu disk=/vm/disk.qcow2 @@ -76,7 +77,7 @@ fi # namespace (127.0.0.11), so names resolve as they do for container instances. dns="" if [ -n "${HC_VM_DNS:-}" ]; then dns="-D $HC_VM_DNS --dns-forward $HC_VM_DNS --dns-host 127.0.0.11"; fi -rm -f /run/passt.dead /tmp/passt.sock /run/qmp.sock /run/serial.sock /run/qga.sock +rm -f /run/passt.dead /tmp/passt.sock /run/qmp.sock /run/serial.sock /run/qga.sock /run/nbd.sock /run/hc-vm-host.ready /run/hc-vm-host.exit if [ "${HC_VM_NET:-}" != "user" ]; then # -d logs each flow: it is what tells why passt exited. The log is appended # to /run/passt.log and cut back to its last MiB when it passes 4 MiB. @@ -121,6 +122,11 @@ if [ "${HC_VM_NET:-}" != "user" ]; then HC_VM_NET=user fi fi +if [ -n "${HC_VM_HOST:-}" ] && [ "${HC_VM_NET:-}" = "user" ]; then + # QEMU on the host has no way into this network namespace but passt. + echo "[homecloud] passt did not start, and a guest run on the host needs it" >&2 + exit 1 +fi if [ "${HC_VM_NET:-}" = "user" ]; then # QEMU's user-mode networking: the guest sits behind NAT at 10.0.2.15 and # only the ports HC_VM_HOSTFWD lists (",hostfwd=tcp::22-:22,...", the ports @@ -135,6 +141,38 @@ if [ "${HC_VM_NET:-}" = "user" ]; then done fi +if [ -n "${HC_VM_HOST:-}" ]; then + # Host mode: QEMU runs on the host and reaches the disks through + # qemu-storage-daemon's NBD server and the network through passt's socket + # (both over docker exec, see host.go). This container lives as long as QEMU: + # the host runner writes its exit status to /run/hc-vm-host.exit. + # (Options apply in order: the NBD server comes before its exports.) + set -- \ + --nbd-server addr.type=unix,addr.path=/run/nbd.sock \ + --blockdev driver=file,node-name=root-file,filename=/vm/disk.qcow2 \ + --blockdev driver=qcow2,node-name=root,file=root-file \ + --blockdev driver=file,node-name=seed,filename=/vm/seed.iso,read-only=on \ + --export type=nbd,id=x-root,node-name=root,name=root,writable=on \ + --export type=nbd,id=x-seed,node-name=seed,name=seed + if [ "$HC_VM_ARCH" = "aarch64" ]; then + set -- "$@" \ + --blockdev driver=file,node-name=fw-vars,filename=/vm/efivars.fd \ + --export type=nbd,id=x-fw-vars,node-name=fw-vars,name=fw-vars,writable=on + fi + for d in ${HC_VM_DISKS:-}; do + id=${d%%:*} + set -- "$@" \ + --blockdev "driver=file,node-name=d-$id,filename=/disks/$id/disk.img" \ + --export "type=nbd,id=x-$id,node-name=d-$id,name=$id,writable=on" + done + qemu-storage-daemon "$@" & + n=0 + while [ ! -S /run/nbd.sock ] && [ $n -lt 100 ]; do n=$((n+1)); sleep 0.1; done + [ -S /run/nbd.sock ] || { echo "[homecloud] qemu-storage-daemon did not start" >&2; exit 1; } + touch /run/hc-vm-host.ready + set -- /bin/sh -c 'while [ ! -f /run/hc-vm-host.exit ]; do sleep 0.2; done; exit "$(cat /run/hc-vm-host.exit)"' +fi + echo "[homecloud] starting the virtual machine" # QEMU runs as a child so that this script can watch passt: if passt exits, the # guest has lost its network (passt also exits with QEMU, which is fine). diff --git a/cli/internal/svc/ec2/vm/host.go b/cli/internal/svc/ec2/vm/host.go new file mode 100644 index 00000000..a981e377 --- /dev/null +++ b/cli/internal/svc/ec2/vm/host.go @@ -0,0 +1,141 @@ +package vm + +import ( + "fmt" + "path/filepath" + "strconv" +) + +// Host-run guests (macOS). +// +// Docker Desktop and OrbStack run containers in a Linux VM without nested +// virtualization, so QEMU in the VM container could only emulate the CPU. On +// an Apple Silicon Mac HomeCloud can instead run QEMU on the host itself with +// Hypervisor.framework (-accel hvf). The VM container stays: it still holds +// the instance's private address, security groups, published ports, passt, the +// disks (Docker volumes) and the console log. In host mode its entrypoint +// prepares the disks and the seed as usual, starts passt and serves the disks +// with qemu-storage-daemon (NBD) on Unix sockets, then waits. A host runner +// (HomeCloud on the Mac) starts QEMU and joins the guest's network card and +// disks to those sockets through `docker exec` streams, so nothing listens on a +// network port and only a Docker API client (root-equivalent anyway) can reach +// them. The guest's serial console is copied into the container's log, and the +// container exits when QEMU does, so stop, start, reboot, console output and +// state reconciliation work as for a guest inside the container. + +// Paths in a VM container in host mode. +const ( + // NBDSocket is where qemu-storage-daemon serves the guest's disks. + NBDSocket = "/run/nbd.sock" + // PasstSocket is passt's socket for the guest's network card. + PasstSocket = "/tmp/passt.sock" + // HostReadyFile appears once the disks and passt are ready for QEMU. + HostReadyFile = "/run/hc-vm-host.ready" + // HostExitFile is where the host runner writes QEMU's exit status, which + // ends the container. + HostExitFile = "/run/hc-vm-host.exit" +) + +// NBD export names. Extra volumes are exported under their volume ID. +const ( + ExportRoot = "root" + ExportSeed = "seed" + ExportFWVars = "fw-vars" +) + +// HostFirmware is the UEFI firmware of host-run aarch64 guests, relative to +// QEMU's installation prefix (Homebrew's qemu ships it). Debian's AAVMF build, +// which guests in the container boot, hangs before its boot manager under +// Hypervisor.framework; both keep their variables in the same format, so a +// guest's variable store (on its root volume) works with either. +const HostFirmware = "share/qemu/edk2-aarch64-code.fd" + +// Files of a host-run guest in its run directory on the host. +const ( + HostNetSocket = "net.sock" // QEMU's network card (relayed to passt) + HostNBDSocket = "nbd.sock" // QEMU's disks (relayed to qemu-storage-daemon) + HostSerialSocket = "serial.sock" // the guest's serial console + HostQGASocket = "qga.sock" // qemu-guest-agent + HostQMPSocket = "qmp.sock" // QMP (power controls) + HostConsoleLog = "console.log" // everything the guest printed on the serial console +) + +// HostArgs is the QEMU command line (without the binary) for a guest run on +// the host with Hypervisor.framework. dir is the guest's run directory on the +// host: QEMU connects to the network and disk relays there and serves the +// serial console, guest agent and QMP sockets there. firmware is the UEFI +// firmware file on the host (aarch64; see HostFirmware). +// +// The devices match Args, in the same order and at the same addresses, so the +// guest sees the same hardware whichever way it runs (its network +// configuration names the network card). +func (m Machine) HostArgs(dir, firmware string) []string { + vcpus := max(1, m.VCPUs) + p := func(name string) string { return filepath.Join(dir, name) } + nbd := func(node, export string, readOnly bool) string { + s := fmt.Sprintf("driver=nbd,node-name=%s,server.type=unix,server.path=%s,export=%s", node, p(HostNBDSocket), export) + if readOnly { + s += ",read-only=on" + } + return s + } + args := []string{ + "-name", m.Name, + "-nodefaults", "-no-user-config", + } + machine := "q35" + if m.Arch == ArchAArch64 { + // The firmware is QEMU's own on the host; its variables live on the root volume. + args = append(args, + "-blockdev", "driver=file,node-name=fw-code,read-only=on,filename="+firmware, + "-blockdev", nbd("fw-vars", ExportFWVars, false), + ) + machine = "virt,gic-version=max,pflash0=fw-code,pflash1=fw-vars" + } + args = append(args, + "-machine", machine, + "-accel", "hvf", "-cpu", "host", + "-smp", strconv.Itoa(vcpus), "-m", strconv.FormatInt(m.MemoryMB, 10), + "-netdev", "stream,id=net0,server=off,addr.type=unix,addr.path="+p(HostNetSocket), + "-device", "virtio-net-pci,netdev=net0,mac="+m.MAC()+",romfile=,addr=0x3", + "-blockdev", nbd("root", ExportRoot, false), + "-device", "virtio-blk-pci,drive=root,bootindex=1", + ) + for i, d := range m.Disks { + node := fmt.Sprintf("data%d", i) + args = append(args, + "-blockdev", nbd(node, d.ID, false), + "-device", fmt.Sprintf("virtio-blk-pci,drive=%s,serial=%s", node, d.Serial()), + ) + } + args = append(args, + "-blockdev", nbd("seed", ExportSeed, true), + "-device", "virtio-blk-pci,drive=seed", + "-device", "virtio-rng-pci", + "-display", "none", "-monitor", "none", + "-chardev", "socket,id=ser0,path="+p(HostSerialSocket)+",server=on,wait=off,logfile="+p(HostConsoleLog)+",logappend=on", + "-serial", "chardev:ser0", + "-chardev", "socket,id=qga0,path="+p(HostQGASocket)+",server=on,wait=off", + "-device", "virtio-serial-pci,id=vser0", + "-device", "virtserialport,bus=vser0.0,chardev=qga0,name="+QGAChannel, + "-qmp", "unix:"+p(HostQMPSocket)+",server=on,wait=off", + ) + return args +} + +// RelayCommand is what a `docker exec` runs in the VM container to join its +// standard input and output to a Unix socket there (passt's or the NBD server's). +func RelayCommand(socket string) []string { + return []string{"socat", "-t", "5", "-", "UNIX-CONNECT:" + socket} +} + +// ConsoleCommand is what a `docker exec` runs in the VM container to copy its +// standard input (the guest's console) to the container's log. +func ConsoleCommand() []string { + return []string{"/bin/sh", "-c", "exec cat >/proc/1/fd/1"} +} + +// HostExitCommand records QEMU's exit status in the VM container, which then exits with it. +func HostExitCommand(status int) []string { + return []string{"/bin/sh", "-c", "echo " + strconv.Itoa(status) + " >" + HostExitFile} +} diff --git a/cli/internal/svc/ec2/vm/host_test.go b/cli/internal/svc/ec2/vm/host_test.go new file mode 100644 index 00000000..9794408c --- /dev/null +++ b/cli/internal/svc/ec2/vm/host_test.go @@ -0,0 +1,72 @@ +package vm + +import ( + "strings" + "testing" +) + +func TestQEMUHostHVF(t *testing.T) { + m := Machine{Name: "i-0abc", Arch: ArchAArch64, VCPUs: 2, MemoryMB: 2048, Disks: []Disk{{ID: "vol-0123456789abcdef01234"}}} + dir := "/Users/me/.homecloud/guests/i-0abc" + s := joined(m.HostArgs(dir, "/opt/homebrew/share/qemu/edk2-aarch64-code.fd")) + nbd := "server.type=unix,server.path=" + dir + "/nbd.sock" + for _, want := range []string{ + "-machine virt,gic-version=max,pflash0=fw-code,pflash1=fw-vars", + "-accel hvf -cpu host", + "-smp 2 -m 2048", + "driver=file,node-name=fw-code,read-only=on,filename=/opt/homebrew/share/qemu/edk2-aarch64-code.fd", + "driver=nbd,node-name=fw-vars," + nbd + ",export=fw-vars", + "driver=nbd,node-name=root," + nbd + ",export=root", + "-device virtio-blk-pci,drive=root,bootindex=1", + "driver=nbd,node-name=data0," + nbd + ",export=vol-0123456789abcdef01234", + "-device virtio-blk-pci,drive=data0,serial=vol-0123456789abcdef", + "driver=nbd,node-name=seed," + nbd + ",export=seed,read-only=on", + "-netdev stream,id=net0,server=off,addr.type=unix,addr.path=" + dir + "/net.sock", + "virtio-net-pci,netdev=net0,mac=" + m.MAC() + ",romfile=,addr=0x3", + "-chardev socket,id=ser0,path=" + dir + "/serial.sock,server=on,wait=off,logfile=" + dir + "/console.log,logappend=on -serial chardev:ser0", + "-chardev socket,id=qga0,path=" + dir + "/qga.sock,server=on,wait=off", + "-device virtserialport,bus=vser0.0,chardev=qga0,name=org.qemu.guest_agent.0", + "-qmp unix:" + dir + "/qmp.sock,server=on,wait=off", + "-nodefaults", + } { + if !strings.Contains(s, want) { + t.Errorf("command line lacks %q:\n%s", want, s) + } + } + for _, bad := range []string{"-sandbox", "kvm", "tcg", "/vm/", "/run/", "/tmp/"} { + if strings.Contains(s, bad) { + t.Errorf("host command line has %q:\n%s", bad, s) + } + } + // The read-only exports must stay read-only, the writable ones writable. + if strings.Contains(s, "export=root,read-only") || strings.Contains(s, "export=fw-vars,read-only") { + t.Errorf("a writable disk is read-only:\n%s", s) + } +} + +// The guest sees the same devices in the same order whichever way it runs: +// its network configuration names the card, and disks keep their names. +func TestQEMUHostDevicesMatchContainer(t *testing.T) { + m := Machine{Name: "i-1", Arch: ArchAArch64, VCPUs: 1, MemoryMB: 1024, Disks: []Disk{{ID: "vol-a"}, {ID: "vol-b"}}} + devices := func(args []string) []string { + var out []string + for i := 0; i+1 < len(args); i++ { + if args[i] == "-device" { + out = append(out, args[i+1]) + } + } + return out + } + if c, h := devices(m.Args()), devices(m.HostArgs("/d", "/fw")); joined(c) != joined(h) { + t.Errorf("devices differ:\ncontainer %v\nhost %v", c, h) + } +} + +func TestHostHelperCommands(t *testing.T) { + if got := joined(RelayCommand(NBDSocket)); got != "socat -t 5 - UNIX-CONNECT:/run/nbd.sock" { + t.Errorf("relay %q", got) + } + if got := joined(HostExitCommand(3)); got != "/bin/sh -c echo 3 >/run/hc-vm-host.exit" { + t.Errorf("exit %q", got) + } +} diff --git a/cli/internal/svc/ec2/vm_access.go b/cli/internal/svc/ec2/vm_access.go index a1cce466..7bb978d6 100644 --- a/cli/internal/svc/ec2/vm_access.go +++ b/cli/internal/svc/ec2/vm_access.go @@ -66,7 +66,8 @@ func readLine(br *bufio.Reader, max int) ([]byte, error) { // qgaCall sends one command to the guest agent and returns its reply. ctx // bounds the whole call, including waiting for the agent and for other callers. -func (s *Service) qgaCall(ctx context.Context, cid string, req []byte) (vm.QGAReply, error) { +func (s *Service) qgaCall(ctx context.Context, inst Instance, req []byte) (vm.QGAReply, error) { + cid := inst.ContainerID slot := qgaSlot(cid) select { case slot <- struct{}{}: @@ -76,20 +77,35 @@ func (s *Service) qgaCall(ctx context.Context, cid string, req []byte) (vm.QGARe } ectx, cancel := context.WithCancel(ctx) defer cancel() - ex, err := s.env.Docker.C.CreateExec(docker.CreateExecOptions{ - Container: cid, Cmd: vm.QGAAttachCommand(), AttachStdin: true, AttachStdout: true, AttachStderr: true, Context: ectx, - }) - if err != nil { - return vm.QGAReply{}, err - } inR, inW := io.Pipe() outR, outW := io.Pipe() done := make(chan struct{}) - go func() { - defer close(done) - defer outW.Close() - _ = s.env.Docker.C.StartExec(ex.ID, docker.StartExecOptions{InputStream: inR, OutputStream: outW, ErrorStream: io.Discard, Context: ectx}) - }() + if vmHostMode(inst) { + // QEMU runs on the host: its agent socket is in the guest's run directory. + c, err := s.vmHostDial(ectx, inst, vm.HostQGASocket) + if err != nil { + return vm.QGAReply{}, errGuestAgent + } + go func() { <-ectx.Done(); c.Close() }() + go func() { _, _ = io.Copy(c, inR) }() + go func() { + defer close(done) + defer outW.Close() + _, _ = io.Copy(outW, c) + }() + } else { + ex, err := s.env.Docker.C.CreateExec(docker.CreateExecOptions{ + Container: cid, Cmd: vm.QGAAttachCommand(), AttachStdin: true, AttachStdout: true, AttachStderr: true, Context: ectx, + }) + if err != nil { + return vm.QGAReply{}, err + } + go func() { + defer close(done) + defer outW.Close() + _ = s.env.Docker.C.StartExec(ex.ID, docker.StartExecOptions{InputStream: inR, OutputStream: outW, ErrorStream: io.Discard, Context: ectx}) + }() + } defer func() { inW.Close() cancel() @@ -176,7 +192,7 @@ func (s *Service) guestExec(ctx context.Context, inst Instance, command string, call := func(req []byte) (vm.QGAReply, error) { cctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() - return s.qgaCall(cctx, inst.ContainerID, req) + return s.qgaCall(cctx, inst, req) } r, err := call(vm.QGAExecRequest(path, args)) if err != nil { diff --git a/cli/internal/svc/ec2/vm_host.go b/cli/internal/svc/ec2/vm_host.go new file mode 100644 index 00000000..97f1317c --- /dev/null +++ b/cli/internal/svc/ec2/vm_host.go @@ -0,0 +1,622 @@ +package ec2 + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "os/exec" + "path/filepath" + goruntime "runtime" + "strings" + "sync" + "time" + + docker "github.com/fsouza/go-dockerclient" + "github.com/homecloudhq/homecloud/cli/internal/core" + "github.com/homecloudhq/homecloud/cli/internal/runtime" + "github.com/homecloudhq/homecloud/cli/internal/store" + "github.com/homecloudhq/homecloud/cli/internal/svc/ec2/vm" +) + +// Host-run VM guests: QEMU with Hypervisor.framework on a Mac (see host.go in +// the vm package for the design). This file chooses the backend, runs the +// host side of a guest (the "host runner") and gives the rest of EC2 the +// guest's console, agent and QMP sockets. + +// labelVMHost marks a VM container whose guest runs on the host. +const labelVMHost = "homecloud.vm-host" + +// virtHVF is Instance.Virtualization for a guest run on the host with Hypervisor.framework. +const virtHVF = "hvf" + +// VMHostCommand, when set, returns the command that runs the host runner of a +// guest in its own process (`homecloud vm-host DIR`), so that guests outlive a +// restart of the server. Without it (tests) the runner is a goroutine. +var VMHostCommand func(dir string) *exec.Cmd + +// vmAccel is how VM instances launched now run: "hvf" (QEMU on this Mac with +// Hypervisor.framework), "kvm" (QEMU in the VM container with /dev/kvm) or +// "emulated" (QEMU in the VM container, TCG). HC_VM_ACCEL (or the vm_accel +// setting) =hvf|kvm|tcg forces one; =container skips the host backend but +// still probes for KVM. +func (s *Service) vmAccel() string { + s.vm.accelOnce.Do(func() { + switch s.vmAccelSetting() { + case "hvf": + s.vm.accel = virtHVF + if why := vmHostUnsupported(hostArch(s)); why != "" { + log.Printf("ec2: VM acceleration hvf was chosen, but %s: VM launches will fail", why) + } + return + case "tcg", "kvm", "container": + default: + // (HC_VM_NET=user is user-mode networking in the container, which a guest on the host cannot have.) + if goruntime.GOOS == "darwin" && os.Getenv("HC_VM_NET") != "user" { + if why := s.vmHostProbe(); why != "" { + log.Printf("ec2: VM instances cannot run on this Mac with Hypervisor.framework (%s); they run in the Docker VM instead", why) + } else { + log.Printf("ec2: VM instances run on this Mac with Hypervisor.framework (%s)", s.vm.qemu) + s.vm.accel = virtHVF + return + } + } + } + s.vm.accel = "emulated" + if s.vmKVM() { + s.vm.accel = "kvm" + } + }) + return s.vm.accel +} + +// vmAccelSetting is the configured VM backend: HC_VM_ACCEL, else the +// server's vm_accel setting ("" or "auto" chooses). +func (s *Service) vmAccelSetting() string { + if v := os.Getenv("HC_VM_ACCEL"); v != "" { + return v + } + return s.env.Cfg.VMAccel +} + +// vmQEMUBinary finds qemu-system- for host-run guests: HC_VM_QEMU, the +// PATH, then Homebrew's prefixes (a server started by launchd has a short PATH). +func vmQEMUBinary(arch vm.Arch) string { + if p := os.Getenv("HC_VM_QEMU"); p != "" { + return p + } + if p, err := exec.LookPath(arch.QEMU()); err == nil { + return p + } + for _, dir := range []string{"/opt/homebrew/bin", "/usr/local/bin"} { + p := filepath.Join(dir, arch.QEMU()) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p + } + } + return "" +} + +// vmHostFirmware finds the UEFI firmware for host-run aarch64 guests: +// HC_VM_FIRMWARE, or the one installed with QEMU (Homebrew's qemu has it in +// /share/qemu). +func vmHostFirmware(qemu string) string { + if p := os.Getenv("HC_VM_FIRMWARE"); p != "" { + return p + } + if qemu == "" { + return "" + } + if real, err := filepath.EvalSymlinks(qemu); err == nil { + qemu = real + } + p := filepath.Join(filepath.Dir(filepath.Dir(qemu)), vm.HostFirmware) + if st, err := os.Stat(p); err != nil || st.IsDir() || strings.Contains(p, ",") { + return "" // (a comma would end QEMU's option) + } + return p +} + +// vmHostUnsupported says why guests cannot run on this host with +// Hypervisor.framework, or "" when nothing rules it out before trying. +func vmHostUnsupported(dockerArch string) string { + switch { + case goruntime.GOOS != "darwin": + return "it needs macOS" + case goruntime.GOARCH != "arm64": + return "it needs an Apple Silicon Mac" + case runtime.InContainer(): + return "HomeCloud runs in a container" + case dockerArch != "arm64": + return "the Docker host is not arm64" + case vmQEMUBinary(vm.ArchAArch64) == "": + return "qemu-system-aarch64 was not found (brew install qemu)" + case vmHostFirmware(vmQEMUBinary(vm.ArchAArch64)) == "": + return "QEMU's UEFI firmware (" + vm.HostFirmware + ") was not found next to qemu-system-aarch64; set HC_VM_FIRMWARE" + } + return "" +} + +// vmHostProbe checks that QEMU starts with Hypervisor.framework here: it +// boots an empty machine, paused, and quits it through QMP. +func (s *Service) vmHostProbe() string { + if why := vmHostUnsupported(hostArch(s)); why != "" { + return why + } + qemu := vmQEMUBinary(vm.ArchAArch64) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, qemu, "-machine", "virt", "-accel", "hvf", "-cpu", "host", "-m", "64", + "-nodefaults", "-no-user-config", "-display", "none", "-S", "-qmp", "stdio") + cmd.Stdin = strings.NewReader(`{"execute":"qmp_capabilities"}` + "\n" + `{"execute":"quit"}` + "\n") + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + if err := cmd.Run(); err != nil || !strings.Contains(out.String(), `"QMP"`) { + return fmt.Sprintf("%s -accel hvf failed: %v %s", qemu, err, strings.TrimSpace(errb.String())) + } + s.vm.qemu = qemu + return "" +} + +// vmHostBase is the directory holding the run directories of host-run guests. +// Unix socket paths are short (104 bytes on macOS): a data directory with a +// long path uses the temporary directory instead. +func (s *Service) vmHostBase() string { + base := filepath.Join(s.env.Cfg.DataDir, "vm") + if s.env.Cfg.DataDir == "" || len(base) > 64 { + base = filepath.Join(os.TempDir(), fmt.Sprintf("homecloud-vm-%d", os.Getuid())) + } + return base +} + +// vmHostDir is the run directory of a host-run guest. +func (s *Service) vmHostDir(id string) string { return filepath.Join(s.vmHostBase(), id) } + +// vmHostConfig is what the host runner of a guest reads from its run directory. +type vmHostConfig struct { + Instance string `json:"instance"` + Container string `json:"container"` + QEMU string `json:"qemu"` + Args []string `json:"args"` +} + +const vmHostConfigFile = "config.json" + +// vmHostWatch starts, once, the loop that gives every running host-mode VM +// container its host runner, whoever started the container (launch, start, +// reboot, rebuild, or a container left running across a server restart). +func (s *Service) vmHostWatch() { + if s.env.Docker == nil { + return + } + s.vm.watchOnce.Do(func() { + go func() { + defer core.Recover("ec2 vm host watch") + for { + s.vmHostScan() + time.Sleep(time.Second) + } + }() + }) +} + +// vmHostScan starts a host runner for each running host-mode VM container of +// this installation that has none. +func (s *Service) vmHostScan() { + cs, err := s.env.Docker.C.ListContainers(docker.ListContainersOptions{Filters: map[string][]string{ + "label": {labelVMHost + "=1", core.LabelAccount + "=" + runtime.Account}, + "status": {"running"}, + }}) + if err != nil { + return + } + for _, c := range cs { + inst, err := store.Get[Instance](s.env.Store, cInstances, c.Labels[core.LabelResource]) + if err != nil || inst.State == "terminated" { + continue + } + if err := s.vmHostEnsure(inst, c.ID); err != nil { + log.Printf("ec2: %s: starting the host side of the guest: %v", inst.ID, err) + } + } +} + +// vmHostEnsure starts the host runner of a guest unless one is running. +func (s *Service) vmHostEnsure(inst Instance, cid string) error { + dir := s.vmHostDir(inst.ID) + s.vm.spawnMu.Lock() + defer s.vm.spawnMu.Unlock() + if t, ok := s.vm.spawned[cid]; ok && time.Since(t) < 5*time.Second { + return nil // just started: it may not hold its lock yet + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := os.Chmod(dir, 0o700); err != nil { + return err + } + if vmHostRunning(dir) { + return nil + } + qemu := s.vm.qemu + if qemu == "" { + qemu = vmQEMUBinary(s.vmMachine(inst).Arch) + } + cfg := vmHostConfig{Instance: inst.ID, Container: cid, QEMU: qemu, Args: s.vmMachine(inst).HostArgs(dir, vmHostFirmware(qemu))} + b, _ := json.MarshalIndent(cfg, "", " ") + if err := os.WriteFile(filepath.Join(dir, vmHostConfigFile), b, 0o600); err != nil { + return err + } + if s.vm.spawned == nil { + s.vm.spawned = map[string]time.Time{} + } + for k, t := range s.vm.spawned { + if time.Since(t) > time.Minute { + delete(s.vm.spawned, k) + } + } + s.vm.spawned[cid] = time.Now() + if VMHostCommand == nil { + go func() { + defer core.Recover("ec2 vm host " + inst.ID) + if err := RunVMHost(context.Background(), s.env.Docker, dir); err != nil && !errors.Is(err, errVMHostBusy) { + log.Printf("ec2: %s: host runner: %v", inst.ID, err) + } + }() + return nil + } + cmd := VMHostCommand(dir) + lf, err := os.OpenFile(filepath.Join(dir, "runner.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + defer lf.Close() + cmd.Stdout, cmd.Stderr = lf, lf + detach(cmd) + if err := cmd.Start(); err != nil { + return err + } + go func() { _ = cmd.Wait() }() + return nil +} + +var errVMHostBusy = errors.New("another host runner serves this guest") + +// RunVMHost is the host runner of one guest (dir is its run directory, with +// config.json): it waits for the VM container to have the disks and passt +// ready, starts QEMU, relays its network card and disks into the container, +// copies the serial console into the container's log, and ends the container +// with QEMU's exit status. QEMU is killed when the container stops. +func RunVMHost(ctx context.Context, d *runtime.Docker, dir string) error { + b, err := os.ReadFile(filepath.Join(dir, vmHostConfigFile)) + if err != nil { + return err + } + var cfg vmHostConfig + if err := json.Unmarshal(b, &cfg); err != nil { + return err + } + unlock, ok, err := lockDir(dir) + if err != nil { + return err + } + if !ok { + return errVMHostBusy + } + defer unlock() + h := &vmHost{d: d, cfg: cfg, dir: dir} + return h.run(ctx) +} + +type vmHost struct { + d *runtime.Docker + cfg vmHostConfig + dir string + + consoleMu sync.Mutex + console io.WriteCloser +} + +func (h *vmHost) path(name string) string { return filepath.Join(h.dir, name) } + +func (h *vmHost) logf(format string, args ...any) { + log.Printf("ec2: %s: host runner: %s", h.cfg.Instance, fmt.Sprintf(format, args...)) +} + +// write sends bytes to the container's log. +func (h *vmHost) Write(p []byte) (int, error) { + h.consoleMu.Lock() + defer h.consoleMu.Unlock() + if h.console == nil { + return len(p), nil + } + if _, err := h.console.Write(p); err != nil { + h.console = nil + } + return len(p), nil +} + +func (h *vmHost) running() bool { return h.d.State(h.cfg.Container) == "running" } + +func (h *vmHost) run(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + killStaleQEMU(h.path("qemu.pid")) + + // The container's entrypoint prepares the disks (which can mean copying an + // image) and starts passt and the NBD server. + deadline := time.Now().Add(30 * time.Minute) + for { + if !h.running() { + return fmt.Errorf("the VM container %.12s is not running", h.cfg.Container) + } + ectx, ecancel := context.WithTimeout(ctx, 20*time.Second) + res, err := h.d.Exec(ectx, h.cfg.Container, []string{"test", "-f", vm.HostReadyFile}, nil) + ecancel() + if err == nil && res.ExitCode == 0 { + break + } + if time.Now().After(deadline) || ctx.Err() != nil { + return errors.New("the VM container did not get ready") + } + time.Sleep(200 * time.Millisecond) + } + + // The container's log gets the guest's console (and QEMU's own messages). + pr, pw := io.Pipe() + h.console = pw + consoleDone := make(chan struct{}) + go func() { + defer close(consoleDone) + h.execStream(ctx, vm.ConsoleCommand(), pr, io.Discard) + _ = pr.Close() + }() + + for _, f := range []string{vm.HostNetSocket, vm.HostNBDSocket, vm.HostSerialSocket, vm.HostQGASocket, vm.HostQMPSocket, vm.HostConsoleLog} { + _ = os.Remove(h.path(f)) + } + var wg sync.WaitGroup + var listeners []net.Listener + closeListeners := func() { + for _, ln := range listeners { + _ = ln.Close() + } + } + defer closeListeners() + for _, r := range []struct{ name, target string }{{vm.HostNetSocket, vm.PasstSocket}, {vm.HostNBDSocket, vm.NBDSocket}} { + ln, err := net.Listen("unix", h.path(r.name)) + if err != nil { + fmt.Fprintf(h, "[homecloud] could not start QEMU on the host: %v\n", err) + h.finish(ctx, 1, consoleDone) + return err + } + listeners = append(listeners, ln) + wg.Add(1) + go func(ln net.Listener, target string) { + defer wg.Done() + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + h.execStream(ctx, vm.RelayCommand(target), c, c) + }() + } + }(ln, r.target) + } + + cmd := exec.Command(h.cfg.QEMU, h.cfg.Args...) + cmd.Dir = h.dir + cmd.Stdout, cmd.Stderr = h, h + start := time.Now() + if err := cmd.Start(); err != nil { + fmt.Fprintf(h, "[homecloud] could not start QEMU on the host: %v\n", err) + h.finish(ctx, 1, consoleDone) + return err + } + _ = os.WriteFile(h.path("qemu.pid"), []byte(fmt.Sprint(cmd.Process.Pid)), 0o600) + h.logf("QEMU started (pid %d)", cmd.Process.Pid) + exited := make(chan struct{}) + go func() { // QEMU goes when the container goes (stop, kill, terminate) + t := time.NewTicker(2 * time.Second) + defer t.Stop() + for { + select { + case <-exited: + return + case <-t.C: + if !h.running() { + h.logf("the VM container stopped: killing QEMU") + _ = cmd.Process.Kill() + return + } + } + } + }() + tailDone := make(chan struct{}) + go func() { defer close(tailDone); h.tail(exited) }() + err := cmd.Wait() + close(exited) + <-tailDone + _ = os.Remove(h.path("qemu.pid")) + status := 0 + if err != nil { + status = 1 + var ee *exec.ExitError + if errors.As(err, &ee) && ee.ExitCode() > 0 { + status = ee.ExitCode() + } + fmt.Fprintf(h, "[homecloud] QEMU exited: %v\n", err) + } + h.logf("QEMU exited after %s (%v)", time.Since(start).Round(time.Second), err) + h.finish(ctx, status, consoleDone) + cancel() + closeListeners() + wg.Wait() + return nil +} + +// finish records QEMU's exit status in the container (which then exits) and +// closes the console stream. +func (h *vmHost) finish(ctx context.Context, status int, consoleDone <-chan struct{}) { + if h.running() { + ectx, cancel := context.WithTimeout(ctx, 20*time.Second) + if _, err := h.d.Exec(ectx, h.cfg.Container, vm.HostExitCommand(status), nil); err != nil { + h.logf("record the exit status in the container: %v", err) + } + cancel() + // Hold the run directory until the container is down: a container still + // running without a runner would get a new one (and a new QEMU). + for n := 0; n < 100 && h.running(); n++ { + time.Sleep(100 * time.Millisecond) + } + } + h.consoleMu.Lock() + if h.console != nil { + _ = h.console.Close() + h.console = nil + } + h.consoleMu.Unlock() + select { + case <-consoleDone: + case <-time.After(10 * time.Second): + } +} + +// tail copies what QEMU logs of the serial console to the container's log +// until QEMU has exited, then copies the rest. +func (h *vmHost) tail(exited <-chan struct{}) { + var f *os.File + defer func() { + if f != nil { + f.Close() + } + }() + buf := make([]byte, 64<<10) + for { + done := false + select { + case <-exited: + done = true + default: + } + if f == nil { + f, _ = os.Open(h.path(vm.HostConsoleLog)) + } + if f != nil { + for { + n, err := f.Read(buf) + if n > 0 { + _, _ = h.Write(buf[:n]) + } + if err != nil || n == 0 { + break + } + } + } + if done { + return + } + time.Sleep(100 * time.Millisecond) + } +} + +// execStream runs cmd in the VM container with its standard input and output +// joined to in and out, until either side ends. +func (h *vmHost) execStream(ctx context.Context, cmd []string, in io.Reader, out io.Writer) { + ex, err := h.d.C.CreateExec(docker.CreateExecOptions{ + Container: h.cfg.Container, Cmd: cmd, AttachStdin: true, AttachStdout: true, AttachStderr: true, Context: ctx, + }) + if err != nil { + if h.running() { + h.logf("docker exec %v: %v", cmd, err) + } + return + } + var stderr bytes.Buffer + err = h.d.C.StartExec(ex.ID, docker.StartExecOptions{InputStream: in, OutputStream: out, ErrorStream: &stderr, Context: ctx}) + if errors.Is(err, net.ErrClosed) { + err = nil // QEMU closed its end (it exited or let go of a disk) + } + if (err != nil || stderr.Len() > 0) && h.running() && ctx.Err() == nil { + h.logf("docker exec %v: %v %s", cmd, err, strings.TrimSpace(stderr.String())) + } +} + +// vmHostCleanup removes a terminated instance's run directory once its host +// runner (which kills QEMU when the container goes) has exited. +func (s *Service) vmHostCleanup(id string) { + defer core.Recover("ec2 vm host cleanup " + id) + dir := s.vmHostDir(id) + if _, err := os.Stat(dir); err != nil { + return + } + for n := 0; n < 60 && vmHostRunning(dir); n++ { + time.Sleep(time.Second) + } + if err := os.RemoveAll(dir); err != nil { + log.Printf("ec2: %s: remove the guest's run directory: %v", id, err) + } +} + +// ---- the guest's sockets, for the rest of EC2 ---- + +// vmHostDial connects to one of a host-run guest's sockets (vm.HostQGASocket, ...). +func (s *Service) vmHostDial(ctx context.Context, inst Instance, name string) (net.Conn, error) { + var dl net.Dialer + return dl.DialContext(ctx, "unix", filepath.Join(s.vmHostDir(inst.ID), name)) +} + +// vmQMP sends one QMP command to a host-run guest's QEMU. +func (s *Service) vmQMP(ctx context.Context, inst Instance, command string) error { + c, err := s.vmHostDial(ctx, inst, vm.HostQMPSocket) + if err != nil { + return err + } + defer c.Close() + if dl, ok := ctx.Deadline(); ok { + _ = c.SetDeadline(dl) + } + br := bufio.NewReader(c) + if _, err := br.ReadBytes('\n'); err != nil { // the greeting + return err + } + for _, cmd := range []string{"qmp_capabilities", command} { + if _, err := fmt.Fprintf(c, `{"execute":%q}`+"\n", cmd); err != nil { + return err + } + for { // skip events up to the reply + line, err := br.ReadBytes('\n') + if err != nil { + return err + } + var r struct { + Return json.RawMessage `json:"return"` + Error *struct { + Desc string `json:"desc"` + } `json:"error"` + } + if json.Unmarshal(line, &r) != nil { + continue + } + if r.Error != nil { + return errors.New(r.Error.Desc) + } + if r.Return != nil { + break + } + } + } + return nil +} + +// vmHostMode reports whether an instance's guest runs on the host. +func vmHostMode(i Instance) bool { return i.Virtualization == virtHVF } diff --git a/cli/internal/svc/ec2/vm_host_unix.go b/cli/internal/svc/ec2/vm_host_unix.go new file mode 100644 index 00000000..b0e654b1 --- /dev/null +++ b/cli/internal/svc/ec2/vm_host_unix.go @@ -0,0 +1,72 @@ +//go:build !windows + +package ec2 + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" +) + +// lockDir takes the host runner's lock on a guest's run directory. ok is +// false when another runner holds it. +func lockDir(dir string) (unlock func(), ok bool, err error) { + f, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, false, err + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, false, nil + } + return nil, false, err + } + return func() { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN); f.Close() }, true, nil +} + +// vmHostRunning reports whether a host runner holds a guest's run directory. +func vmHostRunning(dir string) bool { + unlock, ok, err := lockDir(dir) + if err != nil { + return false + } + if ok { + unlock() + return false + } + return true +} + +// detach makes a host runner its own session, so it outlives the server and +// a Ctrl-C in the server's terminal. +func detach(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} +} + +// killStaleQEMU kills the QEMU a host runner that died left behind (its +// network and disks went with the runner). +func killStaleQEMU(pidFile string) { + b, err := os.ReadFile(pidFile) + if err != nil { + return + } + defer os.Remove(pidFile) + pid, err := strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil || pid <= 1 { + return + } + out, err := exec.Command("ps", "-o", "comm=", "-p", strconv.Itoa(pid)).Output() + if err != nil || !strings.Contains(string(out), "qemu-system") { + return // gone, or the number was reused + } + _ = syscall.Kill(pid, syscall.SIGKILL) + for n := 0; n < 50 && syscall.Kill(pid, 0) == nil; n++ { + time.Sleep(100 * time.Millisecond) + } +} diff --git a/cli/internal/svc/ec2/vm_host_windows.go b/cli/internal/svc/ec2/vm_host_windows.go new file mode 100644 index 00000000..6074f93d --- /dev/null +++ b/cli/internal/svc/ec2/vm_host_windows.go @@ -0,0 +1,18 @@ +package ec2 + +import ( + "errors" + "os/exec" +) + +// Guests never run on a Windows host (vmHostUnsupported). + +func lockDir(string) (func(), bool, error) { + return nil, false, errors.New("host-run guests need macOS") +} + +func vmHostRunning(string) bool { return false } + +func detach(*exec.Cmd) {} + +func killStaleQEMU(string) {} diff --git a/cli/internal/svc/ec2/vm_live_test.go b/cli/internal/svc/ec2/vm_live_test.go index f6d57b3d..4bfeed2b 100644 --- a/cli/internal/svc/ec2/vm_live_test.go +++ b/cli/internal/svc/ec2/vm_live_test.go @@ -272,7 +272,7 @@ func TestVMInstanceLifecycle(t *testing.T) { }) i := f.vmWaitState(t, id, "running") t.Logf("%s is running (%s, guest %s) after boot", id, i.Virtualization, ip) - if i.Virtualization != "kvm" && i.Virtualization != "emulated" || i.VMUser != vmTestUser { + if i.Virtualization != "kvm" && i.Virtualization != "emulated" && i.Virtualization != "hvf" || i.VMUser != vmTestUser { t.Fatalf("instance reports virtualization %q, user %q", i.Virtualization, i.VMUser) } // The guest has the instance's private address with passt, or sits behind NAT in the diff --git a/cli/internal/svc/ec2/vm_security_test.go b/cli/internal/svc/ec2/vm_security_test.go index 632caee2..dd0e9985 100644 --- a/cli/internal/svc/ec2/vm_security_test.go +++ b/cli/internal/svc/ec2/vm_security_test.go @@ -4,6 +4,7 @@ import ( "bufio" "errors" "strings" + "sync" "testing" "github.com/homecloudhq/homecloud/cli/internal/svc/ec2/vm" @@ -51,7 +52,7 @@ func TestVMFlattenHasNoNetwork(t *testing.T) { // The VM container: no extra capabilities, the image cache and AMI read-only, // the console log bounded, and Docker's seccomp profile widened only by passt's four calls. func TestVMContainerConfinement(t *testing.T) { - archOnce.Do(func() { arch = "x86_64" }) // no Docker here + fakeHostArch(t, "x86_64") // no Docker here env := svctest.Env(t) s := New(env, vpc.New(env)) inst := Instance{ID: "i-vmsec", State: "running", Virtualization: "kvm", VMBase: "ubuntu-24.04", VMAMI: "hc-ami-x", @@ -93,6 +94,40 @@ func TestVMContainerConfinement(t *testing.T) { } } +// fakeHostArch pins hostArch for a test without Docker, and forgets it after, +// so that later tests in the binary ask the real Docker host. +func fakeHostArch(t *testing.T, a string) { + archOnce = sync.Once{} + archOnce.Do(func() { arch = a }) + t.Cleanup(func() { archOnce, arch = sync.Once{}, "" }) +} + +// A guest run on the Mac (hvf): the VM container runs no QEMU and needs no +// device; its entrypoint serves the disks and passt to the host runner. +func TestVMContainerHostMode(t *testing.T) { + fakeHostArch(t, "arm64") // no Docker here + env := svctest.Env(t) + s := New(env, vpc.New(env)) + inst := Instance{ID: "i-vmhvf", State: "running", Virtualization: virtHVF, VMBase: "ubuntu-24.04", + VCPUs: 2, MemoryMB: 2048, PrivateIP: "10.0.0.6", PrivateDNS: "ip-10-0-0-6.internal", VMDiskGB: 8, + Volumes: []VolumeAttachment{{VolumeID: "vol-root", MountPath: vm.DiskDir, Device: "/dev/xvda"}}} + spec := s.vmRunSpec(inst, "net") + if len(spec.Cmd) != 0 || len(spec.Devices) != 0 || len(spec.CapAdd) != 0 { + t.Errorf("host-mode VM container: cmd %v, devices %v, caps %v", spec.Cmd, spec.Devices, spec.CapAdd) + } + if spec.Env["HC_VM_HOST"] != "1" || spec.Labels[labelVMHost] != "1" { + t.Errorf("host mode not marked: env %v, labels %v", spec.Env, spec.Labels) + } + if spec.MemoryMB >= inst.MemoryMB { + t.Errorf("the container is sized for the guest (%d MiB), which runs on the host", spec.MemoryMB) + } + // Fallback: without passt the guest moves into the container, emulated. + inst.Virtualization, inst.VMNetwork = "emulated", "user" + if spec := s.vmRunSpec(inst, "net"); spec.Env["HC_VM_HOST"] != "" || len(spec.Cmd) == 0 { + t.Errorf("emulated guest in host mode: %v", spec.Env) + } +} + // Extra volumes of a VM launch are placed by device; the caller cannot put one // where the root disk goes or reuse the root device. func TestVMExtraVolumes(t *testing.T) { diff --git a/console/components/ec2/instance-actions.tsx b/console/components/ec2/instance-actions.tsx index 2ba4b9e7..cd532ead 100644 --- a/console/components/ec2/instance-actions.tsx +++ b/console/components/ec2/instance-actions.tsx @@ -73,19 +73,20 @@ export function PublicPorts({ instance, empty = "-", compact }: { instance: Inst ) } -export const isVMInstance = (i: Pick) => i.virtualization === "kvm" || i.virtualization === "emulated" +export const isVMInstance = (i: Pick) => !!i.virtualization export const isVMImage = (im: Pick) => im.virtualization === "vm" || !!im.vm_base -/** "Container", "VM · KVM" or "VM · emulated". */ +/** "Container", "VM · KVM", "VM · HVF" or "VM · emulated". */ export function instanceKind(i: Pick) { if (!isVMInstance(i)) return "Container" + if (i.virtualization === "hvf") return "VM · HVF" return i.virtualization === "kvm" ? "VM · KVM" : "VM · emulated" } -/** Type badge for an instance: Container / VM · KVM / VM · emulated (slow). */ +/** Type badge for an instance: Container / VM · KVM / VM · HVF / VM · emulated (slow). */ export function InstanceKindBadge({ instance }: { instance: Pick }) { - const accent = !isVMInstance(instance) ? "info" : instance.virtualization === "kvm" ? "violet" : "warning" + const accent = !isVMInstance(instance) ? "info" : instance.virtualization === "kvm" || instance.virtualization === "hvf" ? "violet" : "warning" return ( {instanceKind(instance)} diff --git a/console/components/ec2/instance-detail.tsx b/console/components/ec2/instance-detail.tsx index b5a4c1d9..04ef9750 100644 --- a/console/components/ec2/instance-detail.tsx +++ b/console/components/ec2/instance-detail.tsx @@ -213,7 +213,11 @@ function VirtualizationValue({ inst }: { inst: Instance }) { - {inst.virtualization === "kvm" ? "Own kernel, hardware accelerated (KVM)" : "Own kernel, CPU emulated (no /dev/kvm on the host, so it is slow)"} + {inst.virtualization === "kvm" + ? "Own kernel, hardware accelerated (KVM)" + : inst.virtualization === "hvf" + ? "Own kernel, hardware accelerated (QEMU on this Mac with Hypervisor.framework)" + : "Own kernel, CPU emulated (no /dev/kvm on the host, so it is slow)"} ) diff --git a/console/components/ec2/launch-wizard.tsx b/console/components/ec2/launch-wizard.tsx index 86eb831f..543d500b 100644 --- a/console/components/ec2/launch-wizard.tsx +++ b/console/components/ec2/launch-wizard.tsx @@ -460,7 +460,7 @@ export function LaunchWizard() { onSelect={() => chooseKind("vm")} icon={Monitor} title="Virtual machine" - description="Has its own kernel and stronger isolation. Boots in about a minute; needs KVM on the host for full speed." + description="Has its own kernel and stronger isolation. Boots in about a minute; needs KVM on Linux, or QEMU on an Apple Silicon Mac, for full speed." /> {isVM && caps.data?.vm === "emulated" && ( @@ -471,7 +471,11 @@ export function LaunchWizard() { This host has no KVM ( /dev/kvm), so virtual machines will be slow. Booting can take several - minutes. Containers are not affected. + minutes. Containers are not affected. On an Apple Silicon + Mac, install QEMU ( + brew install qemu) + and restart HomeCloud to run them at full speed with + Hypervisor.framework. )} diff --git a/console/lib/types/ec2.ts b/console/lib/types/ec2.ts index f85f6ec8..2a404739 100644 --- a/console/lib/types/ec2.ts +++ b/console/lib/types/ec2.ts @@ -42,15 +42,15 @@ export interface Instance { iam_profile_arn?: string iam_profile_id?: string metadata_options?: MetadataOptions - /** VM instances: "kvm" (hardware accelerated) or "emulated" (no /dev/kvm, slow). Absent for containers. */ - virtualization?: "kvm" | "emulated" | (string & {}) + /** VM instances: "kvm" (hardware accelerated), "hvf" (QEMU on the Mac with Hypervisor.framework) or "emulated" (no /dev/kvm, slow). Absent for containers. */ + virtualization?: "kvm" | "hvf" | "emulated" | (string & {}) vm_user?: string } /** What this host can run; GET /api/v1/ec2/capabilities. */ export interface Ec2Capabilities { - /** "kvm" = VMs accelerated, "emulated" = no /dev/kvm (slow), "unavailable" */ - vm: "kvm" | "emulated" | "unavailable" | (string & {}) + /** "kvm" or "hvf" = VMs accelerated, "emulated" = no /dev/kvm (slow), "unavailable" */ + vm: "kvm" | "hvf" | "emulated" | "unavailable" | (string & {}) } /** Instance metadata service (IMDS) options. */ diff --git a/docs/architecture.md b/docs/architecture.md index ba49d155..8fbed3b4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -76,7 +76,9 @@ The data directory defaults to `~/.homecloud` (override with `--data-dir` or `HO `homecloud backup` archives a VM's root volume as a standalone qcow2 (the volume carries the label `homecloud.vm-root`, and the server hands such volumes to a flatten step while the VM container is paused); the image cache is left out. A restore brings volumes back but no containers: when HomeCloud then finds a VM instance whose container is gone but whose root volume exists it recreates the container and leaves the instance `stopped`; starting it boots the restored disk. A container instance whose container is gone is recreated the same way from its image, with its EBS volumes and file systems mounted again (changes to its own root file system, outside volumes, are not in backups). If the container cannot be recreated yet (the image cannot be pulled, say), the instance stays `stopped` with the reason and its volumes are kept; it is only terminated when none of its volumes is left. -The guest runs with `-accel kvm -cpu host` when the Docker host has `/dev/kvm` (probed once by creating a container with the device; `HC_VM_ACCEL=kvm|tcg` overrides) and is otherwise emulated (`-accel tcg`, logged as a warning). The instance reports `virtualization: "kvm"` or `"emulated"`. Docker for Mac (OrbStack, Docker Desktop) has no `/dev/kvm`, so guests there are emulated; a native macOS `hvf` guest is not implemented. `-smp` and `-m` come from the instance type, and the container's limits are a little higher for QEMU's own overhead. An instance is `pending` until the guest is up: cloud-init's final message (or a login prompt for a while) on the serial console, with a generous timeout (30 minutes emulated). Changing security-group rules that change which ports are published rebuilds the container like for container instances; the disk is in the volume, but the guest is shut down and boots again. **Security trade-off.** passt sandboxes itself (user, mount and pid namespaces, `pivot_root`, then runs as `nobody`). Docker's default seccomp profile blocks that, so the VM container uses a custom profile: Docker's default (moby v27.4.1, embedded in `vm/seccomp-docker-default.json`, deny by default) plus an unconditional allow for exactly `unshare`, `mount`, `umount2` and `pivot_root`, the minimum found by removing each syscall in turn until passt failed. Everything else the default profile forbids (`bpf`, `keyctl`, `kexec_load`, module loading, `perf_event_open`, ...) stays forbidden, and the kernel still checks capabilities, which passt only has inside the namespaces it creates. AppArmor is not narrowed: on Ubuntu hosts the `docker-default` profile denies the remount and `pivot_root` of passt's sandbox (verified on Ubuntu 24.04 with `kernel.apparmor_restrict_unprivileged_userns=1`), and a narrower profile would have to be loaded on the host, which HomeCloud (a Docker client) cannot do, so the VM container runs with `apparmor=unconfined`. An administrator who loads a profile allowing mount, umount and `pivot_root` inside user namespaces can name it with `HC_VM_APPARMOR`. The container still gets only Docker's default capabilities, no `SYS_ADMIN`, no privileged mode and no host devices except `/dev/kvm` when accelerated; the trust is that of a root process in a container without MAC confinement running QEMU and passt, so use VM images only where you would accept that. passt also opens a socket per forwarded port, so the VM container's open-file limit is raised to 1048576 (Docker's default on some hosts is 65536 or 1024, which made passt fail with "Too many open files"). Instances report `vm_network: "passt"` or `"user"`. If passt cannot start (for example a host whose security modules forbid the user namespace), or when the server runs with `HC_VM_NET=user`, the container falls back to QEMU's user-mode networking: the guest then sits behind NAT at 10.0.2.15, still reaches the metadata service and DNS, and only the TCP/UDP ports that the instance's security groups open (at most 64, whatever their source) are forwarded to it; a rule that opens another port rebuilds the container (the guest reboots), for the automatic fallback too; the server logs why passt failed. The container's entrypoint stays up as a supervisor of QEMU and passt: passt runs with `-d`, its log (`/run/passt.log`, kept under 4 MiB) and, when it dies under a running guest, its exit status and the container's memory counters go to the container log and `/run/passt.dead` is created. HomeCloud notices that (when the instance is read, at most every 20 seconds) or during the boot wait, records `vm_network: "user"` for the instance and rebuilds the container with user-mode networking, so a passt failure costs the guest its private address and a reboot, not its connectivity. The mode sticks to the instance until it is terminated. +The guest runs with `-accel kvm -cpu host` when the Docker host has `/dev/kvm` (probed once by creating a container with the device; `HC_VM_ACCEL=kvm|tcg` overrides) and is otherwise emulated (`-accel tcg`, logged as a warning). The instance reports `virtualization: "kvm"` or `"emulated"`. Docker for Mac (OrbStack, Docker Desktop) has no `/dev/kvm` (their Linux VM has no nested virtualization), so on a Mac guests run on the Mac itself instead when they can (below), and are emulated in the container otherwise. `-smp` and `-m` come from the instance type, and the container's limits are a little higher for QEMU's own overhead. An instance is `pending` until the guest is up: cloud-init's final message (or a login prompt for a while) on the serial console, with a generous timeout (30 minutes emulated). Changing security-group rules that change which ports are published rebuilds the container like for container instances; the disk is in the volume, but the guest is shut down and boots again. **Security trade-off.** passt sandboxes itself (user, mount and pid namespaces, `pivot_root`, then runs as `nobody`). Docker's default seccomp profile blocks that, so the VM container uses a custom profile: Docker's default (moby v27.4.1, embedded in `vm/seccomp-docker-default.json`, deny by default) plus an unconditional allow for exactly `unshare`, `mount`, `umount2` and `pivot_root`, the minimum found by removing each syscall in turn until passt failed. Everything else the default profile forbids (`bpf`, `keyctl`, `kexec_load`, module loading, `perf_event_open`, ...) stays forbidden, and the kernel still checks capabilities, which passt only has inside the namespaces it creates. AppArmor is not narrowed: on Ubuntu hosts the `docker-default` profile denies the remount and `pivot_root` of passt's sandbox (verified on Ubuntu 24.04 with `kernel.apparmor_restrict_unprivileged_userns=1`), and a narrower profile would have to be loaded on the host, which HomeCloud (a Docker client) cannot do, so the VM container runs with `apparmor=unconfined`. An administrator who loads a profile allowing mount, umount and `pivot_root` inside user namespaces can name it with `HC_VM_APPARMOR`. The container still gets only Docker's default capabilities, no `SYS_ADMIN`, no privileged mode and no host devices except `/dev/kvm` when accelerated; the trust is that of a root process in a container without MAC confinement running QEMU and passt, so use VM images only where you would accept that. passt also opens a socket per forwarded port, so the VM container's open-file limit is raised to 1048576 (Docker's default on some hosts is 65536 or 1024, which made passt fail with "Too many open files"). Instances report `vm_network: "passt"` or `"user"`. If passt cannot start (for example a host whose security modules forbid the user namespace), or when the server runs with `HC_VM_NET=user`, the container falls back to QEMU's user-mode networking: the guest then sits behind NAT at 10.0.2.15, still reaches the metadata service and DNS, and only the TCP/UDP ports that the instance's security groups open (at most 64, whatever their source) are forwarded to it; a rule that opens another port rebuilds the container (the guest reboots), for the automatic fallback too; the server logs why passt failed. The container's entrypoint stays up as a supervisor of QEMU and passt: passt runs with `-d`, its log (`/run/passt.log`, kept under 4 MiB) and, when it dies under a running guest, its exit status and the container's memory counters go to the container log and `/run/passt.dead` is created. HomeCloud notices that (when the instance is read, at most every 20 seconds) or during the boot wait, records `vm_network: "user"` for the instance and rebuilds the container with user-mode networking, so a passt failure costs the guest its private address and a reboot, not its connectivity. The mode sticks to the instance until it is terminated. + +**VM guests on a Mac (`hvf`).** On an Apple Silicon Mac with QEMU installed (`brew install qemu`), QEMU runs on the Mac with Hypervisor.framework (`-accel hvf -cpu host`) instead of in the VM container, and the instance reports `virtualization: "hvf"`. HomeCloud chooses this when it runs on macOS/arm64 (not in a container), the Docker host is arm64, `qemu-system-aarch64` is on the `PATH` or in Homebrew's prefix (or `HC_VM_QEMU`), and a probe (an empty paused machine with `-accel hvf`, quit through QMP) succeeds; otherwise it falls back to the container as above and logs why. `serve --vm-accel` (saved in `config.json` as `vm_accel`) or `HC_VM_ACCEL` forces a backend: `hvf`, `container` (in the container, KVM if available), `kvm` or `tcg`; `auto` is the default. The VM container stays and keeps everything that makes the guest an instance: the private address and passt, security groups, published ports, the metadata route, the disks in their Docker volumes and the console log. With `HC_VM_HOST=1` its entrypoint prepares the disks and the seed as usual, starts passt, serves the root disk (the qcow2 overlay, with its backing image read in the container), the extra volumes, the seed and the UEFI variable store as NBD exports of `qemu-storage-daemon` on a Unix socket, and then only waits. A host runner joins QEMU on the Mac to those sockets: QEMU's network card (`-netdev stream`) and NBD disks connect to Unix sockets in the guest's run directory (`/vm/`, mode 0700), and the runner relays each connection into the container through a `docker exec socat - UNIX-CONNECT:...` stream. Nothing listens on a TCP port, so nothing but a Docker API client (already root-equivalent) and the Mac user who runs HomeCloud can reach the guest's network card or disks; unlike a published port, other containers cannot reach them through `host.docker.internal`. The guest has the same devices at the same PCI slots as in the container, so a guest can move between the backends. The UEFI firmware is the one installed with QEMU (Homebrew's `share/qemu/edk2-aarch64-code.fd`, or `HC_VM_FIRMWARE`): Debian's AAVMF build, which guests in the container boot, hangs before its boot manager under Hypervisor.framework; both use the same variable store format. The serial console is logged to `console.log` in the run directory and copied into the container's log through another `docker exec` (so `GetConsoleOutput` and the boot detection are unchanged); the serial console, guest agent and QMP sockets live in the run directory, where the terminal, run-command, guest metrics and the power button connect directly. The container lives as long as QEMU: when QEMU exits (the guest powered off) the runner writes its exit status into the container, which exits with it, and when the container stops or is removed (stop, terminate, a rebuild) the runner kills QEMU. So stop, start, reboot, rebuilds and state reconciliation are those of the container backend. The server starts a runner for every running host-mode VM container that has none (a loop over `homecloud.vm-host=1` containers, every second), as a separate process (`homecloud vm-host `, its own session, holding a lock on the run directory), so guests keep running across a server restart; a runner that died leaves a pid file by which the next one kills the orphaned QEMU (the guest reboots). A guest on the Mac cannot use user-mode networking with the instance's address: if passt cannot start for it, the instance falls back to the container, emulated. QEMU runs as the user who runs HomeCloud, without `-sandbox` (seccomp is Linux-only); it is confined by macOS process isolation, not by the VM container's seccomp profile. **Guest agent, run-command, terminal, metrics.** The command line adds a virtio-serial port `org.qemu.guest_agent.0` backed by a socket in the VM container (`/run/qga.sock`), and cloud-init's vendor-data (a script, so it survives a user-data `packages:` or `runcmd:`) installs and starts `qemu-guest-agent`. *Run-command* (`POST /api/v1/ec2/instances/{id}/commands`, `ssm:SendCommand` as for containers) sends `guest-exec` for `/bin/sh -c` (wrapped in the guest's `timeout(1)` so the command is killed at its timeout; status `TimedOut`) and polls `guest-exec-status` for stdout, stderr and the exit code. Each agent call is a `docker exec` of `socat` into the VM container that does the `guest-sync-delimited` handshake first; QEMU serves one agent client at a time, so calls to one guest are serialized. The agent needs the image's network on first boot to install it (images are Debian/Ubuntu); until it is up run-command fails with a clear error. The *browser terminal* cannot be a shell through the agent (`guest-exec` is not interactive), so it attaches to the guest's **serial console**: the serial port is a QEMU socket chardev (`/run/serial.sock`) with `logfile=/dev/stdout`, which tees everything the guest prints into the container log so `GetConsoleOutput` and the boot-ready detection are unchanged whether or not a terminal is attached. The terminal WebSocket runs `docker exec -t ... socat -,raw,echo=0 UNIX-CONNECT:/run/serial.sock`; the user presses Enter and logs in at the guest's getty. Cloud images have no console password, so set one through user data (`chpasswd`) to use it; one serial session at a time. *CloudWatch* metrics of a VM instance come from the guest, not from QEMU: the collector asks the EC2 service for a sample, which runs one `guest-exec` that reads `/proc/stat` twice a second apart (`CPUUtilization`, no state between samples), `/proc/meminfo` (`MemoryUsed` = total - available, `MemoryUtilization`), `/proc/net/dev` without `lo` (`NetworkIn/Out`), `/proc/diskstats` for whole disks (`DiskReadBytes/WriteBytes`) and the process count, published under the same `HC/EC2` namespace and `InstanceId` dimension as containers. `/proc` through `guest-exec` was chosen over `guest-get-cpustats` (new in QGA 7.1, not in every image) and QMP statistics (which only see the virtual devices). Samples run beside the collector's tick, one per instance at a time, and publish nothing while the agent is not up. `homecloud backup` still sees the VM container: a backup copies the root volume's qcow2 file, which points at the cached cloud image (the cache volume itself is left out of backups). diff --git a/docs/install-server.md b/docs/install-server.md index 3111d229..6dfd42a0 100644 --- a/docs/install-server.md +++ b/docs/install-server.md @@ -548,7 +548,12 @@ container on the instance's VPC network. Container images need nothing of this. - **KVM.** With `/dev/kvm` on the Docker host, guests run hardware-accelerated; without it they are emulated (works, but boots take minutes). Prefer bare metal, a home server, or a VPS with nested virtualization; check with `ls -l /dev/kvm` (`sudo apt install cpu-checker && kvm-ok` tells whether the - CPU and firmware support it). Docker Desktop and OrbStack on macOS run guests emulated. + CPU and firmware support it). Docker Desktop and OrbStack on macOS have no `/dev/kvm`. +- **macOS.** On an Apple Silicon Mac, install QEMU (`brew install qemu`) and HomeCloud runs guests on the + Mac itself with Hypervisor.framework (`virtualization: hvf`), at full speed, with the same networking, + disks and console; the VM container in Docker keeps the instance's address, firewall and volumes. Without + QEMU, guests are emulated in Docker (boots take minutes). `homecloud serve --vm-accel container` keeps them + in Docker; `--vm-accel hvf` insists on the Mac. - **Networking.** passt gives the guest the instance's private address. Where passt cannot sandbox itself, notably Ubuntu 24.04 hosts (`kernel.apparmor_restrict_unprivileged_userns=1`), guests fall back to QEMU user-mode networking: NAT, no private address of their own, only the ports the security groups From d286eb32b039b78bc1328b056f2d170c607a5cd4 Mon Sep 17 00:00:00 2001 From: Suryansh Prajapati <58465650+drk1rd@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:06:22 +0530 Subject: [PATCH 2/2] VM host runner: harden run directories and QEMU lifetime - Refuse a run directory (or its parent) that is not a 0700 directory of this user, so a directory planted in a shared temp dir cannot feed the runner a command line; avoid data directories with a comma. - Only kill a stale QEMU whose command line names this run directory (pid reuse could hit another guest's QEMU), and kill it on terminate before removing the run directory instead of orphaning it. - Kill QEMU only when Docker says the container stopped or is gone, not on a failed Docker API call. - Start a runner only for the instance's current container in hvf mode. - Bound the serial console log on the host. - Reject hvf launches up front where hvf cannot work (e.g. Linux). --- cli/internal/svc/ec2/vm.go | 6 ++ cli/internal/svc/ec2/vm_host.go | 41 ++++++-- cli/internal/svc/ec2/vm_host_unix.go | 31 +++++- cli/internal/svc/ec2/vm_host_unix_test.go | 121 ++++++++++++++++++++++ cli/internal/svc/ec2/vm_host_windows.go | 3 + 5 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 cli/internal/svc/ec2/vm_host_unix_test.go diff --git a/cli/internal/svc/ec2/vm.go b/cli/internal/svc/ec2/vm.go index f6f2387e..e56f99e2 100644 --- a/cli/internal/svc/ec2/vm.go +++ b/cli/internal/svc/ec2/vm.go @@ -141,6 +141,12 @@ func (s *Service) planVM(in RunInput, img Image) (vmPlan, error) { p.ami = img.VMDisk p.root.SizeGB = p.diskGB p.virtualization = s.vmAccel() + if p.virtualization == virtHVF { + // (hvf forced where it cannot work: fail here, not with a host runner per guest.) + if why := vmHostUnsupported(hostArch(s)); why != "" { + return p, core.Errf(http.StatusServiceUnavailable, "Unsupported", "virtual machine instances are set to run with Hypervisor.framework, but %s", why) + } + } return p, nil } diff --git a/cli/internal/svc/ec2/vm_host.go b/cli/internal/svc/ec2/vm_host.go index 97f1317c..2edbb267 100644 --- a/cli/internal/svc/ec2/vm_host.go +++ b/cli/internal/svc/ec2/vm_host.go @@ -166,10 +166,11 @@ func (s *Service) vmHostProbe() string { // vmHostBase is the directory holding the run directories of host-run guests. // Unix socket paths are short (104 bytes on macOS): a data directory with a -// long path uses the temporary directory instead. +// long path uses the temporary directory instead, as does one with a comma, +// which would end a QEMU option holding a socket path. func (s *Service) vmHostBase() string { base := filepath.Join(s.env.Cfg.DataDir, "vm") - if s.env.Cfg.DataDir == "" || len(base) > 64 { + if s.env.Cfg.DataDir == "" || len(base) > 64 || strings.Contains(base, ",") { base = filepath.Join(os.TempDir(), fmt.Sprintf("homecloud-vm-%d", os.Getuid())) } return base @@ -218,7 +219,9 @@ func (s *Service) vmHostScan() { } for _, c := range cs { inst, err := store.Get[Instance](s.env.Store, cInstances, c.Labels[core.LabelResource]) - if err != nil || inst.State == "terminated" { + // Only the instance's own container, and only while it runs on the host: + // a container left over from a rebuild must not get the guest's disks. + if err != nil || inst.State == "terminated" || !vmHostMode(inst) || c.ID != inst.ContainerID { continue } if err := s.vmHostEnsure(inst, c.ID); err != nil { @@ -235,10 +238,10 @@ func (s *Service) vmHostEnsure(inst Instance, cid string) error { if t, ok := s.vm.spawned[cid]; ok && time.Since(t) < 5*time.Second { return nil // just started: it may not hold its lock yet } - if err := os.MkdirAll(dir, 0o700); err != nil { + if err := privateDir(filepath.Dir(dir)); err != nil { return err } - if err := os.Chmod(dir, 0o700); err != nil { + if err := privateDir(dir); err != nil { return err } if vmHostRunning(dir) { @@ -344,6 +347,18 @@ func (h *vmHost) Write(p []byte) (int, error) { func (h *vmHost) running() bool { return h.d.State(h.cfg.Container) == "running" } +// containerGone reports whether the VM container has stopped or is gone. An +// error asking Docker (it restarts, say) is not taken as the container gone: +// that would kill a running guest. +func (h *vmHost) containerGone() bool { + c, err := h.d.C.InspectContainer(h.cfg.Container) + if err != nil { + var nsc *docker.NoSuchContainer + return errors.As(err, &nsc) + } + return !c.State.Running +} + func (h *vmHost) run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -433,7 +448,7 @@ func (h *vmHost) run(ctx context.Context) error { case <-exited: return case <-t.C: - if !h.running() { + if h.containerGone() { h.logf("the VM container stopped: killing QEMU") _ = cmd.Process.Kill() return @@ -491,6 +506,11 @@ func (h *vmHost) finish(ctx context.Context, status int, consoleDone <-chan stru } } +// vmHostConsoleMax bounds the serial console log on the host: what has been +// copied to the container's log is cut away past it, or a guest printing +// without end would fill the host's disk. +const vmHostConsoleMax = 4 << 20 + // tail copies what QEMU logs of the serial console to the container's log // until QEMU has exited, then copies the rest. func (h *vmHost) tail(exited <-chan struct{}) { @@ -521,6 +541,12 @@ func (h *vmHost) tail(exited <-chan struct{}) { break } } + // QEMU appends (logappend=on), so its next write lands at the start. + if off, err := f.Seek(0, io.SeekCurrent); err == nil && off > vmHostConsoleMax { + if os.Truncate(h.path(vm.HostConsoleLog), 0) == nil { + _, _ = f.Seek(0, io.SeekStart) + } + } } if done { return @@ -562,6 +588,9 @@ func (s *Service) vmHostCleanup(id string) { for n := 0; n < 60 && vmHostRunning(dir); n++ { time.Sleep(time.Second) } + if !vmHostRunning(dir) { + killStaleQEMU(filepath.Join(dir, "qemu.pid")) // a runner that died left its QEMU + } if err := os.RemoveAll(dir); err != nil { log.Printf("ec2: %s: remove the guest's run directory: %v", id, err) } diff --git a/cli/internal/svc/ec2/vm_host_unix.go b/cli/internal/svc/ec2/vm_host_unix.go index b0e654b1..b2fb02dd 100644 --- a/cli/internal/svc/ec2/vm_host_unix.go +++ b/cli/internal/svc/ec2/vm_host_unix.go @@ -4,6 +4,7 @@ package ec2 import ( "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -49,8 +50,32 @@ func detach(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} } +// privateDir makes sure dir is a directory of this user that nobody else can +// enter (creating it if need be): a run directory holds the command line a +// host runner executes and the guest's sockets, and its parent may be in a +// shared temporary directory where another user could have made it first. +func privateDir(dir string) error { + if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + st, err := os.Lstat(dir) + if err != nil { + return err + } + sys, ok := st.Sys().(*syscall.Stat_t) + if !st.IsDir() || !ok || int(sys.Uid) != os.Getuid() { + return fmt.Errorf("%s is not a directory of this user", dir) + } + if st.Mode().Perm() != 0o700 { + return os.Chmod(dir, 0o700) + } + return nil +} + // killStaleQEMU kills the QEMU a host runner that died left behind (its -// network and disks went with the runner). +// network and disks went with the runner). The process must still be a QEMU +// of this run directory: its number may since have gone to another process, +// another guest's QEMU included. func killStaleQEMU(pidFile string) { b, err := os.ReadFile(pidFile) if err != nil { @@ -61,8 +86,8 @@ func killStaleQEMU(pidFile string) { if err != nil || pid <= 1 { return } - out, err := exec.Command("ps", "-o", "comm=", "-p", strconv.Itoa(pid)).Output() - if err != nil || !strings.Contains(string(out), "qemu-system") { + out, err := exec.Command("ps", "-ww", "-o", "args=", "-p", strconv.Itoa(pid)).Output() + if err != nil || !strings.Contains(string(out), "qemu-system") || !strings.Contains(string(out), filepath.Dir(pidFile)+string(filepath.Separator)) { return // gone, or the number was reused } _ = syscall.Kill(pid, syscall.SIGKILL) diff --git a/cli/internal/svc/ec2/vm_host_unix_test.go b/cli/internal/svc/ec2/vm_host_unix_test.go new file mode 100644 index 00000000..b48c4bb4 --- /dev/null +++ b/cli/internal/svc/ec2/vm_host_unix_test.go @@ -0,0 +1,121 @@ +//go:build !windows + +package ec2 + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/homecloudhq/homecloud/cli/internal/core" + "github.com/homecloudhq/homecloud/cli/internal/svc" + "github.com/homecloudhq/homecloud/cli/internal/svc/ec2/vm" +) + +// A run directory is this user's alone: made 0700, tightened if it was not, +// and refused when it is not a directory of this user (a symlink planted in a +// shared temporary directory, say). +func TestVMHostPrivateDir(t *testing.T) { + base := t.TempDir() + d := filepath.Join(base, "new") + if err := privateDir(d); err != nil { + t.Fatal(err) + } + if st, _ := os.Stat(d); st.Mode().Perm() != 0o700 { + t.Errorf("new run directory has mode %v", st.Mode().Perm()) + } + loose := filepath.Join(base, "loose") + if err := os.Mkdir(loose, 0o777); err != nil { + t.Fatal(err) + } + _ = os.Chmod(loose, 0o777) + if err := privateDir(loose); err != nil { + t.Fatal(err) + } + if st, _ := os.Stat(loose); st.Mode().Perm() != 0o700 { + t.Errorf("existing run directory left with mode %v", st.Mode().Perm()) + } + link := filepath.Join(base, "link") + if err := os.Symlink(d, link); err != nil { + t.Fatal(err) + } + if err := privateDir(link); err == nil { + t.Error("a symlink was taken for a run directory") + } +} + +// A data directory whose path has a comma (which would end a QEMU option) +// keeps the guests' sockets elsewhere. +func TestVMHostBaseComma(t *testing.T) { + s := &Service{env: &svc.Env{Cfg: core.Config{DataDir: "/tmp/a,b"}}} + if b := s.vmHostBase(); strings.Contains(b, ",") { + t.Errorf("run directory base %q", b) + } + s.env.Cfg.DataDir = "/tmp/hc" + if b := s.vmHostBase(); b != "/tmp/hc/vm" { + t.Errorf("run directory base %q", b) + } +} + +// killStaleQEMU only kills a QEMU of the run directory the pid file is in: +// the number may have gone to another process since, another guest's QEMU included. +func TestVMHostKillStaleQEMUChecksProcess(t *testing.T) { + dir, other := t.TempDir(), t.TempDir() + start := func(d string) *exec.Cmd { + // (Two commands, so the shell stays with this command line.) + cmd := exec.Command("/bin/sh", "-c", "sleep 30; : qemu-system-aarch64 path="+d+"/net.sock") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + return cmd + } + pidFile := filepath.Join(dir, "qemu.pid") + foreign := start(other) + if err := os.WriteFile(pidFile, []byte(fmt.Sprint(foreign.Process.Pid)), 0o600); err != nil { + t.Fatal(err) + } + killStaleQEMU(pidFile) + if err := syscall.Kill(foreign.Process.Pid, 0); err != nil { + t.Errorf("killStaleQEMU killed another guest's QEMU: %v", err) + } + own := start(dir) + if err := os.WriteFile(pidFile, []byte(fmt.Sprint(own.Process.Pid)), 0o600); err != nil { + t.Fatal(err) + } + waited := make(chan error, 1) + go func() { waited <- own.Wait() }() // (reaped, as launchd reaps an orphaned QEMU) + killStaleQEMU(pidFile) + select { + case err := <-waited: + if err == nil { + t.Error("this guest's QEMU exited by itself") + } + case <-time.After(10 * time.Second): + t.Error("killStaleQEMU left this guest's QEMU running") + } +} + +// The serial console log on the host does not grow without bound. +func TestVMHostConsoleLogBounded(t *testing.T) { + h := &vmHost{dir: t.TempDir()} + p := h.path(vm.HostConsoleLog) + if err := os.WriteFile(p, make([]byte, vmHostConsoleMax+1), 0o600); err != nil { + t.Fatal(err) + } + exited := make(chan struct{}) + close(exited) + h.tail(exited) + st, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + if st.Size() != 0 { + t.Errorf("console log is %d bytes after copying", st.Size()) + } +} diff --git a/cli/internal/svc/ec2/vm_host_windows.go b/cli/internal/svc/ec2/vm_host_windows.go index 6074f93d..8f4a6626 100644 --- a/cli/internal/svc/ec2/vm_host_windows.go +++ b/cli/internal/svc/ec2/vm_host_windows.go @@ -2,6 +2,7 @@ package ec2 import ( "errors" + "os" "os/exec" ) @@ -16,3 +17,5 @@ func vmHostRunning(string) bool { return false } func detach(*exec.Cmd) {} func killStaleQEMU(string) {} + +func privateDir(dir string) error { return os.MkdirAll(dir, 0o700) }