Run VM guests on macOS with Hypervisor.framework

README.md

@@ -135,7 +135,7 @@ Run `homecloud aws-env` to get the environment variables. For Terraform, point t
 
 | Service | AWS equivalent | Status | Notes |
 | --- | --- | --- | --- |
-| Compute | EC2, EBS, AMIs | AWS API | Instances are **containers**; `ami-ubuntu-24-04-vm` and `ami-debian-12-vm` boot real **virtual machines** (QEMU, KVM when the host has `/dev/kvm`, emulated otherwise) on the same VPC networking, with extra volumes, snapshots, images, run-command and a serial console. Key pairs, user data, volumes, snapshots, launch templates, Elastic IPs (records only), IMDSv1/v2, browser shell |
+| Compute | EC2, EBS, AMIs | AWS API | Instances are **containers**; `ami-ubuntu-24-04-vm` and `ami-debian-12-vm` boot real **virtual machines** (QEMU, KVM when the host has `/dev/kvm`, Hypervisor.framework on Apple Silicon Macs with QEMU installed, emulated otherwise) on the same VPC networking, with extra volumes, snapshots, images, run-command and a serial console. Key pairs, user data, volumes, snapshots, launch templates, Elastic IPs (records only), IMDSv1/v2, browser shell |
 | Auto Scaling | EC2 Auto Scaling | AWS API | Target tracking on CPU; scheduled actions and lifecycle hooks are not implemented |
 | Networking | VPC, security groups | AWS API | Security groups enforced inside the VPC; network ACLs recorded, not enforced; no peering; IPv4 only |
 | Load balancing | ELB v2 | AWS API | Application load balancers (HTTP/HTTPS, path/host rules). No network load balancers |
@@ -231,7 +231,7 @@ Releases are signed (from v0.5.0): the release workflow signs `checksums.txt` an
 
 - **Shipped in [0.4.0](https://github.com/solinode/homecloud/releases/tag/v0.4.0):** VM-backed EC2 instances (QEMU, KVM when available), the container image `ghcr.io/solinode/homecloud`, a GitHub Action and testcontainers modules for Go and Python, a nightly Terraform modules compatibility suite ([17 of 17 pass](docs/compatibility.md)), and fixes from two security audits ([platform](docs/security-audit-2026-10.md), [VM instances](docs/security-audit-vm-2026-10.md)).
 - **Shipped in [0.5.0](https://github.com/solinode/homecloud/releases/tag/v0.5.0):** signed releases (Sigstore) with SBOMs and build provenance, verified by the installer and `homecloud upgrade`; automatic backups before upgrades and `homecloud upgrade --rollback`; restores that bring back databases, caches, load balancers and instances on a new machine; Prometheus metrics and a system status page ([docs/monitoring.md](docs/monitoring.md)); Testcontainers modules for Java and Node.
-- **Next:** closing the AWS API gaps users hit most, native VM acceleration on macOS, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms.
+- **Next:** closing the AWS API gaps users hit most, passt networking for VMs on Ubuntu 24.04 hosts ([#90](https://github.com/solinode/homecloud/issues/90)), and multiple accounts with SSO for teams and classrooms.
 - **Planned:** multi-node clusters ([#55](https://github.com/solinode/homecloud/issues/55), [design](docs/design/multi-node.md)), edge compute and hardware integrations ([#56](https://github.com/solinode/homecloud/issues/56), [design](docs/design/edge.md)).
 
 See the [open issues](https://github.com/solinode/homecloud/issues) for everything planned.

cli/cmd/serve.go

@@ -17,6 +17,7 @@ import (
 	"github.com/homecloudhq/homecloud/cli/internal/core"
 	"github.com/homecloudhq/homecloud/cli/internal/httpx"
 	"github.com/homecloudhq/homecloud/cli/internal/server"
+	"github.com/homecloudhq/homecloud/cli/internal/svc/ec2"
 	"github.com/spf13/cobra"
 )
 
@@ -47,6 +48,7 @@ and CLI credentials in the data directory. Settings passed as flags are saved to
 			}
 			server.Version = Version
 			server.Commit = Commit
+			ec2.VMHostCommand = vmHostCommand
 			ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 			defer stop()
 			return server.Run(ctx, cfg, server.Options{ResetRootPassword: resetRoot, AllowNewerData: allowNewerData})
@@ -70,6 +72,7 @@ and CLI credentials in the data directory. Settings passed as flags are saved to
 	f.BoolVar(&allowNewerData, "allow-newer-data", false, "start even if the data directory was last used by a newer HomeCloud (risks corrupting it; prefer homecloud upgrade --rollback)")
 	f.StringVar(&cfg.MetricsAddr, "metrics-addr", cfg.MetricsAddr, "also serve Prometheus metrics WITHOUT authentication on this loopback address, e.g. 127.0.0.1:9464 (the API's /metrics always requires the IAM action homecloud:GetMetrics)")
 	f.BoolVar(&cfg.MetricsAddrInsecure, "metrics-addr-insecure", false, "allow a non-loopback --metrics-addr (its metrics need no credentials: protect it with a firewall)")
+	f.StringVar(&cfg.VMAccel, "vm-accel", "auto", "how VM instances run: auto, hvf (QEMU on this Mac with Hypervisor.framework), container (QEMU in Docker, KVM when available), kvm or tcg")
 	RootCmd.AddCommand(serveCmd)
 
 	var p client.Profile
@@ -219,6 +222,12 @@ func mergeConfig(cmd *cobra.Command, flags core.Config) (core.Config, error) {
 	set("tls-key", func() { cfg.TLSKey = flags.TLSKey })
 	set("metrics-addr", func() { cfg.MetricsAddr = flags.MetricsAddr })
 	set("metrics-addr-insecure", func() { cfg.MetricsAddrInsecure = flags.MetricsAddrInsecure })
+	set("vm-accel", func() { cfg.VMAccel = flags.VMAccel })
+	switch cfg.VMAccel {
+	case "", "auto", "hvf", "container", "kvm", "tcg":
+	default:
+		return cfg, fmt.Errorf("--vm-accel is auto, hvf, container, kvm or tcg, got %q", cfg.VMAccel)
+	}
 	if err := server.ValidateMetricsAddr(cfg.MetricsAddr, cfg.MetricsAddrInsecure); err != nil {
 		return cfg, err
 	}

cli/cmd/vmhost.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+
+	"github.com/homecloudhq/homecloud/cli/internal/runtime"
+	"github.com/homecloudhq/homecloud/cli/internal/svc/ec2"
+	"github.com/spf13/cobra"
+)
+
+// vm-host is started by the server, one per VM instance whose guest runs on
+// this Mac (QEMU with Hypervisor.framework): it runs QEMU and joins it to the
+// instance's VM container. It is a process of its own so that guests keep
+// running when the server restarts.
+func init() {
+	RootCmd.AddCommand(&cobra.Command{
+		Use:    "vm-host DIR",
+		Short:  "Run the host side of a VM instance (started by the server)",
+		Hidden: true,
+		Args:   cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			d, err := runtime.New()
+			if err != nil {
+				return err
+			}
+			// SIGTERM and SIGINT are ignored: the guest stops when its container does.
+			signal.Ignore(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
+			return ec2.RunVMHost(context.Background(), d, args[0])
+		},
+	})
+}
+
+// vmHostCommand is ec2.VMHostCommand for the server: this binary's vm-host.
+func vmHostCommand(dir string) *exec.Cmd {
+	exe, err := os.Executable()
+	if err != nil {
+		exe = os.Args[0]
+	}
+	return exec.Command(exe, "vm-host", dir)
+}

cli/internal/core/core.go

@@ -74,6 +74,12 @@ type Config struct {
 	// a loopback address unless MetricsAddrInsecure is set.
 	MetricsAddr         string `json:"metrics_addr,omitempty"`
 	MetricsAddrInsecure bool   `json:"metrics_addr_insecure,omitempty"`
+	// VMAccel chooses how VM instances run: "auto" (the default: on an Apple
+	// Silicon Mac with QEMU installed, QEMU on the Mac with Hypervisor.framework;
+	// otherwise in the VM container with KVM when the Docker host has /dev/kvm,
+	// emulated if not), "hvf" (always on the Mac), "container" (always in the
+	// container, KVM when available), "kvm" or "tcg". HC_VM_ACCEL overrides it.
+	VMAccel string `json:"vm_accel,omitempty"`
 }
 
 func DefaultDataDir() string {

cli/internal/runtime/self.go

@@ -30,6 +30,9 @@ import (
 // /proc/self/mountinfo (/var/lib/docker/containers/<id>/hostname and friends).
 var containerIDPattern = regexp.MustCompile(`/containers/([0-9a-f]{64})/`)
 
+// InContainer reports whether this process runs in a container.
+func InContainer() bool { return inContainer() }
+
 // inContainer reports whether this process runs in a container.
 func inContainer() bool {
 	if os.Getenv("HOMECLOUD_IN_CONTAINER") == "1" {

cli/internal/svc/ec2/capabilities.go

@@ -5,7 +5,8 @@ import "github.com/homecloudhq/homecloud/cli/internal/httpx"
 // Capabilities tells the console what this host can run, so the launch wizard
 // can warn before someone picks a VM image on a host without KVM.
 type Capabilities struct {
-	// VM is "kvm" when virtual machine instances run hardware-accelerated,
+	// VM is "hvf" when virtual machine instances run on this Mac with
+	// Hypervisor.framework, "kvm" when they run hardware-accelerated,
 	// "emulated" when the Docker host has no usable /dev/kvm (QEMU emulates the
 	// CPU, which is slow), and "unavailable" when VMs cannot run at all.
 	VM string `json:"vm"`
@@ -18,8 +19,5 @@ func (s *Service) capabilities(c *httpx.Ctx) (any, error) {
 	if s.vmRunnerFailure() != nil {
 		return Capabilities{VM: "unavailable"}, nil
 	}
-	if s.vmKVM() {
-		return Capabilities{VM: "kvm"}, nil
-	}
-	return Capabilities{VM: "emulated"}, nil
+	return Capabilities{VM: s.vmAccel()}, nil
 }

cli/internal/svc/ec2/ec2.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"net/netip"
+	goruntime "runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -1024,6 +1025,9 @@ func (s *Service) instanceGone(i Instance) {
 	if s.imds != nil {
 		s.imds.forget(i.ID)
 	}
+	if i.IsVM() && goruntime.GOOS == "darwin" {
+		go s.vmHostCleanup(i.ID)
+	}
 }
 
 func (s *Service) modify(c *httpx.Ctx) (any, error) {
@@ -1460,6 +1464,9 @@ func (s *Service) dns(vpcID string) []string {
 
 // Recover settles instances whose launch or state change was interrupted by a restart.
 func (s *Service) Recover() {
+	if goruntime.GOOS == "darwin" && s.env.Docker != nil {
+		s.vmHostWatch() // guests on this Mac whose containers kept running
+	}
 	for _, i := range store.List[Instance](s.env.Store, cInstances) {
 		switch i.State {
 		case "pending", "stopping", "shutting-down":

cli/internal/svc/ec2/terminal.go

@@ -1,9 +1,11 @@
 package ec2
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"net/http"
+	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/homecloudhq/homecloud/cli/internal/core"
@@ -34,6 +36,10 @@ func (s *Service) terminal(c *httpx.Ctx) (any, error) {
 		defer conn.Close()
 		if i.IsVM() {
 			_ = websocket.Message.Send(conn, "[homecloud] serial console of "+i.ID+": press Enter for a login prompt (one session at a time).\r\n")
+			if vmHostMode(i) {
+				s.hostSerial(conn, i)
+				return
+			}
 		}
 		ex, err := s.env.Docker.C.CreateExec(docker.CreateExecOptions{
 			Container: i.ContainerID, Tty: true, AttachStdin: true, AttachStdout: true, AttachStderr: true,
@@ -83,6 +89,39 @@ func (s *Service) terminal(c *httpx.Ctx) (any, error) {
 	return nil, nil
 }
 
+// hostSerial joins the terminal to the serial console of a guest run on the
+// host (its socket is in the guest's run directory, not in the VM container).
+func (s *Service) hostSerial(conn *websocket.Conn, i Instance) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	sc, err := s.vmHostDial(ctx, i, vm.HostSerialSocket)
+	cancel()
+	if err != nil {
+		_ = websocket.Message.Send(conn, "\r\n[homecloud] could not open the serial console: "+err.Error()+"\r\n")
+		return
+	}
+	defer sc.Close()
+	go func() {
+		defer sc.Close()
+		for {
+			var raw string
+			if err := websocket.Message.Receive(conn, &raw); err != nil {
+				return
+			}
+			var m struct {
+				T string `json:"t"`
+				D string `json:"d"`
+			}
+			if json.Unmarshal([]byte(raw), &m) == nil && m.T == "i" {
+				if _, err := sc.Write([]byte(m.D)); err != nil {
+					return
+				}
+			}
+		}
+	}()
+	_, _ = io.Copy(wsWriter{conn}, sc)
+	_ = websocket.Message.Send(conn, "\r\n[homecloud] session closed\r\n")
+}
+
 type wsWriter struct{ c *websocket.Conn }
 
 func (w wsWriter) Write(p []byte) (int, error) {

cli/internal/svc/ec2/vm.go

@@ -34,7 +34,9 @@ import (
 // it. The root disk is a qcow2 overlay on the cached base image, kept in the
 // instance's root volume (an EBS volume record at /dev/xvda, mounted at /vm).
 // The guest uses hardware acceleration when the Docker host has /dev/kvm and
-// is otherwise emulated.
+// is otherwise emulated, except on an Apple Silicon Mac with QEMU installed,
+// where QEMU runs on the Mac itself with Hypervisor.framework and the VM
+// container keeps the network, disks and console (vm_host.go).
 //
 // Stage 1 covers the instance lifecycle. Extra volumes, snapshots and images
 // of VM disks and backup are not available for VM instances yet. The browser
@@ -56,12 +58,19 @@ type vmState struct {
 	kvmOnce sync.Once
 	kvm     bool
 
+	accelOnce sync.Once
+	accel     string // vmAccel
+	qemu      string // QEMU binary for host-run guests, once probed
+	watchOnce sync.Once
+	spawnMu   sync.Mutex
+	spawned   map[string]time.Time // VM container -> when its host runner was started
+
 	fetchMu sync.Mutex // one download at a time
 }
 
 // vmPlan is what Launch works out once for every instance of a VM launch.
 type vmPlan struct {
-	virtualization string // kvm | emulated
+	virtualization string // hvf | kvm | emulated
 	user           string
 	diskGB         int
 	root           VolumeSpec
@@ -131,18 +140,21 @@ func (s *Service) planVM(in RunInput, img Image) (vmPlan, error) {
 	}
 	p.ami = img.VMDisk
 	p.root.SizeGB = p.diskGB
-	p.virtualization = "emulated"
-	if s.vmKVM() {
-		p.virtualization = "kvm"
+	p.virtualization = s.vmAccel()
+	if p.virtualization == virtHVF {
+		// (hvf forced where it cannot work: fail here, not with a host runner per guest.)
+		if why := vmHostUnsupported(hostArch(s)); why != "" {
+			return p, core.Errf(http.StatusServiceUnavailable, "Unsupported", "virtual machine instances are set to run with Hypervisor.framework, but %s", why)
+		}
 	}
 	return p, nil
 }
 
 // vmKVM reports whether the Docker host offers /dev/kvm. It is probed once by
-// creating a container with the device (HC_VM_ACCEL=kvm|tcg overrides).
+// creating a container with the device (HC_VM_ACCEL or vm_accel = kvm|tcg overrides).
 func (s *Service) vmKVM() bool {
 	s.vm.kvmOnce.Do(func() {
-		switch os.Getenv("HC_VM_ACCEL") {
+		switch s.vmAccelSetting() {
 		case "kvm":
 			s.vm.kvm = true
 			return
@@ -316,6 +328,16 @@ func (s *Service) vmRunSpec(inst Instance, network string) runtime.RunSpec {
 	if mach.KVM {
 		spec.Devices = []string{"/dev/kvm"}
 	}
+	if vmHostMode(inst) {
+		// QEMU runs on the host (vm_host.go): the container only prepares the
+		// disks and serves them and passt, and lives as long as the guest.
+		spec.Cmd = nil
+		spec.Env["HC_VM_HOST"] = "1"
+		spec.Labels[labelVMHost] = "1"
+		spec.NanoCPUs = int64(min(2, s.hostCPU) * 1e9)
+		spec.MemoryMB = 768
+		s.vmHostWatch()
+	}
 	if inst.VMAMI != "" {
 		spec.Env["HC_VM_AMI"] = "1"
 	}
@@ -459,6 +481,12 @@ func (s *Service) launchVM(inst Instance, network string) {
 	s.metadataRoute(ctx, cid, inst.VpcID)
 	s.vpc.ProtectNow(ctx, s.member(inst, cid))
 	err = s.vmWaitReady(inst, since)
+	if err != nil && vmHostMode(inst) {
+		// passt did not start, which a guest on the host cannot do without: run it in the container.
+		if why := s.vmPasstFailure(cid, since); why != "" {
+			err = fmt.Errorf("%w: %s", errPasstExited, why)
+		}
+	}
 	if errors.Is(err, errPasstExited) {
 		// passt died while the guest booted: boot it again in user-mode networking.
 		log.Printf("ec2: %s: %v; restarting the guest with user-mode networking", inst.ID, err)
@@ -505,8 +533,17 @@ var errPasstExited = errors.New("passt exited")
 // vmFallbackToUser records that passt does not work for an instance and
 // rebuilds its container with QEMU's user-mode networking (the guest boots
 // again; it sits behind NAT, and security group changes rebuild it).
+//
+// A guest run on the host (hvf) cannot use user-mode networking with the
+// instance's address: it moves into the VM container, emulated.
 func (s *Service) vmFallbackToUser(id string) error {
-	if _, err := store.Update(s.env.Store, cInstances, id, func(x *Instance) error { x.VMNetwork = "user"; return nil }); err != nil {
+	if _, err := store.Update(s.env.Store, cInstances, id, func(x *Instance) error {
+		x.VMNetwork = "user"
+		if x.Virtualization == virtHVF {
+			x.Virtualization = "emulated"
+		}
+		return nil
+	}); err != nil {
 		return err
 	}
 	return s.recreate(id)
@@ -637,7 +674,7 @@ func tail(s string, n int) string {
 // vmBootTimeout is how long a guest gets to finish booting before the instance
 // is reported running anyway (images without cloud-init never print the marker).
 func vmBootTimeout(inst Instance) time.Duration {
-	if inst.Virtualization == "kvm" {
+	if inst.Virtualization == "kvm" || inst.Virtualization == virtHVF {
 		return 10 * time.Minute
 	}
 	return 30 * time.Minute
@@ -683,7 +720,7 @@ func (s *Service) vmWaitReady(inst Instance, since time.Time) error {
 
 // vmShutdownTimeout is how long the guest gets to power off after the ACPI request.
 func vmShutdownTimeout(inst Instance) time.Duration {
-	if inst.Virtualization == "kvm" {
+	if inst.Virtualization == "kvm" || inst.Virtualization == virtHVF {
 		return 90 * time.Second
 	}
 	return 3 * time.Minute
@@ -697,7 +734,11 @@ func (s *Service) vmPowerOff(inst Instance) bool {
 		return true
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	if _, err := s.env.Docker.Exec(ctx, inst.ContainerID, []string{"/usr/local/bin/hc-vm-ctl", "system_powerdown"}, nil); err != nil {
+	if vmHostMode(inst) {
+		if err := s.vmQMP(ctx, inst, "system_powerdown"); err != nil {
+			log.Printf("ec2: power down %s: %v", inst.ID, err)
+		}
+	} else if _, err := s.env.Docker.Exec(ctx, inst.ContainerID, []string{"/usr/local/bin/hc-vm-ctl", "system_powerdown"}, nil); err != nil {
 		log.Printf("ec2: power down %s: %v", inst.ID, err)
 	}
 	cancel()