test(pivot): assert the machine survives its own pivot

.github/workflows/ci.yml

@@ -50,6 +50,15 @@ jobs:
       - name: pivot_root + extraction VM test
         run: nix build -L .#checks.x86_64-linux.nixos-pivot
 
+      # Lifecycle: the machine must survive its own pivot, reboot when the
+      # entrypoint exits, and refuse a shell entrypoint before doing damage.
+      # These are the regressions nixos-pivot cannot see.
+      - name: pivot lifecycle VM tests
+        run: |
+          nix build -L .#checks.x86_64-linux.nixos-lifecycle-idle
+          nix build -L .#checks.x86_64-linux.nixos-lifecycle-exit-reboots
+          nix build -L .#checks.x86_64-linux.nixos-lifecycle-preflight
+
   goreleaser-check:
     name: goreleaser check + snapshot smoke test
     runs-on: ubuntu-latest

flake.nix

@@ -174,7 +174,13 @@
           };
         };
 
-        checks = lib.optionalAttrs pkgs.stdenv.isLinux {
+        checks = let
+          lifecycleTests = import ./nix/tests/lifecycle.nix {
+            inherit pkgs;
+            lib = pkgs.lib;
+            xmorph-package = xmorph;
+          };
+        in lib.optionalAttrs pkgs.stdenv.isLinux {
           # Source build is the canonical sanity check.
           build = xmorph;
 
@@ -250,6 +256,13 @@
             '';
           };
 
+          # End-to-end lifecycle: does the machine survive its own pivot?
+          # nixos-pivot proves pivot_root works; these prove the box is still
+          # there afterwards, which is the property that actually regressed.
+          nixos-lifecycle-idle = lifecycleTests.idle-stays-up;
+          nixos-lifecycle-exit-reboots = lifecycleTests.exit-reboots;
+          nixos-lifecycle-preflight = lifecycleTests.shell-refused-preflight;
+
           # NixOS VM test: headscale integration (offline, ~2-3 min)
           nixos-headscale = import ./nix/tests/headscale.nix {
             inherit pkgs;

internal/cli/entrypoint_test.go

@@ -0,0 +1,105 @@
+package cli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/ananthb/xmorph/internal/config"
+)
+
+// The last line of defence before anything destructive happens. A bare shell
+// detached from a terminal exits the instant it reads stdin, so pivoting into
+// one gives an operator a reboot loop instead of the machine they asked for.
+// Catching that here costs nothing: the old root is still mounted and the
+// pivot has not started.
+//
+// go test runs with stdin on /dev/null, which is exactly the detached case
+// these tests are about. The converse — a real terminal, where a shell is a
+// perfectly sensible entrypoint — cannot be exercised without allocating a
+// pty, and is left to the --contain path below.
+func TestCheckEntrypointSurvivesDetach(t *testing.T) {
+	for _, tc := range []struct {
+		name       string
+		entrypoint string
+		contain    bool
+		wantErr    bool
+	}{
+		{name: "sh", entrypoint: "/bin/sh", wantErr: true},
+		{name: "bash", entrypoint: "/bin/bash", wantErr: true},
+		{name: "busybox", entrypoint: "/bin/busybox", wantErr: true},
+		{name: "absolute path is not what matters", entrypoint: "/usr/local/bin/ash", wantErr: true},
+		{name: "a real program is fine", entrypoint: "/usr/local/bin/xmorph"},
+		{name: "so is an init", entrypoint: "/sbin/init"},
+		// --contain keeps the caller's terminal and never leaves the machine
+		// without userspace, so a shell there is the normal case.
+		{name: "contain permits a shell", entrypoint: "/bin/sh", contain: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkEntrypointSurvivesDetach(
+				&config.Config{Contain: tc.contain}, tc.entrypoint)
+			if tc.wantErr != (err != nil) {
+				t.Fatalf("checkEntrypointSurvivesDetach(%q, contain=%v) = %v, wantErr %v",
+					tc.entrypoint, tc.contain, err, tc.wantErr)
+			}
+		})
+	}
+}
+
+// An error that only says no is a support ticket. This one has to name the way
+// forward, because the person reading it is usually mid-rescue on a machine
+// they cannot walk over to.
+func TestEntrypointRefusalNamesTheFix(t *testing.T) {
+	err := checkEntrypointSurvivesDetach(&config.Config{}, "/bin/sh")
+	if err == nil {
+		t.Fatal("want a refusal for a detached shell")
+	}
+	for _, want := range []string{"--cmd idle", "--command"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("refusal does not mention %q: %v", want, err)
+		}
+	}
+}
+
+// --ssh.enable with no credentials produced a machine that pivoted, stayed up,
+// and could not be logged into: sshd logged "no auth method configured" to a
+// console nobody was reading and never bound the port. The VM test found it.
+func TestCheckSSHUsable(t *testing.T) {
+	enabled := true
+	for _, tc := range []struct {
+		name    string
+		cfg     config.Config
+		wantErr bool
+	}{
+		{name: "ssh off", cfg: config.Config{}},
+		{
+			name:    "enabled with nothing to authenticate with",
+			cfg:     config.Config{SSHEnable: &enabled},
+			wantErr: true,
+		},
+		{
+			name: "password",
+			cfg:  config.Config{SSHEnable: &enabled, SSHPassword: "hunter2"},
+		},
+		{
+			name: "authorized keys",
+			cfg:  config.Config{SSHEnable: &enabled, SSHAuthorizedKeys: "ssh-ed25519 AAAA"},
+		},
+		// SSHEnabled() is implied by any other ssh.* flag, so this is on too.
+		{name: "implied by password alone", cfg: config.Config{SSHPassword: "hunter2"}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			err := checkSSHUsable(&tc.cfg)
+			if tc.wantErr != (err != nil) {
+				t.Fatalf("checkSSHUsable() = %v, wantErr %v", err, tc.wantErr)
+			}
+			if err == nil {
+				return
+			}
+			for _, want := range []string{"--ssh.authorized-keys", "--ssh.password"} {
+				if !strings.Contains(err.Error(), want) {
+					t.Errorf("refusal does not mention %q: %v", want, err)
+				}
+			}
+		})
+	}
+}

internal/cli/pivot.go

@@ -167,6 +167,9 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface {
 	if err := checkEntrypointSurvivesDetach(cfg, entrypoint); err != nil {
 		return err
 	}
+	if err := checkSSHUsable(cfg); err != nil {
+		return err
+	}
 
 	// Write the postpivot config (read back by `xmorph --init`) and copy
 	// the running binary into the new rootfs.
@@ -462,6 +465,30 @@ func checkEntrypointSurvivesDetach(cfg *config.Config, entrypoint string) error
 		entrypoint, postpivot.BinaryPath)
 }
 
+// checkSSHUsable refuses a pivot that asks for SSH without any way to
+// authenticate to it.
+//
+// The post-pivot sshd needs a password or authorized keys; given neither it
+// logs an error and never listens. That log goes to a console nobody is
+// reading, on a machine whose whole reason for pivoting was to be reachable —
+// so `--ssh.enable` on its own hands back a box that is up, healthy, holding
+// itself open, and impossible to get into. Cheaper to say so here.
+//
+// Found by nix/tests/lifecycle.nix, which is what those tests are for.
+func checkSSHUsable(cfg *config.Config) error {
+	if !cfg.SSHEnabled() {
+		return nil
+	}
+	if cfg.SSHPassword != "" || cfg.SSHAuthorizedKeys != "" {
+		return nil
+	}
+	return errors.New(
+		"SSH is enabled but has no way to authenticate anyone: sshd will refuse " +
+			"to start and the pivoted machine will be unreachable.\n" +
+			"  --ssh.authorized-keys '<pubkey>'   let a key in\n" +
+			"  --ssh.password '<password>'        let a password in")
+}
+
 // resolveEntrypoint picks the effective entrypoint + args + env from the
 // CLI config and the merged ImageConfig. Mirrors src/cmd/pivot.zig:194-236.
 func resolveEntrypoint(cfg *config.Config, ic *oci.ImageConfig) (entrypoint string, args, env []string) {

internal/config/flags.go

@@ -85,7 +85,10 @@ func bindSSH(fs *pflag.FlagSet, cfg *Config) {
 	fs.Var(&tristateBool{dst: &cfg.SSHEnable}, "ssh.enable", "enable SSH in the new rootfs (auto when other ssh.* set)")
 	fs.Lookup("ssh.enable").NoOptDefVal = "true"
 	fs.Var(&tristateUint16{dst: &cfg.SSHPort}, "ssh.port", "SSH listen port (default 22)")
-	fs.StringVar(&cfg.SSHPassword, "ssh.password", "", "root password (default: random)")
+	// No default is generated: sshd needs one of these two or it will not
+	// start, and the pivot is refused up front rather than leaving an
+	// unreachable machine behind (see checkSSHUsable).
+	fs.StringVar(&cfg.SSHPassword, "ssh.password", "", "root password (required unless ssh.authorized-keys is set)")
 	fs.StringVar(&cfg.SSHAuthorizedKeys, "ssh.authorized-keys", "", "authorized public keys (inline)")
 }
 

internal/postpivot/lifecycle_test.go

@@ -0,0 +1,169 @@
+package postpivot
+
+import (
+	"os"
+	"os/signal"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+)
+
+// These tests cover the decision that turned a stumble into a machine needing
+// physical access: what the supervisor does when the entrypoint goes away.
+//
+// The pivot_root syscall was never the problem. It worked perfectly on the
+// machine that had to be power-cycled — the kernel came up on the new rootfs,
+// the entrypoint exited a moment later, and the supervisor returned, leaving a
+// running kernel with no userspace on it. It answered ICMP the entire time.
+//
+// That failure lives here, in a few branches of plain Go, so it is checked
+// here — on every push, in a second, on any platform. The VM tests in
+// nix/tests/lifecycle.nix assert the same properties end-to-end; these are the
+// ones that will still be running in a year.
+
+// captureReboot swaps the reboot hook for the duration of a test and reports
+// whether it fired.
+func captureReboot(t *testing.T) *atomic.Bool {
+	t.Helper()
+	var fired atomic.Bool
+	prev := rebootHook
+	rebootHook = func(string) { fired.Store(true) }
+	t.Cleanup(func() { rebootHook = prev })
+	return &fired
+}
+
+// A clean exit is the dangerous one. The image default entrypoint is a shell;
+// detached, its stdin is /dev/null, it reads EOF and exits 0 immediately. The
+// original code rebooted only on failure, so status 0 — the likeliest outcome
+// of all — was the one case that bricked the box.
+func TestSuperviseRebootsOnCleanExit(t *testing.T) {
+	fired := captureReboot(t)
+
+	code, err := Supervise(SuperviseOptions{
+		Argv:         []string{"/bin/sh", "-c", "exit 0"},
+		RebootOnExit: true,
+	})
+	if err != nil {
+		t.Fatalf("Supervise: %v", err)
+	}
+	if code != 0 {
+		t.Errorf("exit code = %d, want 0", code)
+	}
+	if !fired.Load() {
+		t.Error("entrypoint exited 0 and no reboot was triggered: " +
+			"that leaves a running kernel with no userspace")
+	}
+}
+
+func TestSuperviseRebootsOnFailedExit(t *testing.T) {
+	fired := captureReboot(t)
+
+	code, err := Supervise(SuperviseOptions{
+		Argv:         []string{"/bin/sh", "-c", "exit 3"},
+		RebootOnExit: true,
+	})
+	if err != nil {
+		t.Fatalf("Supervise: %v", err)
+	}
+	if code != 3 {
+		t.Errorf("exit code = %d, want 3", code)
+	}
+	if !fired.Load() {
+		t.Error("entrypoint exited 3 and no reboot was triggered")
+	}
+}
+
+// --contain and the unit tests above run Supervise on a machine that is not
+// pivoted, where rebooting would be spectacularly wrong. The flag has to be
+// honoured in both directions.
+func TestSuperviseHonoursRebootOnExitFalse(t *testing.T) {
+	fired := captureReboot(t)
+
+	if _, err := Supervise(SuperviseOptions{
+		Argv:         []string{"/bin/sh", "-c", "exit 0"},
+		RebootOnExit: false,
+	}); err != nil {
+		t.Fatalf("Supervise: %v", err)
+	}
+	if fired.Load() {
+		t.Error("rebooted with RebootOnExit false")
+	}
+}
+
+// An entrypoint that cannot be exec'd must not be mistaken for one that ran
+// and exited: the machine never got the userspace it was promised, so it still
+// has to go back to the OS on disk.
+func TestSuperviseMissingEntrypoint(t *testing.T) {
+	code, err := Supervise(SuperviseOptions{
+		Argv:         []string{"/nonexistent/xmorph-test-entrypoint"},
+		RebootOnExit: true,
+	})
+	if err == nil {
+		t.Fatal("want an error for a missing entrypoint, got nil")
+	}
+	if code != 127 {
+		t.Errorf("exit code = %d, want 127", code)
+	}
+}
+
+// idle is the other half of the fix: the entrypoint for a pivot whose purpose
+// is access rather than execution. It has to hold the box up indefinitely, and
+// in particular it must not mistake a passing child for a reason to quit — as
+// the supervisor it inherits every orphan on the machine.
+func TestServeUntilSignalBlocksUntilSignalled(t *testing.T) {
+	// Claim SIGTERM for the process before anything sends one. Signal
+	// dispositions are process-wide, so this also guarantees the test does
+	// not die if our TERM lands before ServeUntilSignal has registered.
+	guard := make(chan os.Signal, 4)
+	signal.Notify(guard, syscall.SIGTERM)
+	defer signal.Stop(guard)
+
+	done := make(chan int, 1)
+	go func() { done <- ServeUntilSignal() }()
+
+	// A child that comes and goes raises SIGCHLD. Waking on that and
+	// returning would tear down the SSH server the operator is connected to.
+	child := exitingChild(t)
+	select {
+	case <-done:
+		t.Fatal("ServeUntilSignal returned when a child exited; " +
+			"it must only stop on TERM/INT")
+	case <-time.After(500 * time.Millisecond):
+	}
+	_ = child
+
+	// Now the real stop condition. Retry: there is no way to observe the
+	// moment ServeUntilSignal installs its handler, and a TERM delivered
+	// before then is absorbed by the guard above.
+	deadline := time.After(10 * time.Second)
+	tick := time.NewTicker(50 * time.Millisecond)
+	defer tick.Stop()
+	for {
+		if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil {
+			t.Fatalf("kill: %v", err)
+		}
+		select {
+		case code := <-done:
+			if code != 0 {
+				t.Errorf("ServeUntilSignal returned %d, want 0", code)
+			}
+			return
+		case <-deadline:
+			t.Fatal("ServeUntilSignal did not return after SIGTERM")
+		case <-tick.C:
+		}
+	}
+}
+
+// exitingChild starts and immediately loses a child process, producing the
+// SIGCHLD that ServeUntilSignal must ignore.
+func exitingChild(t *testing.T) int {
+	t.Helper()
+	pid, err := syscall.ForkExec("/bin/sh", []string{"/bin/sh", "-c", "exit 0"},
+		&syscall.ProcAttr{Files: []uintptr{0, 1, 2}})
+	if err != nil {
+		t.Skipf("fork/exec unavailable: %v", err)
+	}
+	return pid
+}