diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d92657f..0664ff8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,15 @@ jobs: - name: pivot_root + extraction VM test run: nix build -L .#checks.x86_64-linux.nixos-pivot + # Lifecycle: the machine must survive its own pivot, reboot when the + # entrypoint exits, and refuse a shell entrypoint before doing damage. + # These are the regressions nixos-pivot cannot see. + - name: pivot lifecycle VM tests + run: | + nix build -L .#checks.x86_64-linux.nixos-lifecycle-idle + nix build -L .#checks.x86_64-linux.nixos-lifecycle-exit-reboots + nix build -L .#checks.x86_64-linux.nixos-lifecycle-preflight + goreleaser-check: name: goreleaser check + snapshot smoke test runs-on: ubuntu-latest diff --git a/flake.nix b/flake.nix index e116781..4f4187e 100644 --- a/flake.nix +++ b/flake.nix @@ -174,7 +174,13 @@ }; }; - checks = lib.optionalAttrs pkgs.stdenv.isLinux { + checks = let + lifecycleTests = import ./nix/tests/lifecycle.nix { + inherit pkgs; + lib = pkgs.lib; + xmorph-package = xmorph; + }; + in lib.optionalAttrs pkgs.stdenv.isLinux { # Source build is the canonical sanity check. build = xmorph; @@ -250,6 +256,13 @@ ''; }; + # End-to-end lifecycle: does the machine survive its own pivot? + # nixos-pivot proves pivot_root works; these prove the box is still + # there afterwards, which is the property that actually regressed. + nixos-lifecycle-idle = lifecycleTests.idle-stays-up; + nixos-lifecycle-exit-reboots = lifecycleTests.exit-reboots; + nixos-lifecycle-preflight = lifecycleTests.shell-refused-preflight; + # NixOS VM test: headscale integration (offline, ~2-3 min) nixos-headscale = import ./nix/tests/headscale.nix { inherit pkgs; diff --git a/internal/cli/entrypoint_test.go b/internal/cli/entrypoint_test.go new file mode 100644 index 0000000..5765961 --- /dev/null +++ b/internal/cli/entrypoint_test.go @@ -0,0 +1,105 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/ananthb/xmorph/internal/config" +) + +// The last line of defence before anything destructive happens. A bare shell +// detached from a terminal exits the instant it reads stdin, so pivoting into +// one gives an operator a reboot loop instead of the machine they asked for. +// Catching that here costs nothing: the old root is still mounted and the +// pivot has not started. +// +// go test runs with stdin on /dev/null, which is exactly the detached case +// these tests are about. The converse — a real terminal, where a shell is a +// perfectly sensible entrypoint — cannot be exercised without allocating a +// pty, and is left to the --contain path below. +func TestCheckEntrypointSurvivesDetach(t *testing.T) { + for _, tc := range []struct { + name string + entrypoint string + contain bool + wantErr bool + }{ + {name: "sh", entrypoint: "/bin/sh", wantErr: true}, + {name: "bash", entrypoint: "/bin/bash", wantErr: true}, + {name: "busybox", entrypoint: "/bin/busybox", wantErr: true}, + {name: "absolute path is not what matters", entrypoint: "/usr/local/bin/ash", wantErr: true}, + {name: "a real program is fine", entrypoint: "/usr/local/bin/xmorph"}, + {name: "so is an init", entrypoint: "/sbin/init"}, + // --contain keeps the caller's terminal and never leaves the machine + // without userspace, so a shell there is the normal case. + {name: "contain permits a shell", entrypoint: "/bin/sh", contain: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := checkEntrypointSurvivesDetach( + &config.Config{Contain: tc.contain}, tc.entrypoint) + if tc.wantErr != (err != nil) { + t.Fatalf("checkEntrypointSurvivesDetach(%q, contain=%v) = %v, wantErr %v", + tc.entrypoint, tc.contain, err, tc.wantErr) + } + }) + } +} + +// An error that only says no is a support ticket. This one has to name the way +// forward, because the person reading it is usually mid-rescue on a machine +// they cannot walk over to. +func TestEntrypointRefusalNamesTheFix(t *testing.T) { + err := checkEntrypointSurvivesDetach(&config.Config{}, "/bin/sh") + if err == nil { + t.Fatal("want a refusal for a detached shell") + } + for _, want := range []string{"--cmd idle", "--command"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal does not mention %q: %v", want, err) + } + } +} + +// --ssh.enable with no credentials produced a machine that pivoted, stayed up, +// and could not be logged into: sshd logged "no auth method configured" to a +// console nobody was reading and never bound the port. The VM test found it. +func TestCheckSSHUsable(t *testing.T) { + enabled := true + for _, tc := range []struct { + name string + cfg config.Config + wantErr bool + }{ + {name: "ssh off", cfg: config.Config{}}, + { + name: "enabled with nothing to authenticate with", + cfg: config.Config{SSHEnable: &enabled}, + wantErr: true, + }, + { + name: "password", + cfg: config.Config{SSHEnable: &enabled, SSHPassword: "hunter2"}, + }, + { + name: "authorized keys", + cfg: config.Config{SSHEnable: &enabled, SSHAuthorizedKeys: "ssh-ed25519 AAAA"}, + }, + // SSHEnabled() is implied by any other ssh.* flag, so this is on too. + {name: "implied by password alone", cfg: config.Config{SSHPassword: "hunter2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + err := checkSSHUsable(&tc.cfg) + if tc.wantErr != (err != nil) { + t.Fatalf("checkSSHUsable() = %v, wantErr %v", err, tc.wantErr) + } + if err == nil { + return + } + for _, want := range []string{"--ssh.authorized-keys", "--ssh.password"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal does not mention %q: %v", want, err) + } + } + }) + } +} diff --git a/internal/cli/pivot.go b/internal/cli/pivot.go index 0241c47..18805bb 100644 --- a/internal/cli/pivot.go +++ b/internal/cli/pivot.go @@ -167,6 +167,9 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface { if err := checkEntrypointSurvivesDetach(cfg, entrypoint); err != nil { return err } + if err := checkSSHUsable(cfg); err != nil { + return err + } // Write the postpivot config (read back by `xmorph --init`) and copy // the running binary into the new rootfs. @@ -462,6 +465,30 @@ func checkEntrypointSurvivesDetach(cfg *config.Config, entrypoint string) error entrypoint, postpivot.BinaryPath) } +// checkSSHUsable refuses a pivot that asks for SSH without any way to +// authenticate to it. +// +// The post-pivot sshd needs a password or authorized keys; given neither it +// logs an error and never listens. That log goes to a console nobody is +// reading, on a machine whose whole reason for pivoting was to be reachable — +// so `--ssh.enable` on its own hands back a box that is up, healthy, holding +// itself open, and impossible to get into. Cheaper to say so here. +// +// Found by nix/tests/lifecycle.nix, which is what those tests are for. +func checkSSHUsable(cfg *config.Config) error { + if !cfg.SSHEnabled() { + return nil + } + if cfg.SSHPassword != "" || cfg.SSHAuthorizedKeys != "" { + return nil + } + return errors.New( + "SSH is enabled but has no way to authenticate anyone: sshd will refuse " + + "to start and the pivoted machine will be unreachable.\n" + + " --ssh.authorized-keys '' let a key in\n" + + " --ssh.password '' let a password in") +} + // resolveEntrypoint picks the effective entrypoint + args + env from the // CLI config and the merged ImageConfig. Mirrors src/cmd/pivot.zig:194-236. func resolveEntrypoint(cfg *config.Config, ic *oci.ImageConfig) (entrypoint string, args, env []string) { diff --git a/internal/config/flags.go b/internal/config/flags.go index 8ff11f8..141df8c 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -85,7 +85,10 @@ func bindSSH(fs *pflag.FlagSet, cfg *Config) { fs.Var(&tristateBool{dst: &cfg.SSHEnable}, "ssh.enable", "enable SSH in the new rootfs (auto when other ssh.* set)") fs.Lookup("ssh.enable").NoOptDefVal = "true" fs.Var(&tristateUint16{dst: &cfg.SSHPort}, "ssh.port", "SSH listen port (default 22)") - fs.StringVar(&cfg.SSHPassword, "ssh.password", "", "root password (default: random)") + // No default is generated: sshd needs one of these two or it will not + // start, and the pivot is refused up front rather than leaving an + // unreachable machine behind (see checkSSHUsable). + fs.StringVar(&cfg.SSHPassword, "ssh.password", "", "root password (required unless ssh.authorized-keys is set)") fs.StringVar(&cfg.SSHAuthorizedKeys, "ssh.authorized-keys", "", "authorized public keys (inline)") } diff --git a/internal/postpivot/lifecycle_test.go b/internal/postpivot/lifecycle_test.go new file mode 100644 index 0000000..9d35a7e --- /dev/null +++ b/internal/postpivot/lifecycle_test.go @@ -0,0 +1,169 @@ +package postpivot + +import ( + "os" + "os/signal" + "sync/atomic" + "syscall" + "testing" + "time" +) + +// These tests cover the decision that turned a stumble into a machine needing +// physical access: what the supervisor does when the entrypoint goes away. +// +// The pivot_root syscall was never the problem. It worked perfectly on the +// machine that had to be power-cycled — the kernel came up on the new rootfs, +// the entrypoint exited a moment later, and the supervisor returned, leaving a +// running kernel with no userspace on it. It answered ICMP the entire time. +// +// That failure lives here, in a few branches of plain Go, so it is checked +// here — on every push, in a second, on any platform. The VM tests in +// nix/tests/lifecycle.nix assert the same properties end-to-end; these are the +// ones that will still be running in a year. + +// captureReboot swaps the reboot hook for the duration of a test and reports +// whether it fired. +func captureReboot(t *testing.T) *atomic.Bool { + t.Helper() + var fired atomic.Bool + prev := rebootHook + rebootHook = func(string) { fired.Store(true) } + t.Cleanup(func() { rebootHook = prev }) + return &fired +} + +// A clean exit is the dangerous one. The image default entrypoint is a shell; +// detached, its stdin is /dev/null, it reads EOF and exits 0 immediately. The +// original code rebooted only on failure, so status 0 — the likeliest outcome +// of all — was the one case that bricked the box. +func TestSuperviseRebootsOnCleanExit(t *testing.T) { + fired := captureReboot(t) + + code, err := Supervise(SuperviseOptions{ + Argv: []string{"/bin/sh", "-c", "exit 0"}, + RebootOnExit: true, + }) + if err != nil { + t.Fatalf("Supervise: %v", err) + } + if code != 0 { + t.Errorf("exit code = %d, want 0", code) + } + if !fired.Load() { + t.Error("entrypoint exited 0 and no reboot was triggered: " + + "that leaves a running kernel with no userspace") + } +} + +func TestSuperviseRebootsOnFailedExit(t *testing.T) { + fired := captureReboot(t) + + code, err := Supervise(SuperviseOptions{ + Argv: []string{"/bin/sh", "-c", "exit 3"}, + RebootOnExit: true, + }) + if err != nil { + t.Fatalf("Supervise: %v", err) + } + if code != 3 { + t.Errorf("exit code = %d, want 3", code) + } + if !fired.Load() { + t.Error("entrypoint exited 3 and no reboot was triggered") + } +} + +// --contain and the unit tests above run Supervise on a machine that is not +// pivoted, where rebooting would be spectacularly wrong. The flag has to be +// honoured in both directions. +func TestSuperviseHonoursRebootOnExitFalse(t *testing.T) { + fired := captureReboot(t) + + if _, err := Supervise(SuperviseOptions{ + Argv: []string{"/bin/sh", "-c", "exit 0"}, + RebootOnExit: false, + }); err != nil { + t.Fatalf("Supervise: %v", err) + } + if fired.Load() { + t.Error("rebooted with RebootOnExit false") + } +} + +// An entrypoint that cannot be exec'd must not be mistaken for one that ran +// and exited: the machine never got the userspace it was promised, so it still +// has to go back to the OS on disk. +func TestSuperviseMissingEntrypoint(t *testing.T) { + code, err := Supervise(SuperviseOptions{ + Argv: []string{"/nonexistent/xmorph-test-entrypoint"}, + RebootOnExit: true, + }) + if err == nil { + t.Fatal("want an error for a missing entrypoint, got nil") + } + if code != 127 { + t.Errorf("exit code = %d, want 127", code) + } +} + +// idle is the other half of the fix: the entrypoint for a pivot whose purpose +// is access rather than execution. It has to hold the box up indefinitely, and +// in particular it must not mistake a passing child for a reason to quit — as +// the supervisor it inherits every orphan on the machine. +func TestServeUntilSignalBlocksUntilSignalled(t *testing.T) { + // Claim SIGTERM for the process before anything sends one. Signal + // dispositions are process-wide, so this also guarantees the test does + // not die if our TERM lands before ServeUntilSignal has registered. + guard := make(chan os.Signal, 4) + signal.Notify(guard, syscall.SIGTERM) + defer signal.Stop(guard) + + done := make(chan int, 1) + go func() { done <- ServeUntilSignal() }() + + // A child that comes and goes raises SIGCHLD. Waking on that and + // returning would tear down the SSH server the operator is connected to. + child := exitingChild(t) + select { + case <-done: + t.Fatal("ServeUntilSignal returned when a child exited; " + + "it must only stop on TERM/INT") + case <-time.After(500 * time.Millisecond): + } + _ = child + + // Now the real stop condition. Retry: there is no way to observe the + // moment ServeUntilSignal installs its handler, and a TERM delivered + // before then is absorbed by the guard above. + deadline := time.After(10 * time.Second) + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + for { + if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil { + t.Fatalf("kill: %v", err) + } + select { + case code := <-done: + if code != 0 { + t.Errorf("ServeUntilSignal returned %d, want 0", code) + } + return + case <-deadline: + t.Fatal("ServeUntilSignal did not return after SIGTERM") + case <-tick.C: + } + } +} + +// exitingChild starts and immediately loses a child process, producing the +// SIGCHLD that ServeUntilSignal must ignore. +func exitingChild(t *testing.T) int { + t.Helper() + pid, err := syscall.ForkExec("/bin/sh", []string{"/bin/sh", "-c", "exit 0"}, + &syscall.ProcAttr{Files: []uintptr{0, 1, 2}}) + if err != nil { + t.Skipf("fork/exec unavailable: %v", err) + } + return pid +} diff --git a/internal/postpivot/supervise.go b/internal/postpivot/supervise.go index 660879d..db60b61 100644 --- a/internal/postpivot/supervise.go +++ b/internal/postpivot/supervise.go @@ -107,7 +107,7 @@ func Supervise(opts SuperviseOptions) (exitCode int, err error) { if opts.RebootOnExit { slog.Warn("entrypoint exited; no userspace left, rebooting into the on-disk OS", "code", code) - rebootSystem(opts.OldRootPath) + rebootHook(opts.OldRootPath) } return code, nil } @@ -182,6 +182,17 @@ func exitStatusFrom(cmd *exec.Cmd, waitErr error) int { return 0 } +// rebootHook is what Supervise calls once it decides the machine has to go +// back to the OS on disk. It is a variable so tests can assert on that +// decision: the real implementation never returns, and a test that reboots +// the machine running it is not a test. +// +// "Did we decide to reboot?" is the whole of the bug this indirection exists +// for. A pivot that leaves a kernel with no userspace needs someone on site +// with a power cable, so the decision has to be checked on every push, not +// only when a VM happens to be available. +var rebootHook = rebootSystem + // rebootSystem sleeps 5s (for log flush), then hands off to the // platform doReboot. func rebootSystem(oldRoot string) { diff --git a/nix/tests/lifecycle.nix b/nix/tests/lifecycle.nix new file mode 100644 index 0000000..ca999a0 --- /dev/null +++ b/nix/tests/lifecycle.nix @@ -0,0 +1,194 @@ +# End-to-end lifecycle tests for `xmorph pivot`. +# +# The existing nixos-pivot check runs Go unit tests inside a VM: it proves +# pivot_root and the mount ordering are correct. That is necessary and not +# sufficient. A pivot can execute flawlessly and still leave a machine with a +# running kernel and no userspace — answering ICMP, dead on every port, +# recoverable only by physically power-cycling it. That failure shipped, and +# nixos-pivot passed the whole time, because nothing asserted the machine was +# still there afterwards. +# +# So these tests assert liveness *after* a real pivot_root. Two things about +# the environment decide how, and both are easy to get wrong: +# +# * The NixOS backdoor that machine.succeed() speaks to is a systemd service +# on the guest. It dies with the old root. Nothing on the pivoted machine +# can be asked anything, so the assertions come from a second VM and from +# the serial console. +# +# * The guest is booted with `console=ttyS0 console=tty0`, and the last one +# wins: /dev/console is the graphics console, which the driver never reads. +# wait_for_console_text watches the serial line. Output has to go to +# /dev/ttyS0 by name — writing to /dev/console is silence. +# +# The same properties are asserted far more cheaply by the Go tests in +# internal/postpivot/lifecycle_test.go, which is where a regression will +# actually be caught first. These exist because the Go tests substitute the +# reboot, and at some point someone has to check the real thing. +{ pkgs, lib, xmorph-package }: + +let + # Minimal offline rootfs. No network in CI, so --rootfs with a local tarball + # rather than --image. /bin/sh exists because rootfs verification requires + # one of /sbin/init, /bin/sh, /bin/bash — not because we supervise it. + test-rootfs = pkgs.runCommand "xmorph-lifecycle-rootfs" { + nativeBuildInputs = [ pkgs.gnutar pkgs.gzip ]; + } '' + mkdir -p rootfs/{bin,usr/local/bin,etc,dev,proc,sys,tmp,var/run} + cp ${pkgs.pkgsStatic.busybox}/bin/busybox rootfs/bin/ + for cmd in sh ls cat echo sleep true false; do + ln -sf busybox rootfs/bin/$cmd + done + tar -czf $out -C rootfs . + ''; + + # The machine that gets pivoted. 2G matches nixos-pivot; the pivot builds a + # tmpfs rootfs and the headroom check refuses to start without room for it. + # + # The firewall is off because nftables rules live in the kernel and survive + # pivot_root — leaving them up would block the post-pivot SSH port and make + # a live machine look dead, which is the one distinction being drawn here. + target = { ... }: { + environment.systemPackages = [ xmorph-package ]; + virtualisation.memorySize = 2048; + networking.firewall.enable = false; + }; + + # Launch a pivot the way an operator does: detached, no terminal, output on + # the serial console because every other channel goes away with the old root. + # + # Deliberately one line. This gets interpolated into a Python string literal, + # and a shell `\` continuation would put a raw newline inside that literal, + # which Python rejects before the VM ever boots. + pivotCmd = args: + "setsid xmorph pivot --force --skip-verify --no-init-coord --verbose " + + "--rootfs ${test-rootfs} ${args} > /dev/ttyS0 2>&1 < /dev/null &"; +in +{ + # The headline case: a rescue pivot must leave the machine REACHABLE. + # + # This is the test that would have caught the incident. Before the fix the + # supervised entrypoint was a shell, it read EOF on a detached stdin, exited + # 0, and took the SSH server down with it — while pivot_root itself, and + # therefore nixos-pivot, stayed perfectly green. + # + # "Reachable" has to mean a TCP service answering, not a ping. The bricked + # machine replied to ping for hours; the kernel does that on its own, and it + # is precisely what made the failure so slow to spot. So the assertion is a + # connection to xmorph's own post-pivot SSH server, made from another VM, + # after the machine that would normally answer questions has stopped + # existing. Nothing in NixOS listens on 22 here, so the port is xmorph's + # alone: open means userspace lived through the pivot. + idle-stays-up = pkgs.testers.nixosTest { + name = "xmorph-lifecycle-idle-stays-up"; + nodes = { + inherit target; + prober = { ... }: { + environment.systemPackages = [ pkgs.netcat-openbsd ]; + }; + }; + testScript = '' + start_all() + target.wait_for_unit("multi-user.target") + prober.wait_for_unit("multi-user.target") + + # Nothing is listening yet — otherwise the assertion below proves nothing. + prober.fail("nc -z -w 2 target 22") + + # A password, because sshd will not start without one — nothing here + # authenticates, the assertion is only that the port answers. + target.execute( + "${pivotCmd "--entrypoint /usr/local/bin/xmorph --cmd idle --ssh.password=lifecycle-test"}" + ) + + # xmorph idle logs this once it is holding the box up. + target.wait_for_console_text("serving; no entrypoint to supervise") + + # The real assertion, and the only one that distinguishes this from the + # incident: someone else can still open a connection to it. + prober.wait_until_succeeds("nc -z -w 2 target 22", timeout=120) + + # And it has to KEEP holding. A machine that pivots and then reboots + # moments later is the loop this design exists to avoid. + prober.succeed("sleep 20") + prober.succeed("nc -z -w 2 target 22") + + # Pull the plug, and do it here rather than leaving it to the driver. + # When the script ends the driver runs execute("sync") on every machine + # that is_up(), and execute() calls connect(), which waits on the + # backdoor shell in a loop with no way out. The backdoor died with the + # old root — that is the premise of this whole test — so the run would + # sit there until the global timeout and be scored as a failure with + # every assertion already passed. crash() goes through QMP and needs + # nothing from the guest. + target.crash() + ''; + }; + + # An entrypoint that exits must reboot into the on-disk OS rather than leave + # a kernel with nothing on it. Recovery is directly observable: the machine + # boots again, which also restores the backdoor and lets the driver back in. + # + # allow_reboot is not optional. Without it the VM is started with + # -no-reboot, and the guest rebooting takes QEMU down with it — the correct + # behaviour would be scored as a crash. + exit-reboots = pkgs.testers.nixosTest { + name = "xmorph-lifecycle-exit-reboots"; + nodes.target = target; + testScript = '' + target.start(allow_reboot=True) + target.wait_for_unit("multi-user.target") + first_boot = target.succeed("cat /proc/sys/kernel/random/boot_id").strip() + + # /bin/true exits 0 immediately. Status 0 is the important part: the + # original bug treated a clean exit as success and simply returned, + # which is what turned a stumble into a box needing physical access. + target.execute("${pivotCmd "--entrypoint /bin/true"}") + + target.wait_for_console_text("no userspace left, rebooting") + + # The backdoor went down with the old root; the reboot brings a new one. + target.connected = False + target.wait_for_unit("multi-user.target") + + second_boot = target.succeed("cat /proc/sys/kernel/random/boot_id").strip() + assert first_boot != second_boot, ( + f"boot_id unchanged ({first_boot}); the machine never actually rebooted" + ) + + # Back on the real OS, not still in the pivoted rootfs. + target.succeed("test -d /nix/store") + ''; + }; + + # A bare shell cannot survive being detached, so the pivot must be refused + # before anything destructive happens — while the old root is still intact + # and aborting is free. No pivot happens here, so the backdoor lives and the + # whole thing is ordinary machine.fail(). + shell-refused-preflight = pkgs.testers.nixosTest { + name = "xmorph-lifecycle-shell-refused-preflight"; + nodes.target = target; + testScript = '' + target.wait_for_unit("multi-user.target") + + # No TTY here, which is the whole point: this is how it arrives over SSH. + out = target.fail( + "xmorph pivot --force --skip-verify --no-init-coord " + "--rootfs ${test-rootfs} --entrypoint /bin/sh < /dev/null 2>&1" + ) + assert "shell with no terminal attached" in out, out + assert "--cmd idle" in out, "the error must name the way forward: " + out + + # Nothing may have happened. The machine is still itself, still on the + # original root, still running the init it started with. + target.succeed("test -d /nix/store") + target.succeed("systemctl is-active multi-user.target") + target.succeed("test ! -e /mnt/oldroot/nix") + ''; + }; + + # resolv.conf handling is covered by unit tests in internal/postpivot, which + # can exercise the absent / usable / stub-only / dangling-symlink cases far + # more precisely than a VM can. Deliberately not duplicated here: a VM test + # that only asserts "a file exists" would add minutes to CI and catch less. +}