diff --git a/cmd/microcloud/ask.go b/cmd/microcloud/ask.go index c346d0149..82b2186fa 100644 --- a/cmd/microcloud/ask.go +++ b/cmd/microcloud/ask.go @@ -332,6 +332,8 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { } } + lxd := sh.Services[types.LXD].(*service.LXDService) + availableDisks := map[string]map[string]api.ResourcesStorageDisk{} for name, state := range c.state { // Skip this system if it already has the local storage pool configured @@ -342,11 +344,17 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { continue } - if len(state.AvailableDisks) == 0 { + // Also offer non-pristine disks as they might contain an existing local storage pool which can be reused. + // This allows recovered systems to set up local storage without wiping their existing storage pool. + disks := make(map[string]api.ResourcesStorageDisk, len(state.AvailableDisks)+len(state.ReusableDisks)) + maps.Copy(disks, state.AvailableDisks) + maps.Copy(disks, state.ReusableDisks) + + if len(disks) == 0 { continue } - availableDisks[name] = state.AvailableDisks + availableDisks[name] = disks } // When adding new systems to a cluster which already has the local storage pool, @@ -398,8 +406,8 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { } } - lxd := sh.Services[types.LXD].(*service.LXDService) toWipe := map[string]string{} + toRecover := map[string]bool{} wipeable, err := lxd.HasExtension(context.Background(), lxd.Name(), lxd.Address(), nil, "storage_pool_source_wipe") if err != nil { return fmt.Errorf("Failed to check for source.wipe extension: %w", err) @@ -435,28 +443,74 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { return errors.New("Failed to add local storage pool: Some peers don't have an available disk") } - if wipeable { - newRows := make([][]string, len(answers)) - for row := range answers { - newRows[row] = make([]string, len(header)) - for j, h := range header { - newRows[row][j] = answers[row][h] + selectedInfo := map[string]api.ResourcesStorageDisk{} + for target, path := range selected { + for _, disk := range availableDisks[target] { + if service.FormatDiskPath(disk) == path { + selectedInfo[target] = disk + break } } + } + + reuseDisks := map[string]bool{} + for _, target := range slices.Sorted(maps.Keys(selected)) { + if service.IsPristineDisk(selectedInfo[target]) { + continue + } - answers, err := table.Render(context.Background(), c.asker, "Select which disks to wipe:", newRows...) + reuse, err := c.asker.AskBool(fmt.Sprintf("Disk %q on %q is non-pristine. Reuse its existing local storage pool?", selected[target], target), true) if err != nil { - return fmt.Errorf("Failed to confirm which disks to wipe: %w", err) + return err } + reuseDisks[target] = reuse + } + + wipe := map[string]string{} + if wipeable { + newRows := make([][]string, 0, len(answers)) for _, entry := range answers { - target := entry["LOCATION"] - path := entry["PATH"] - toWipe[target] = path + // Disks whose existing storage pool is reused must not be wiped. + if reuseDisks[entry["LOCATION"]] { + continue + } + + row := make([]string, len(header)) + for j, h := range header { + row[j] = entry[h] + } + + newRows = append(newRows, row) + } + + if len(newRows) > 0 { + answers, err := table.Render(context.Background(), c.asker, "Select which disks to wipe:", newRows...) + if err != nil { + return fmt.Errorf("Failed to confirm which disks to wipe: %w", err) + } + + for _, entry := range answers { + target := entry["LOCATION"] + path := entry["PATH"] + wipe[target] = path + } + } + } + + for _, target := range slices.Sorted(maps.Keys(selected)) { + if reuseDisks[target] || wipe[target] != "" { + continue + } + + if len(selectedInfo[target].Partitions) != 0 { + return fmt.Errorf("Disk %q on %q has existing partitions. Either reuse its existing local storage pool or wipe it", selected[target], target) } } selectedDisks = selected + toWipe = wipe + toRecover = reuseDisks return nil }) @@ -468,22 +522,18 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { return nil } - if wipeable { - toWipe = selectedDisks - } - var joinConfigs map[string][]api.ClusterMemberConfigKey var finalConfigs []api.StoragePoolsPost var targetConfigs map[string][]api.StoragePoolsPost if useJoinConfig { joinConfigs = map[string][]api.ClusterMemberConfigKey{} for target, path := range selectedDisks { - joinConfigs[target] = lxd.DefaultZFSStoragePoolJoinConfig(wipeable && toWipe[target] != "", path) + joinConfigs[target] = lxd.DefaultZFSStoragePoolJoinConfig(wipeable && toWipe[target] != "", toRecover[target], path) } } else { targetConfigs = map[string][]api.StoragePoolsPost{} for target, path := range selectedDisks { - targetConfigs[target] = []api.StoragePoolsPost{lxd.DefaultPendingZFSStoragePool(wipeable && toWipe[target] != "", path)} + targetConfigs[target] = []api.StoragePoolsPost{lxd.DefaultPendingZFSStoragePool(wipeable && toWipe[target] != "", toRecover[target], path)} } if len(targetConfigs) > 0 { @@ -497,6 +547,11 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { } for target, path := range selectedDisks { + if toRecover[target] { + fmt.Println(tui.SummarizeResult("Reusing existing local storage pool on %s on %s", path, target)) + continue + } + fmt.Println(tui.SummarizeResult("Using %s on %s for local storage pool", path, target)) } @@ -508,7 +563,8 @@ func (c *initConfig) askLocalPool(sh *service.Handler) error { newAvailableDisks := map[string]map[string]api.ResourcesStorageDisk{} for target, path := range selectedDisks { newAvailableDisks[target] = map[string]api.ResourcesStorageDisk{} - for id, disk := range availableDisks[target] { + // Only pass on the available disks so that reusable disks are not offered for remote storage. + for id, disk := range c.state[target].AvailableDisks { if service.FormatDiskPath(disk) != path { newAvailableDisks[target][id] = disk } diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index c130c6c2d..cf643797b 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -63,6 +63,10 @@ type DirectStorage struct { Path string `yaml:"path"` Wipe bool `yaml:"wipe"` Encrypt bool `yaml:"encrypt"` + + // Recover reuses an existing local storage pool on the disk. + // It is only supported for local storage. + Recover bool `yaml:"recover"` } // InitNetwork represents the structure of the network config in the preseed yaml. @@ -401,6 +405,22 @@ func (p *Preseed) validate(name string, bootstrap bool) error { directCephCount++ } + for _, disk := range system.Storage.Ceph { + if disk.Recover { + return fmt.Errorf("Cannot recover Ceph disks of system %q", system.Name) + } + } + + if system.Storage.Local.Recover { + if system.Storage.Local.Path == "" { + return fmt.Errorf("Missing the local storage disk path to recover on system %q", system.Name) + } + + if system.Storage.Local.Wipe { + return fmt.Errorf("Cannot both wipe and recover the local storage disk of system %q", system.Name) + } + } + if system.Storage.Local.Path != "" { directLocalCount++ } @@ -923,12 +943,12 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map // Setup directly specified disks for ZFS pool. if directLocal.Path != "" { if c.bootstrap { - system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(directLocal.Wipe, directLocal.Path)) + system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(directLocal.Wipe, directLocal.Recover, directLocal.Path)) if s.Name == peer { system.StoragePools = append(system.StoragePools, lxd.DefaultZFSStoragePool()) } } else { - system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(directLocal.Wipe, directLocal.Path)...) + system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(directLocal.Wipe, directLocal.Recover, directLocal.Path)...) } directZFSMatches[peer] = directZFSMatches[peer] + 1 @@ -1102,12 +1122,12 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map if len(matched) > 0 { zfsMachines[peer] = true if c.bootstrap { - system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, service.FormatDiskPath(matched[0]))) + system.TargetStoragePools = append(system.TargetStoragePools, lxd.DefaultPendingZFSStoragePool(filter.Wipe, false, service.FormatDiskPath(matched[0]))) if s.Name == peer { system.StoragePools = append(system.StoragePools, lxd.DefaultZFSStoragePool()) } } else { - system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, service.FormatDiskPath(matched[0]))...) + system.JoinConfig = append(system.JoinConfig, lxd.DefaultZFSStoragePoolJoinConfig(filter.Wipe, false, service.FormatDiskPath(matched[0]))...) } zfsMatches[filter.Find] = zfsMatches[filter.Find] + 1 diff --git a/cmd/microcloud/preseed_test.go b/cmd/microcloud/preseed_test.go index 99dd8577e..8cfc1a49a 100644 --- a/cmd/microcloud/preseed_test.go +++ b/cmd/microcloud/preseed_test.go @@ -322,6 +322,64 @@ func (s *preseedSuite) Test_preseedValidateInvalid() { } } +func (s *preseedSuite) Test_preseedValidateRecover() { + systems := func(n2 InitStorage) []System { + return []System{ + {Name: "n1", Address: "1.0.0.1", Storage: InitStorage{Local: DirectStorage{Path: "/dev/sdb"}}}, + {Name: "n2", Address: "1.0.0.2", Storage: n2}, + } + } + + cases := []struct { + desc string + bootstrap bool + storage InitStorage + err error + }{ + { + desc: "Recover local disk when adding", + storage: InitStorage{Local: DirectStorage{Path: "/dev/sdb", Recover: true}}, + err: nil, + }, + { + desc: "Recover local disk when initializing", + bootstrap: true, + storage: InitStorage{Local: DirectStorage{Path: "/dev/sdb", Recover: true}}, + err: nil, + }, + { + desc: "Recover and wipe local disk", + storage: InitStorage{Local: DirectStorage{Path: "/dev/sdb", Recover: true, Wipe: true}}, + err: errors.New(`Cannot both wipe and recover the local storage disk of system "n2"`), + }, + { + desc: "Recover local disk without path", + storage: InitStorage{Local: DirectStorage{Recover: true}}, + err: errors.New(`Missing the local storage disk path to recover on system "n2"`), + }, + { + desc: "Recover Ceph disk", + storage: InitStorage{ + Local: DirectStorage{Path: "/dev/sdb"}, + Ceph: []DirectStorage{{Path: "/dev/sdc", Recover: true}}, + }, + err: errors.New(`Cannot recover Ceph disks of system "n2"`), + }, + } + + for _, c := range cases { + s.T().Log(c.desc) + + p := Preseed{SessionPassphrase: "foo", InitiatorAddress: "1.0.0.1", Systems: systems(c.storage)} + err := p.validate("n1", c.bootstrap) + if c.err == nil { + s.NoError(err) + } else { + s.EqualError(err, c.err.Error()) + } + } +} + func (s *preseedSuite) Test_preseedMatchDisksMemory() { unit1, err := units.ParseByteSizeString("1MiB") s.NoError(err) diff --git a/doc/reference/preseed.yaml b/doc/reference/preseed.yaml index 37aa4e14c..ae46941a0 100644 --- a/doc/reference/preseed.yaml +++ b/doc/reference/preseed.yaml @@ -33,6 +33,8 @@ session_timeout: 300 # `ovn_uplink_interface` is optional and represents the name of the interface reserved for use with OVN. # `ovn_underlay_ip` is optional and represents the Geneve Encap IP for each system. # `storage` is optional and represents explicit paths to disks for each system. +# `local.recover` is optional and reuses the existing local storage pool on a non-pristine disk instead of creating a new one. +# It cannot be combined with `wipe`. systems: - name: micro01 address: 10.0.0.1 diff --git a/doc/reference/requirements.md b/doc/reference/requirements.md index b429bd97c..e2ac8c05e 100644 --- a/doc/reference/requirements.md +++ b/doc/reference/requirements.md @@ -46,6 +46,10 @@ Local storage is fast and convenient. It is commonly used in production and is t The default file system type for local storage is ZFS. +MicroCloud also offers unmounted non-pristine disks for local storage. +A non-pristine disk has existing partitions or a file system, for example a disk that still contains the local storage pool after reinstalling a machine. +If you select a non-pristine disk, MicroCloud asks whether to reuse the existing local storage pool on it instead of creating a new one. + (reference-requirements-storage-distributed)= ### Distributed storage diff --git a/service/lxd_config.go b/service/lxd_config.go index 19c944581..e1d94452a 100644 --- a/service/lxd_config.go +++ b/service/lxd_config.go @@ -174,12 +174,17 @@ func (s LXDService) DefaultOVNNetwork(ipv4Gateway string, ipv4Range string, ipv6 // DefaultPendingZFSStoragePool returns the default local storage configuration when // creating a pending pool on a specific cluster member target. -func (s LXDService) DefaultPendingZFSStoragePool(wipe bool, path string) api.StoragePoolsPost { +// If recoverSource is set, an existing storage pool on the source is reused instead of creating a new one. +func (s LXDService) DefaultPendingZFSStoragePool(wipe bool, recoverSource bool, path string) api.StoragePoolsPost { cfg := map[string]string{"source": path} if wipe { cfg["source.wipe"] = strconv.FormatBool(wipe) } + if recoverSource { + cfg["source.recover"] = strconv.FormatBool(recoverSource) + } + return api.StoragePoolsPost{ Name: DefaultZFSPool, Driver: "zfs", @@ -204,7 +209,8 @@ func (s LXDService) DefaultZFSStoragePool() api.StoragePoolsPost { // DefaultZFSStoragePoolJoinConfig returns the default local storage configuration when // joining an existing cluster. -func (s LXDService) DefaultZFSStoragePoolJoinConfig(wipe bool, path string) []api.ClusterMemberConfigKey { +// If recoverSource is set, an existing storage pool on the source is reused instead of creating a new one. +func (s LXDService) DefaultZFSStoragePoolJoinConfig(wipe bool, recoverSource bool, path string) []api.ClusterMemberConfigKey { wipeDisk := api.ClusterMemberConfigKey{ Entity: "storage-pool", Name: DefaultZFSPool, @@ -212,6 +218,13 @@ func (s LXDService) DefaultZFSStoragePoolJoinConfig(wipe bool, path string) []ap Value: "true", } + recoverDisk := api.ClusterMemberConfigKey{ + Entity: "storage-pool", + Name: DefaultZFSPool, + Key: "source.recover", + Value: "true", + } + sourceTemplate := api.ClusterMemberConfigKey{ Entity: "storage-pool", Name: DefaultZFSPool, @@ -224,6 +237,10 @@ func (s LXDService) DefaultZFSStoragePoolJoinConfig(wipe bool, path string) []ap joinConfig = append(joinConfig, wipeDisk) } + if recoverSource { + joinConfig = append(joinConfig, recoverDisk) + } + return joinConfig } diff --git a/service/system_information.go b/service/system_information.go index a93197ae3..2252f9bcb 100644 --- a/service/system_information.go +++ b/service/system_information.go @@ -28,6 +28,10 @@ type SystemInformation struct { // AvailableDisks is the list of disks available for use on the system. AvailableDisks map[string]api.ResourcesStorageDisk + // ReusableDisks is the list of unmounted non-pristine disks on the system. + // They might contain an existing storage pool which can be recovered. + ReusableDisks map[string]api.ResourcesStorageDisk + // AvailableUplinkInterfaces is the list of networks that can be used for the OVN uplink network. AvailableUplinkInterfaces map[string]api.Network @@ -81,6 +85,7 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul ClusterName: connectInfo.Name, ClusterAddress: connectInfo.Address, AvailableDisks: map[string]api.ResourcesStorageDisk{}, + ReusableDisks: map[string]api.ResourcesStorageDisk{}, AvailableUplinkInterfaces: map[string]api.Network{}, AvailableCephInterfaces: map[string]DedicatedInterface{}, AvailableOVNInterfaces: map[string]DedicatedInterface{}, @@ -125,12 +130,6 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul if allResources != nil { for _, disk := range allResources.Storage.Disks { - // Exclude non-pristine disks with partitions. - // Disks already used for local storage (zfs) contain a partition and are therefore excluded by this check. - if len(disk.Partitions) != 0 { - continue - } - // Exclude cdrom drives as viable storage disk. if disk.Type == "cdrom" { continue @@ -148,6 +147,17 @@ func (sh *Handler) CollectSystemInformation(ctx context.Context, connectInfo mul continue } + // Keep non-pristine disks separately as they might contain an existing storage pool that can be reused. + // Disks already used for local storage (zfs) contain a partition and are therefore non-pristine. + // Mounted non-pristine disks are in use and are excluded entirely. + if !IsPristineDisk(disk) { + if !isDiskMounted(disk) { + s.ReusableDisks[disk.ID] = disk + } + + continue + } + s.AvailableDisks[disk.ID] = disk } } @@ -395,6 +405,26 @@ func ClustersConflict(systems map[string]SystemInformation, services map[types.S return false, "" } +// IsPristineDisk returns true if the disk has neither partitions nor a filesystem and is not mounted. +func IsPristineDisk(disk api.ResourcesStorageDisk) bool { + return disk.DeviceFSUUID == "" && len(disk.Partitions) == 0 && !isDiskMounted(disk) +} + +// isDiskMounted returns true if the disk or any of its partitions is mounted. +func isDiskMounted(disk api.ResourcesStorageDisk) bool { + if disk.Mounted { + return true + } + + for _, partition := range disk.Partitions { + if partition.Mounted { + return true + } + } + + return false +} + // FormatDiskPath returns a disk's path representation. func FormatDiskPath(disk api.ResourcesStorageDisk) string { devicePath := "/dev/" + disk.ID diff --git a/service/system_information_test.go b/service/system_information_test.go new file mode 100644 index 000000000..5bef6150e --- /dev/null +++ b/service/system_information_test.go @@ -0,0 +1,70 @@ +package service + +import ( + "testing" + + "github.com/canonical/lxd/shared/api" + "github.com/stretchr/testify/suite" +) + +type diskSuite struct { + suite.Suite +} + +func TestDiskSuite(t *testing.T) { + suite.Run(t, new(diskSuite)) +} + +func (s *diskSuite) Test_IsPristineDisk() { + cases := []struct { + desc string + disk api.ResourcesStorageDisk + pristine bool + }{ + { + desc: "Empty disk", + disk: api.ResourcesStorageDisk{ID: "sdc"}, + pristine: true, + }, + { + desc: "Disk with a filesystem (btrfs, lvm)", + disk: api.ResourcesStorageDisk{ + ID: "sdd", + DeviceFSUUID: "f391ed2f-b18f-4cae-ae4a-a45059cc42e9", + }, + pristine: false, + }, + { + desc: "Disk with partitions but no filesystem (zfs)", + disk: api.ResourcesStorageDisk{ + ID: "sdb", + Partitions: []api.ResourcesStorageDiskPartition{ + {ID: "sdb1", DeviceFSUUID: "16839297897617386792"}, + {ID: "sdb9"}, + }, + }, + pristine: false, + }, + { + desc: "Mounted disk without filesystem or partitions", + disk: api.ResourcesStorageDisk{ID: "sde", Mounted: true}, + pristine: false, + }, + { + desc: "Disk with a mounted partition", + disk: api.ResourcesStorageDisk{ + ID: "sda", + Partitions: []api.ResourcesStorageDiskPartition{ + {ID: "sda1", Mounted: true}, + }, + }, + pristine: false, + }, + } + + for i, c := range cases { + s.T().Logf("%d: %s", i, c.desc) + + s.Equal(c.pristine, IsPristineDisk(c.disk)) + } +}