diff --git a/api/disk/disk.go b/api/disk/disk.go new file mode 100644 index 000000000..dbca1b572 --- /dev/null +++ b/api/disk/disk.go @@ -0,0 +1,4 @@ +package disk + +//go:generate mkdir -p disk_v1alpha +//go:generate go run ../../pkg/rpc/cmd/rpcgen -pkg disk_v1alpha -input rpc.yml -output disk_v1alpha/rpc.gen.go diff --git a/api/disk/disk_v1alpha/rpc.gen.go b/api/disk/disk_v1alpha/rpc.gen.go new file mode 100644 index 000000000..b2987f41f --- /dev/null +++ b/api/disk/disk_v1alpha/rpc.gen.go @@ -0,0 +1,1745 @@ +package disk_v1alpha + +import ( + "context" + "encoding/json" + "slices" + + "github.com/fxamacker/cbor/v2" + rpc "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/standard" + "miren.dev/runtime/pkg/rpc/stream" +) + +type transferData struct { + Done *int64 `cbor:"0,keyasint,omitempty" json:"done,omitempty"` + Total *int64 `cbor:"1,keyasint,omitempty" json:"total,omitempty"` + BytesPerSecond *int64 `cbor:"2,keyasint,omitempty" json:"bytes_per_second,omitempty"` + EtaSeconds *int64 `cbor:"3,keyasint,omitempty" json:"eta_seconds,omitempty"` +} + +type Transfer struct { + data transferData +} + +func (v *Transfer) HasDone() bool { + return v.data.Done != nil +} + +func (v *Transfer) Done() int64 { + if v.data.Done == nil { + return 0 + } + return *v.data.Done +} + +func (v *Transfer) SetDone(done int64) { + v.data.Done = &done +} + +func (v *Transfer) HasTotal() bool { + return v.data.Total != nil +} + +func (v *Transfer) Total() int64 { + if v.data.Total == nil { + return 0 + } + return *v.data.Total +} + +func (v *Transfer) SetTotal(total int64) { + v.data.Total = &total +} + +func (v *Transfer) HasBytesPerSecond() bool { + return v.data.BytesPerSecond != nil +} + +func (v *Transfer) BytesPerSecond() int64 { + if v.data.BytesPerSecond == nil { + return 0 + } + return *v.data.BytesPerSecond +} + +func (v *Transfer) SetBytesPerSecond(bytes_per_second int64) { + v.data.BytesPerSecond = &bytes_per_second +} + +func (v *Transfer) HasEtaSeconds() bool { + return v.data.EtaSeconds != nil +} + +func (v *Transfer) EtaSeconds() int64 { + if v.data.EtaSeconds == nil { + return 0 + } + return *v.data.EtaSeconds +} + +func (v *Transfer) SetEtaSeconds(eta_seconds int64) { + v.data.EtaSeconds = &eta_seconds +} + +func (v *Transfer) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *Transfer) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *Transfer) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *Transfer) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type ProgressUpdate interface { + Which() string + Message() string + SetMessage(string) + Transfer() *Transfer + SetTransfer(*Transfer) + Warning() string + SetWarning(string) + Error() string + SetError(string) +} + +type progressUpdate struct { + U_Message *string `cbor:"1,keyasint,omitempty" json:"message,omitempty"` + U_Transfer **Transfer `cbor:"2,keyasint,omitempty" json:"transfer,omitempty"` + U_Warning *string `cbor:"3,keyasint,omitempty" json:"warning,omitempty"` + U_Error *string `cbor:"4,keyasint,omitempty" json:"error,omitempty"` +} + +func (v *progressUpdate) Which() string { + if v.U_Message != nil { + return "message" + } + if v.U_Transfer != nil { + return "transfer" + } + if v.U_Warning != nil { + return "warning" + } + if v.U_Error != nil { + return "error" + } + return "" +} + +func (v *progressUpdate) Message() string { + if v.U_Message == nil { + return "" + } + return *v.U_Message +} + +func (v *progressUpdate) SetMessage(val string) { + v.U_Transfer = nil + v.U_Warning = nil + v.U_Error = nil + v.U_Message = &val +} + +func (v *progressUpdate) Transfer() *Transfer { + if v.U_Transfer == nil { + return nil + } + return *v.U_Transfer +} + +func (v *progressUpdate) SetTransfer(val *Transfer) { + v.U_Message = nil + v.U_Warning = nil + v.U_Error = nil + v.U_Transfer = &val +} + +func (v *progressUpdate) Warning() string { + if v.U_Warning == nil { + return "" + } + return *v.U_Warning +} + +func (v *progressUpdate) SetWarning(val string) { + v.U_Message = nil + v.U_Transfer = nil + v.U_Error = nil + v.U_Warning = &val +} + +func (v *progressUpdate) Error() string { + if v.U_Error == nil { + return "" + } + return *v.U_Error +} + +func (v *progressUpdate) SetError(val string) { + v.U_Message = nil + v.U_Transfer = nil + v.U_Warning = nil + v.U_Error = &val +} + +type progressData struct { + progressUpdate +} + +type Progress struct { + data progressData +} + +func (v *Progress) Update() ProgressUpdate { + return &v.data.progressUpdate +} + +func (v *Progress) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *Progress) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *Progress) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *Progress) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type restorePointData struct { + Id *string `cbor:"0,keyasint,omitempty" json:"id,omitempty"` + CreatedAt *standard.Timestamp `cbor:"1,keyasint,omitempty" json:"created_at,omitempty"` + SizeBytes *int64 `cbor:"2,keyasint,omitempty" json:"size_bytes,omitempty"` + ImageSizeBytes *int64 `cbor:"3,keyasint,omitempty" json:"image_size_bytes,omitempty"` + Name *string `cbor:"4,keyasint,omitempty" json:"name,omitempty"` + Mode *string `cbor:"5,keyasint,omitempty" json:"mode,omitempty"` +} + +type RestorePoint struct { + data restorePointData +} + +func (v *RestorePoint) HasId() bool { + return v.data.Id != nil +} + +func (v *RestorePoint) Id() string { + if v.data.Id == nil { + return "" + } + return *v.data.Id +} + +func (v *RestorePoint) SetId(id string) { + v.data.Id = &id +} + +func (v *RestorePoint) HasCreatedAt() bool { + return v.data.CreatedAt != nil +} + +func (v *RestorePoint) CreatedAt() *standard.Timestamp { + return v.data.CreatedAt +} + +func (v *RestorePoint) SetCreatedAt(created_at *standard.Timestamp) { + v.data.CreatedAt = created_at +} + +func (v *RestorePoint) HasSizeBytes() bool { + return v.data.SizeBytes != nil +} + +func (v *RestorePoint) SizeBytes() int64 { + if v.data.SizeBytes == nil { + return 0 + } + return *v.data.SizeBytes +} + +func (v *RestorePoint) SetSizeBytes(size_bytes int64) { + v.data.SizeBytes = &size_bytes +} + +func (v *RestorePoint) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *RestorePoint) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *RestorePoint) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *RestorePoint) HasName() bool { + return v.data.Name != nil +} + +func (v *RestorePoint) Name() string { + if v.data.Name == nil { + return "" + } + return *v.data.Name +} + +func (v *RestorePoint) SetName(name string) { + v.data.Name = &name +} + +func (v *RestorePoint) HasMode() bool { + return v.data.Mode != nil +} + +func (v *RestorePoint) Mode() string { + if v.data.Mode == nil { + return "" + } + return *v.data.Mode +} + +func (v *RestorePoint) SetMode(mode string) { + v.data.Mode = &mode +} + +func (v *RestorePoint) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RestorePoint) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RestorePoint) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RestorePoint) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type backupResultData struct { + ImageSizeBytes *int64 `cbor:"0,keyasint,omitempty" json:"image_size_bytes,omitempty"` + CompressedSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"compressed_size_bytes,omitempty"` + Checksum *string `cbor:"2,keyasint,omitempty" json:"checksum,omitempty"` + RestorePointId *string `cbor:"3,keyasint,omitempty" json:"restore_point_id,omitempty"` +} + +type BackupResult struct { + data backupResultData +} + +func (v *BackupResult) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *BackupResult) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *BackupResult) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *BackupResult) HasCompressedSizeBytes() bool { + return v.data.CompressedSizeBytes != nil +} + +func (v *BackupResult) CompressedSizeBytes() int64 { + if v.data.CompressedSizeBytes == nil { + return 0 + } + return *v.data.CompressedSizeBytes +} + +func (v *BackupResult) SetCompressedSizeBytes(compressed_size_bytes int64) { + v.data.CompressedSizeBytes = &compressed_size_bytes +} + +func (v *BackupResult) HasChecksum() bool { + return v.data.Checksum != nil +} + +func (v *BackupResult) Checksum() string { + if v.data.Checksum == nil { + return "" + } + return *v.data.Checksum +} + +func (v *BackupResult) SetChecksum(checksum string) { + v.data.Checksum = &checksum +} + +func (v *BackupResult) HasRestorePointId() bool { + return v.data.RestorePointId != nil +} + +func (v *BackupResult) RestorePointId() string { + if v.data.RestorePointId == nil { + return "" + } + return *v.data.RestorePointId +} + +func (v *BackupResult) SetRestorePointId(restore_point_id string) { + v.data.RestorePointId = &restore_point_id +} + +func (v *BackupResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *BackupResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *BackupResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *BackupResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type deletedDiskData struct { + DiskName *string `cbor:"0,keyasint,omitempty" json:"disk_name,omitempty"` + VolumeId *string `cbor:"1,keyasint,omitempty" json:"volume_id,omitempty"` + SizeGb *int64 `cbor:"2,keyasint,omitempty" json:"size_gb,omitempty"` + Filesystem *string `cbor:"3,keyasint,omitempty" json:"filesystem,omitempty"` + DeletedAt *standard.Timestamp `cbor:"4,keyasint,omitempty" json:"deleted_at,omitempty"` + ExpiresAt *standard.Timestamp `cbor:"5,keyasint,omitempty" json:"expires_at,omitempty"` + VolumeMode *string `cbor:"6,keyasint,omitempty" json:"volume_mode,omitempty"` +} + +type DeletedDisk struct { + data deletedDiskData +} + +func (v *DeletedDisk) HasDiskName() bool { + return v.data.DiskName != nil +} + +func (v *DeletedDisk) DiskName() string { + if v.data.DiskName == nil { + return "" + } + return *v.data.DiskName +} + +func (v *DeletedDisk) SetDiskName(disk_name string) { + v.data.DiskName = &disk_name +} + +func (v *DeletedDisk) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *DeletedDisk) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *DeletedDisk) SetVolumeId(volume_id string) { + v.data.VolumeId = &volume_id +} + +func (v *DeletedDisk) HasSizeGb() bool { + return v.data.SizeGb != nil +} + +func (v *DeletedDisk) SizeGb() int64 { + if v.data.SizeGb == nil { + return 0 + } + return *v.data.SizeGb +} + +func (v *DeletedDisk) SetSizeGb(size_gb int64) { + v.data.SizeGb = &size_gb +} + +func (v *DeletedDisk) HasFilesystem() bool { + return v.data.Filesystem != nil +} + +func (v *DeletedDisk) Filesystem() string { + if v.data.Filesystem == nil { + return "" + } + return *v.data.Filesystem +} + +func (v *DeletedDisk) SetFilesystem(filesystem string) { + v.data.Filesystem = &filesystem +} + +func (v *DeletedDisk) HasDeletedAt() bool { + return v.data.DeletedAt != nil +} + +func (v *DeletedDisk) DeletedAt() *standard.Timestamp { + return v.data.DeletedAt +} + +func (v *DeletedDisk) SetDeletedAt(deleted_at *standard.Timestamp) { + v.data.DeletedAt = deleted_at +} + +func (v *DeletedDisk) HasExpiresAt() bool { + return v.data.ExpiresAt != nil +} + +func (v *DeletedDisk) ExpiresAt() *standard.Timestamp { + return v.data.ExpiresAt +} + +func (v *DeletedDisk) SetExpiresAt(expires_at *standard.Timestamp) { + v.data.ExpiresAt = expires_at +} + +func (v *DeletedDisk) HasVolumeMode() bool { + return v.data.VolumeMode != nil +} + +func (v *DeletedDisk) VolumeMode() string { + if v.data.VolumeMode == nil { + return "" + } + return *v.data.VolumeMode +} + +func (v *DeletedDisk) SetVolumeMode(volume_mode string) { + v.data.VolumeMode = &volume_mode +} + +func (v *DeletedDisk) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DeletedDisk) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DeletedDisk) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DeletedDisk) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type undeleteResultData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + DiskId *string `cbor:"1,keyasint,omitempty" json:"disk_id,omitempty"` + VolumeId *string `cbor:"2,keyasint,omitempty" json:"volume_id,omitempty"` + ImagePath *string `cbor:"3,keyasint,omitempty" json:"image_path,omitempty"` +} + +type UndeleteResult struct { + data undeleteResultData +} + +func (v *UndeleteResult) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *UndeleteResult) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *UndeleteResult) SetDisk(disk string) { + v.data.Disk = &disk +} + +func (v *UndeleteResult) HasDiskId() bool { + return v.data.DiskId != nil +} + +func (v *UndeleteResult) DiskId() string { + if v.data.DiskId == nil { + return "" + } + return *v.data.DiskId +} + +func (v *UndeleteResult) SetDiskId(disk_id string) { + v.data.DiskId = &disk_id +} + +func (v *UndeleteResult) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *UndeleteResult) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *UndeleteResult) SetVolumeId(volume_id string) { + v.data.VolumeId = &volume_id +} + +func (v *UndeleteResult) HasImagePath() bool { + return v.data.ImagePath != nil +} + +func (v *UndeleteResult) ImagePath() string { + if v.data.ImagePath == nil { + return "" + } + return *v.data.ImagePath +} + +func (v *UndeleteResult) SetImagePath(image_path string) { + v.data.ImagePath = &image_path +} + +func (v *UndeleteResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *UndeleteResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *UndeleteResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *UndeleteResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type restoreResultData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + ImageSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"image_size_bytes,omitempty"` + Created *bool `cbor:"2,keyasint,omitempty" json:"created,omitempty"` +} + +type RestoreResult struct { + data restoreResultData +} + +func (v *RestoreResult) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *RestoreResult) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *RestoreResult) SetDisk(disk string) { + v.data.Disk = &disk +} + +func (v *RestoreResult) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *RestoreResult) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *RestoreResult) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *RestoreResult) HasCreated() bool { + return v.data.Created != nil +} + +func (v *RestoreResult) Created() bool { + if v.data.Created == nil { + return false + } + return *v.data.Created +} + +func (v *RestoreResult) SetCreated(created bool) { + v.data.Created = &created +} + +func (v *RestoreResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RestoreResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RestoreResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RestoreResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupBackupArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + ToCloud *bool `cbor:"1,keyasint,omitempty" json:"to_cloud,omitempty"` + Pin *string `cbor:"2,keyasint,omitempty" json:"pin,omitempty"` + Data *rpc.Capability `cbor:"3,keyasint,omitempty" json:"data,omitempty"` + Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` + TransferId *string `cbor:"5,keyasint,omitempty" json:"transfer_id,omitempty"` + Offset *int64 `cbor:"6,keyasint,omitempty" json:"offset,omitempty"` +} + +type DiskBackupBackupArgs struct { + call rpc.Call + data diskBackupBackupArgsData +} + +func (v *DiskBackupBackupArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupBackupArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupBackupArgs) HasToCloud() bool { + return v.data.ToCloud != nil +} + +func (v *DiskBackupBackupArgs) ToCloud() bool { + if v.data.ToCloud == nil { + return false + } + return *v.data.ToCloud +} + +func (v *DiskBackupBackupArgs) HasPin() bool { + return v.data.Pin != nil +} + +func (v *DiskBackupBackupArgs) Pin() string { + if v.data.Pin == nil { + return "" + } + return *v.data.Pin +} + +func (v *DiskBackupBackupArgs) HasData() bool { + return v.data.Data != nil +} + +func (v *DiskBackupBackupArgs) Data() *stream.SendStreamClient[[]byte] { + if v.data.Data == nil { + return nil + } + return &stream.SendStreamClient[[]byte]{Client: v.call.NewClient(v.data.Data)} +} + +func (v *DiskBackupBackupArgs) HasProgress() bool { + return v.data.Progress != nil +} + +func (v *DiskBackupBackupArgs) Progress() *stream.SendStreamClient[*Progress] { + if v.data.Progress == nil { + return nil + } + return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} +} + +func (v *DiskBackupBackupArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupBackupArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupBackupArgs) HasOffset() bool { + return v.data.Offset != nil +} + +func (v *DiskBackupBackupArgs) Offset() int64 { + if v.data.Offset == nil { + return 0 + } + return *v.data.Offset +} + +func (v *DiskBackupBackupArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupBackupArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupBackupArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupBackupArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupBackupResultsData struct { + Result **BackupResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupBackupResults struct { + call rpc.Call + data diskBackupBackupResultsData +} + +func (v *DiskBackupBackupResults) SetResult(result **BackupResult) { + v.data.Result = result +} + +func (v *DiskBackupBackupResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupBackupResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupBackupResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupBackupResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListBackupsArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` +} + +type DiskBackupListBackupsArgs struct { + call rpc.Call + data diskBackupListBackupsArgsData +} + +func (v *DiskBackupListBackupsArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupListBackupsArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupListBackupsArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListBackupsArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListBackupsArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListBackupsArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListBackupsResultsData struct { + Points *[]*RestorePoint `cbor:"0,keyasint,omitempty" json:"points,omitempty"` +} + +type DiskBackupListBackupsResults struct { + call rpc.Call + data diskBackupListBackupsResultsData +} + +func (v *DiskBackupListBackupsResults) SetPoints(points []*RestorePoint) { + x := slices.Clone(points) + v.data.Points = &x +} + +func (v *DiskBackupListBackupsResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListBackupsResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListBackupsResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListBackupsResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupRestoreArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + RestorePoint *string `cbor:"1,keyasint,omitempty" json:"restore_point,omitempty"` + Data *rpc.Capability `cbor:"2,keyasint,omitempty" json:"data,omitempty"` + Force *bool `cbor:"3,keyasint,omitempty" json:"force,omitempty"` + Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` + TransferId *string `cbor:"5,keyasint,omitempty" json:"transfer_id,omitempty"` + Offset *int64 `cbor:"6,keyasint,omitempty" json:"offset,omitempty"` +} + +type DiskBackupRestoreArgs struct { + call rpc.Call + data diskBackupRestoreArgsData +} + +func (v *DiskBackupRestoreArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupRestoreArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupRestoreArgs) HasRestorePoint() bool { + return v.data.RestorePoint != nil +} + +func (v *DiskBackupRestoreArgs) RestorePoint() string { + if v.data.RestorePoint == nil { + return "" + } + return *v.data.RestorePoint +} + +func (v *DiskBackupRestoreArgs) HasData() bool { + return v.data.Data != nil +} + +func (v *DiskBackupRestoreArgs) Data() *stream.RecvStreamClient[[]byte] { + if v.data.Data == nil { + return nil + } + return &stream.RecvStreamClient[[]byte]{Client: v.call.NewClient(v.data.Data)} +} + +func (v *DiskBackupRestoreArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *DiskBackupRestoreArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *DiskBackupRestoreArgs) HasProgress() bool { + return v.data.Progress != nil +} + +func (v *DiskBackupRestoreArgs) Progress() *stream.SendStreamClient[*Progress] { + if v.data.Progress == nil { + return nil + } + return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} +} + +func (v *DiskBackupRestoreArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupRestoreArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupRestoreArgs) HasOffset() bool { + return v.data.Offset != nil +} + +func (v *DiskBackupRestoreArgs) Offset() int64 { + if v.data.Offset == nil { + return 0 + } + return *v.data.Offset +} + +func (v *DiskBackupRestoreArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupRestoreArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupRestoreArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupRestoreArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupRestoreResultsData struct { + Result **RestoreResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupRestoreResults struct { + call rpc.Call + data diskBackupRestoreResultsData +} + +func (v *DiskBackupRestoreResults) SetResult(result **RestoreResult) { + v.data.Result = result +} + +func (v *DiskBackupRestoreResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupRestoreResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupRestoreResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupRestoreResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupTransferOffsetArgsData struct { + TransferId *string `cbor:"0,keyasint,omitempty" json:"transfer_id,omitempty"` +} + +type DiskBackupTransferOffsetArgs struct { + call rpc.Call + data diskBackupTransferOffsetArgsData +} + +func (v *DiskBackupTransferOffsetArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupTransferOffsetArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupTransferOffsetArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupTransferOffsetArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupTransferOffsetResultsData struct { + ReceivedBytes *int64 `cbor:"0,keyasint,omitempty" json:"received_bytes,omitempty"` +} + +type DiskBackupTransferOffsetResults struct { + call rpc.Call + data diskBackupTransferOffsetResultsData +} + +func (v *DiskBackupTransferOffsetResults) SetReceivedBytes(received_bytes int64) { + v.data.ReceivedBytes = &received_bytes +} + +func (v *DiskBackupTransferOffsetResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupTransferOffsetResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListDeletedArgsData struct{} + +type DiskBackupListDeletedArgs struct { + call rpc.Call + data diskBackupListDeletedArgsData +} + +func (v *DiskBackupListDeletedArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListDeletedArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListDeletedArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListDeletedArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListDeletedResultsData struct { + Disks *[]*DeletedDisk `cbor:"0,keyasint,omitempty" json:"disks,omitempty"` + RetentionDays *int32 `cbor:"1,keyasint,omitempty" json:"retention_days,omitempty"` +} + +type DiskBackupListDeletedResults struct { + call rpc.Call + data diskBackupListDeletedResultsData +} + +func (v *DiskBackupListDeletedResults) SetDisks(disks []*DeletedDisk) { + x := slices.Clone(disks) + v.data.Disks = &x +} + +func (v *DiskBackupListDeletedResults) SetRetentionDays(retention_days int32) { + v.data.RetentionDays = &retention_days +} + +func (v *DiskBackupListDeletedResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListDeletedResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListDeletedResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListDeletedResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupUndeleteArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + VolumeId *string `cbor:"1,keyasint,omitempty" json:"volume_id,omitempty"` +} + +type DiskBackupUndeleteArgs struct { + call rpc.Call + data diskBackupUndeleteArgsData +} + +func (v *DiskBackupUndeleteArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupUndeleteArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupUndeleteArgs) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *DiskBackupUndeleteArgs) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *DiskBackupUndeleteArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupUndeleteArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupUndeleteArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupUndeleteArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupUndeleteResultsData struct { + Result **UndeleteResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupUndeleteResults struct { + call rpc.Call + data diskBackupUndeleteResultsData +} + +func (v *DiskBackupUndeleteResults) SetResult(result **UndeleteResult) { + v.data.Result = result +} + +func (v *DiskBackupUndeleteResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupUndeleteResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupUndeleteResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupUndeleteResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type DiskBackupBackup struct { + rpc.Call + args DiskBackupBackupArgs + results DiskBackupBackupResults +} + +func (t *DiskBackupBackup) Args() *DiskBackupBackupArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupBackup) Results() *DiskBackupBackupResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupListBackups struct { + rpc.Call + args DiskBackupListBackupsArgs + results DiskBackupListBackupsResults +} + +func (t *DiskBackupListBackups) Args() *DiskBackupListBackupsArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupListBackups) Results() *DiskBackupListBackupsResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupRestore struct { + rpc.Call + args DiskBackupRestoreArgs + results DiskBackupRestoreResults +} + +func (t *DiskBackupRestore) Args() *DiskBackupRestoreArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupRestore) Results() *DiskBackupRestoreResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupTransferOffset struct { + rpc.Call + args DiskBackupTransferOffsetArgs + results DiskBackupTransferOffsetResults +} + +func (t *DiskBackupTransferOffset) Args() *DiskBackupTransferOffsetArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupTransferOffset) Results() *DiskBackupTransferOffsetResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupListDeleted struct { + rpc.Call + args DiskBackupListDeletedArgs + results DiskBackupListDeletedResults +} + +func (t *DiskBackupListDeleted) Args() *DiskBackupListDeletedArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupListDeleted) Results() *DiskBackupListDeletedResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupUndelete struct { + rpc.Call + args DiskBackupUndeleteArgs + results DiskBackupUndeleteResults +} + +func (t *DiskBackupUndelete) Args() *DiskBackupUndeleteArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupUndelete) Results() *DiskBackupUndeleteResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackup interface { + Backup(ctx context.Context, state *DiskBackupBackup) error + ListBackups(ctx context.Context, state *DiskBackupListBackups) error + Restore(ctx context.Context, state *DiskBackupRestore) error + TransferOffset(ctx context.Context, state *DiskBackupTransferOffset) error + ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error + Undelete(ctx context.Context, state *DiskBackupUndelete) error +} + +type reexportDiskBackup struct { + client rpc.Client +} + +func (reexportDiskBackup) Backup(ctx context.Context, state *DiskBackupBackup) error { + panic("not implemented") +} + +func (reexportDiskBackup) ListBackups(ctx context.Context, state *DiskBackupListBackups) error { + panic("not implemented") +} + +func (reexportDiskBackup) Restore(ctx context.Context, state *DiskBackupRestore) error { + panic("not implemented") +} + +func (reexportDiskBackup) TransferOffset(ctx context.Context, state *DiskBackupTransferOffset) error { + panic("not implemented") +} + +func (reexportDiskBackup) ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error { + panic("not implemented") +} + +func (reexportDiskBackup) Undelete(ctx context.Context, state *DiskBackupUndelete) error { + panic("not implemented") +} + +func (t reexportDiskBackup) CapabilityClient() rpc.Client { + return t.client +} + +func AdaptDiskBackup(t DiskBackup) *rpc.Interface { + methods := []rpc.Method{ + { + Name: "backup", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "to_cloud", "pin", "data", "progress", "transfer_id", "offset"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Backup(ctx, &DiskBackupBackup{Call: call}) + }, + }, + { + Name: "listBackups", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.ListBackups(ctx, &DiskBackupListBackups{Call: call}) + }, + }, + { + Name: "restore", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "restore_point", "data", "force", "progress", "transfer_id", "offset"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Restore(ctx, &DiskBackupRestore{Call: call}) + }, + }, + { + Name: "transferOffset", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"transfer_id"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.TransferOffset(ctx, &DiskBackupTransferOffset{Call: call}) + }, + }, + { + Name: "listDeleted", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.ListDeleted(ctx, &DiskBackupListDeleted{Call: call}) + }, + }, + { + Name: "undelete", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "volume_id"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Undelete(ctx, &DiskBackupUndelete{Call: call}) + }, + }, + } + + return rpc.NewInterface(methods, t) +} + +type DiskBackupClient struct { + rpc.Client +} + +func NewDiskBackupClient(client rpc.Client) *DiskBackupClient { + return &DiskBackupClient{Client: client} +} + +func (c DiskBackupClient) Export() DiskBackup { + return reexportDiskBackup{client: c.Client} +} + +type DiskBackupClientBackupResults struct { + client rpc.Client + data diskBackupBackupResultsData +} + +func (v *DiskBackupClientBackupResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientBackupResults) Result() *BackupResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Backup(ctx context.Context, disk string, to_cloud bool, pin string, data stream.SendStream[[]byte], progress stream.SendStream[*Progress], transfer_id string, offset int64) (*DiskBackupClientBackupResults, error) { + args := DiskBackupBackupArgs{} + caps := map[rpc.OID]*rpc.InlineCapability{} + args.data.Disk = &disk + args.data.ToCloud = &to_cloud + args.data.Pin = &pin + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[[]byte](data), data) + args.data.Data = c + caps[oid] = ic + } + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[*Progress](progress), progress) + args.data.Progress = c + caps[oid] = ic + } + args.data.TransferId = &transfer_id + args.data.Offset = &offset + + var ret diskBackupBackupResultsData + + err := v.CallWithCaps(ctx, "backup", &args, &ret, caps) + if err != nil { + return nil, err + } + + return &DiskBackupClientBackupResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientListBackupsResults struct { + client rpc.Client + data diskBackupListBackupsResultsData +} + +func (v *DiskBackupClientListBackupsResults) HasPoints() bool { + return v.data.Points != nil +} + +func (v *DiskBackupClientListBackupsResults) Points() []*RestorePoint { + if v.data.Points == nil { + return nil + } + return *v.data.Points +} + +func (v DiskBackupClient) ListBackups(ctx context.Context, disk string) (*DiskBackupClientListBackupsResults, error) { + args := DiskBackupListBackupsArgs{} + args.data.Disk = &disk + + var ret diskBackupListBackupsResultsData + + err := v.Call(ctx, "listBackups", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientListBackupsResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientRestoreResults struct { + client rpc.Client + data diskBackupRestoreResultsData +} + +func (v *DiskBackupClientRestoreResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientRestoreResults) Result() *RestoreResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_point string, data stream.RecvStream[[]byte], force bool, progress stream.SendStream[*Progress], transfer_id string, offset int64) (*DiskBackupClientRestoreResults, error) { + args := DiskBackupRestoreArgs{} + caps := map[rpc.OID]*rpc.InlineCapability{} + args.data.Disk = &disk + args.data.RestorePoint = &restore_point + { + ic, oid, c := v.NewInlineCapability(stream.AdaptRecvStream[[]byte](data), data) + args.data.Data = c + caps[oid] = ic + } + args.data.Force = &force + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[*Progress](progress), progress) + args.data.Progress = c + caps[oid] = ic + } + args.data.TransferId = &transfer_id + args.data.Offset = &offset + + var ret diskBackupRestoreResultsData + + err := v.CallWithCaps(ctx, "restore", &args, &ret, caps) + if err != nil { + return nil, err + } + + return &DiskBackupClientRestoreResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientTransferOffsetResults struct { + client rpc.Client + data diskBackupTransferOffsetResultsData +} + +func (v *DiskBackupClientTransferOffsetResults) HasReceivedBytes() bool { + return v.data.ReceivedBytes != nil +} + +func (v *DiskBackupClientTransferOffsetResults) ReceivedBytes() int64 { + if v.data.ReceivedBytes == nil { + return 0 + } + return *v.data.ReceivedBytes +} + +func (v DiskBackupClient) TransferOffset(ctx context.Context, transfer_id string) (*DiskBackupClientTransferOffsetResults, error) { + args := DiskBackupTransferOffsetArgs{} + args.data.TransferId = &transfer_id + + var ret diskBackupTransferOffsetResultsData + + err := v.Call(ctx, "transferOffset", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientTransferOffsetResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientListDeletedResults struct { + client rpc.Client + data diskBackupListDeletedResultsData +} + +func (v *DiskBackupClientListDeletedResults) HasDisks() bool { + return v.data.Disks != nil +} + +func (v *DiskBackupClientListDeletedResults) Disks() []*DeletedDisk { + if v.data.Disks == nil { + return nil + } + return *v.data.Disks +} + +func (v *DiskBackupClientListDeletedResults) HasRetentionDays() bool { + return v.data.RetentionDays != nil +} + +func (v *DiskBackupClientListDeletedResults) RetentionDays() int32 { + if v.data.RetentionDays == nil { + return 0 + } + return *v.data.RetentionDays +} + +func (v DiskBackupClient) ListDeleted(ctx context.Context) (*DiskBackupClientListDeletedResults, error) { + args := DiskBackupListDeletedArgs{} + + var ret diskBackupListDeletedResultsData + + err := v.Call(ctx, "listDeleted", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientListDeletedResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientUndeleteResults struct { + client rpc.Client + data diskBackupUndeleteResultsData +} + +func (v *DiskBackupClientUndeleteResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientUndeleteResults) Result() *UndeleteResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Undelete(ctx context.Context, disk string, volume_id string) (*DiskBackupClientUndeleteResults, error) { + args := DiskBackupUndeleteArgs{} + args.data.Disk = &disk + args.data.VolumeId = &volume_id + + var ret diskBackupUndeleteResultsData + + err := v.Call(ctx, "undelete", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientUndeleteResults{client: v.Client, data: ret}, nil +} diff --git a/api/disk/rpc.yml b/api/disk/rpc.yml new file mode 100644 index 000000000..eb246c2a5 --- /dev/null +++ b/api/disk/rpc.yml @@ -0,0 +1,320 @@ +apiVersion: miren.dev/rpc/v1 +kind: IDL + +imports: + stream: + path: ../../pkg/rpc/stream/stream.yml + import: miren.dev/runtime/pkg/rpc/stream + standard: + path: ../../pkg/rpc/standard/standard.yml + import: miren.dev/runtime/pkg/rpc/standard + +types: + - type: Transfer + doc: > + How far a byte transfer has got. Backup and restore move whole disk + images, so a client that showed nothing until completion would look + indistinguishable from a hang. + fields: + - name: done + type: int64 + index: 0 + doc: Bytes moved so far + - name: total + type: int64 + index: 1 + doc: Bytes expected in total, or 0 when the server cannot know yet + - name: bytes_per_second + type: int64 + index: 2 + doc: Recent throughput + - name: eta_seconds + type: int64 + index: 3 + doc: Estimated seconds remaining, or 0 when total is unknown + + - type: Progress + doc: > + One event in the progress stream a client passes to backup or restore. + Modelled on build's Status: a discriminated union, so new event kinds do + not break existing clients. + fields: + - name: update + type: union + union: + - name: message + type: string + index: 1 + doc: A step the server has started, for display + - name: transfer + type: Transfer + index: 2 + doc: Byte-movement progress + - name: warning + type: string + index: 3 + doc: > + Something the operator should know but which does not stop the + operation, such as backing up a disk that is currently mounted. + - name: error + type: string + index: 4 + doc: > + A failure being reported before the call returns. The call's own + error is still authoritative. + + - type: RestorePoint + doc: > + A moment in time a disk can be restored to. Universal-mode volumes reach + one by fetching a full image and accelerator-mode volumes by replaying log + segments (RFD-64), but that split is the server's business — a caller + picks a point, not a backup type. + fields: + - name: id + type: string + index: 0 + doc: Opaque identifier to pass back to restore + - name: created_at + type: standard.Timestamp + index: 1 + doc: When the restore point was taken + - name: size_bytes + type: int64 + index: 2 + doc: Compressed size held in the cloud + - name: image_size_bytes + type: int64 + index: 3 + doc: Size of the disk image the point restores to, 0 when not recorded + - name: name + type: string + index: 4 + doc: Operator-supplied name, set when the point was pinned against cleanup + - name: mode + type: string + index: 5 + doc: Volume mode the point was taken from (universal or accelerator) + + - type: BackupResult + fields: + - name: image_size_bytes + type: int64 + index: 0 + doc: Size of the image that was read + - name: compressed_size_bytes + type: int64 + index: 1 + doc: Size of the resulting snapshot + - name: checksum + type: string + index: 2 + doc: SHA-256 of the uncompressed image, hex encoded + - name: restore_point_id + type: string + index: 3 + doc: Identifier of the uploaded restore point, empty for a local backup + + - type: DeletedDisk + doc: > + A disk that was deleted but whose data is still sitting in the + soft-delete holding area, waiting out its retention period. + fields: + - name: disk_name + type: string + index: 0 + - name: volume_id + type: string + index: 1 + doc: Identifies which one to recover when several share a name + - name: size_gb + type: int64 + index: 2 + - name: filesystem + type: string + index: 3 + - name: deleted_at + type: standard.Timestamp + index: 4 + - name: expires_at + type: standard.Timestamp + index: 5 + doc: When the retention period runs out and the data is reclaimed + - name: volume_mode + type: string + index: 6 + + - type: UndeleteResult + fields: + - name: disk + type: string + index: 0 + doc: Name of the recovered disk + - name: disk_id + type: string + index: 1 + doc: Entity id of the recreated disk + - name: volume_id + type: string + index: 2 + - name: image_path + type: string + index: 3 + + - type: RestoreResult + fields: + - name: disk + type: string + index: 0 + doc: Name of the restored disk + - name: image_size_bytes + type: int64 + index: 1 + doc: Size of the image that was written + - name: created + type: bool + index: 2 + doc: True when the disk did not exist and was created by this restore + +interfaces: + - name: DiskBackup + doc: > + Backup and restore of disk images, driven by a remote client (RFD-108). + + Two steps genuinely have to happen on the server: moving the image bytes, + and talking to miren.cloud with the cluster's key. Both stay here. The + client orchestrates over this interface whether it runs on the server host + or on a laptop, so there is no local-versus-remote mode to detect and no + chance of mistaking "the data directory happens to exist here" for "this + client is colocated with the cluster it targets". + + Every method is admin-level, matching the other disk mutations. + methods: + - name: backup + doc: > + Snapshot a disk. With to_cloud set, the server compresses the image + and uploads it to miren.cloud as a restore point using the cluster's + own key, and no image bytes reach the client. Otherwise the compressed + snapshot is streamed down data, which is how a cluster with no cloud + still backs up to a file on the client. + parameters: + - name: disk + type: string + doc: Disk name to back up + - name: to_cloud + type: bool + doc: Upload to miren.cloud instead of streaming to the client + - name: pin + type: string + doc: Name the uploaded restore point, pinning it against cleanup + - name: data + type: stream.SendStream[[]byte] + doc: Stream the client receives the compressed snapshot on + - name: progress + type: stream.SendStream[*Progress] + doc: Stream the client receives progress events on + - name: transfer_id + type: string + doc: > + Names this transfer so a dropped connection can pick it up again. + The client chooses it, because a value the server handed back + would be lost with the call that carried it. Retrying with the + same id resumes; a new id starts over. + - name: offset + type: int64 + doc: > + Byte of the compressed snapshot to start sending from. A resuming + client passes what it already has on disk. + results: + - name: result + type: '*BackupResult' + + - name: listBackups + doc: > + List the restore points available for a disk, newest first, so a + client can show them and let the operator pick one. + parameters: + - name: disk + type: string + doc: Disk name to list restore points for + results: + - name: points + type: list + element: RestorePoint + + - name: restore + doc: > + Rebuild a disk's image. With restore_point set, the server downloads + that point from miren.cloud. Otherwise it reads a snapshot the client + streams up data. Restoring a disk the cluster has never seen creates + it, which is what makes this usable for disaster recovery onto a fresh + host. + parameters: + - name: disk + type: string + doc: Disk name to restore to + - name: restore_point + type: string + doc: Restore point to fetch from miren.cloud, empty to read from data + - name: data + type: stream.RecvStream[[]byte] + doc: Stream the server reads a client-supplied snapshot from + - name: force + type: bool + doc: Overwrite an existing disk image + - name: progress + type: stream.SendStream[*Progress] + doc: Stream the client receives progress events on + - name: transfer_id + type: string + doc: > + Names this transfer so a dropped connection can pick it up again. + Required when streaming a snapshot up; ignored for a restore point, + which the server fetches itself. + - name: offset + type: int64 + doc: > + Byte of the snapshot the client is sending from. It must match + what the server already holds, which transferOffset reports. + results: + - name: result + type: '*RestoreResult' + + - name: transferOffset + doc: > + How much of a transfer the server already holds, so a client resuming + an interrupted restore knows where to start sending. Unknown ids + report zero, which is what makes the first attempt and a retry the + same code path on the client. + parameters: + - name: transfer_id + type: string + results: + - name: received_bytes + type: int64 + + - name: listDeleted + doc: > + List disks whose data is still recoverable from the soft-delete + holding area, newest deletion first. + results: + - name: disks + type: list + element: DeletedDisk + - name: retention_days + type: int32 + doc: How long this server keeps deleted disk data before reclaiming it + + - name: undelete + doc: > + Recover a deleted disk: move its data back out of the holding area + and recreate the entities that point at it. + parameters: + - name: disk + type: string + doc: Name of the deleted disk to recover + - name: volume_id + type: string + doc: Which volume to recover, when several deleted disks share a name + results: + - name: result + type: '*UndeleteResult' diff --git a/blackbox/disk_backup_test.go b/blackbox/disk_backup_test.go new file mode 100644 index 000000000..d2cd5f045 --- /dev/null +++ b/blackbox/disk_backup_test.go @@ -0,0 +1,120 @@ +//go:build blackbox + +package blackbox + +import ( + "fmt" + "strings" + "testing" + "time" + + "miren.dev/runtime/blackbox/harness" +) + +// TestDiskBackupRestoreOverRPC exercises the RFD-108 claim: backup and restore +// work from a client, without a shell on the server. +// +// The proof is in what the commands are NOT given. Every invocation here is a +// plain `m disk ...` with no sudo and no --data-path, where the old commands +// needed both — they opened /var/lib/miren directly and stopped early when it +// was not there. +func TestDiskBackupRestoreOverRPC(t *testing.T) { + c := harness.NewCluster(t) + m := harness.NewMiren(t, c) + + diskName := harness.UniqueAppName(t, "bk-disk") + restoredName := diskName + "-restored" + snapshotPath := fmt.Sprintf("/tmp/%s.miren.zst", diskName) + + t.Log("Creating source disk...") + m.MustRun("debug", "disk", "create", "-n", diskName, "-s", "1") + waitDiskProvisioned(t, m, diskName) + + // No sudo, no --data-path. + t.Log("Backing up over RPC...") + r := m.MustRun("disk", "backup", "-n", diskName, "-o", snapshotPath) + r.RequireContains(t, "Backup complete") + r.RequireContains(t, "Checksum:") + + // The snapshot has to have actually landed on the client side, since that + // is the half the server streamed rather than wrote. + r = m.RunCmd("test", "-s", snapshotPath) + r.RequireSuccess(t) + + // Restoring into a name the cluster has never seen must create the disk. + // This is the disaster-recovery path: on a rebuilt host there is nothing to + // restore into. + t.Log("Restoring into a new disk over RPC...") + r = m.MustRun("disk", "restore", "-s", snapshotPath, "-n", restoredName) + r.RequireContains(t, "Restore complete") + r.RequireContains(t, "Created disk") + + waitDiskProvisioned(t, m, restoredName) + + // A universal-mode disk is mounted by the volume controller whether or not + // anything leased it, and a loop device holds the image's inode rather than + // its path. Restoring over it would leave the mounted filesystem on the old + // image while reporting success, so it must be refused instead. + t.Log("Checking that restore refuses a disk that is in use...") + r = m.Run("disk", "restore", "-s", snapshotPath, "-n", diskName, "--force") + if r.Success() { + t.Errorf("restore into a mounted disk should have been refused, got:\n%s", r.Stdout+r.Stderr) + } + r.RequireContains(t, "in use") +} + +// TestDiskBackupRejectsPinWithoutCloud covers the flag combination an operator +// is most likely to try first, since --pin only means anything to miren.cloud. +func TestDiskBackupRejectsPinWithoutCloud(t *testing.T) { + c := harness.NewCluster(t) + m := harness.NewMiren(t, c) + + r := m.Run("disk", "backup", "-n", "does-not-matter", "--pin", "some-name") + if r.Success() { + t.Fatalf("--pin without --cloud should have been rejected, got:\n%s", r.Stdout+r.Stderr) + } + r.RequireContains(t, "--cloud") +} + +func waitDiskProvisioned(t *testing.T, m *harness.Miren, name string) { + t.Helper() + harness.Poll(t, "disk provisioned: "+name, 60*time.Second, 2*time.Second, + func() (bool, string) { + r := m.Run("debug", "disk", "list") + if !r.Success() { + return false, "debug disk list failed" + } + if diskIsProvisioned(r.Stdout+r.Stderr, name) { + return true, "" + } + return false, "disk not yet provisioned" + }, + ) +} + +// diskIsProvisioned reports whether the named disk's own entry says provisioned. +// +// Asking whether the whole listing mentions the name and mentions "provisioned" +// is not the same question: any other provisioned disk in the cluster answers +// the second half, so the poll would return while this disk was still coming +// up. The name and the status sit on different lines of one entry, so the match +// has to run over the entry rather than a line. +func diskIsProvisioned(output, name string) bool { + lines := strings.Split(output, "\n") + + for i, line := range lines { + if !strings.Contains(line, "Name:") || !strings.Contains(line, name) { + continue + } + // Scan this entry's fields, stopping where the next entry begins. + for _, field := range lines[i+1:] { + if strings.HasPrefix(strings.TrimSpace(field), "ID:") { + break + } + if strings.Contains(field, "Status:") { + return strings.Contains(field, "provisioned") + } + } + } + return false +} diff --git a/blackbox/disk_undelete_test.go b/blackbox/disk_undelete_test.go index d96dd022c..ca770f188 100644 --- a/blackbox/disk_undelete_test.go +++ b/blackbox/disk_undelete_test.go @@ -60,17 +60,16 @@ func TestDiskUndelete(t *testing.T) { }, ) - // Step 6: Verify the disk appears in list-deleted - // These commands need sudo because /var/lib/miren/disk-data is owned by root + // Step 6: Verify the disk appears in list-deleted. + // No sudo and no --data-path: these go through the server now, so they work + // from a client that cannot see /var/lib/miren at all. t.Log("Checking list-deleted...") - r = m.RunCmd("sudo", "m", "disk", "list-deleted") - r.RequireSuccess(t) + r = m.MustRun("disk", "list-deleted") r.RequireContains(t, diskName) // Step 7: Undelete the disk t.Log("Undeleting disk...") - r = m.RunCmd("sudo", "m", "disk", "undelete", "-n", diskName) - r.RequireSuccess(t) + r = m.MustRun("disk", "undelete", "-n", diskName) r.RequireContains(t, "Disk restored successfully") // Step 8: Verify the disk is back and provisioned @@ -89,8 +88,7 @@ func TestDiskUndelete(t *testing.T) { ) // Step 9: Verify it's no longer in list-deleted - r = m.RunCmd("sudo", "m", "disk", "list-deleted") - r.RequireSuccess(t) + r = m.MustRun("disk", "list-deleted") if r.OutputContains(diskName) { t.Error("disk should no longer appear in list-deleted after undelete") } diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 8b6e2aeff..eeb590d43 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1338,6 +1338,13 @@ Warning: These commands are intended for advanced users and developers. They may d.Dispatch("debug disk lease-delete", Infer("debug disk lease-delete", "Delete a disk lease entity", DebugDiskLeaseDelete)) d.Dispatch("debug disk lease-status", Infer("debug disk lease-status", "Show detailed status of a disk lease", DebugDiskLeaseStatus)) d.Dispatch("debug disk mounts", Infer("debug disk mounts", "List all mounted disks from /proc/mounts", DebugDiskMounts)) + // Break-glass: `miren disk backup`/`restore` drive the server and are the + // supported commands. These touch the image directly, for a host whose RPC + // listener is down, and so must run on the server. + d.Dispatch("debug disk backup", Infer("debug disk backup", "Back up a disk by reading its image directly (break-glass)", DebugDiskBackup)) + d.Dispatch("debug disk restore", Infer("debug disk restore", "Restore a disk by writing its image directly (break-glass)", DebugDiskRestore)) + d.Dispatch("debug disk undelete", Infer("debug disk undelete", "Recover a deleted disk by moving its data directly (break-glass)", DebugDiskUndelete)) + d.Dispatch("debug disk list-deleted", Infer("debug disk list-deleted", "Read the soft-delete holding area directly (break-glass)", DebugDiskListDeleted)) // Debug saga commands d.Dispatch("debug saga", Section("debug saga", "Saga execution debug commands", "", WithSectionDescription(sagaSectionDescription))) diff --git a/cli/commands/debug_disk_backup.go b/cli/commands/debug_disk_backup.go new file mode 100644 index 000000000..2c766eb27 --- /dev/null +++ b/cli/commands/debug_disk_backup.go @@ -0,0 +1,238 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/components/coordinate" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cloudauth" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/registration" + "miren.dev/runtime/pkg/snapshot" +) + +// DebugDiskBackup backs up a disk by reading its image directly, without going +// through the server. +// +// This is break-glass, not a second supported way to back up. `miren disk +// backup` drives the server over RPC and is the command to use. This one exists +// for the case that one cannot cover: the server's RPC listener being down on a +// host you can still get a shell on. It therefore has to run on the server and +// keeps --data-path. +func DebugDiskBackup(ctx *Context, opts struct { + ConfigCentric + Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` + Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + Cloud bool `long:"cloud" description:"Also upload the snapshot to miren.cloud as a restore point"` + Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` +}) (retErr error) { + if _, err := os.Stat(opts.DataPath); err != nil { + return fmt.Errorf("data path %s not found — disk backup must be run on the server", opts.DataPath) + } + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + resolver := diskresolve.New(eac, nil) + + target, err := snapshot.PrepareBackup(ctx, resolver, opts.Name, opts.DataPath) + if err != nil { + return err + } + + imgInfo, err := os.Stat(target.ImagePath) + if err != nil { + return fmt.Errorf("disk image not found at %s: %w", target.ImagePath, err) + } + + // Nothing here freezes the filesystem or takes a copy-on-write clone, so a + // backup of a live disk is a sequential read of a file the loop device is + // still writing. The head and tail of the image come from different moments, + // which is weaker than the power-loss state fsck and Postgres recovery are + // built for. Say so plainly: the operator is the one deciding this is safe. + // + // The kernel is the only thing that actually knows, so ask it. Not knowing + // counts as in use, since this flag only ever warns a reader off relying on + // the snapshot. + inUse := true + if dev, lerr := diskio.NewRealDiskMountOps(ctx.Log).FindLoopByBacking(target.ImagePath); lerr != nil { + ctx.Warn("Could not tell whether the disk is in use: %v", lerr) + } else { + inUse = dev != "" + if inUse { + ctx.Warn("Disk is in use (%s) and may be written during the backup.", dev) + ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") + ctx.Warn("Detach the disk first for a backup you can rely on.") + } + } + + outputPath := opts.Output + if outputPath == "" { + outputPath = fmt.Sprintf("%s-%s.miren.zst", opts.Name, time.Now().Format("20060102-150405")) + } + + ctx.Info("Backing up disk %q (%s)", opts.Name, formatBytes(imgInfo.Size())) + ctx.Info("Image: %s", target.ImagePath) + ctx.Info("Output: %s", outputPath) + + start := time.Now() + + imgFile, err := os.Open(target.ImagePath) + if err != nil { + return fmt.Errorf("opening image file: %w", err) + } + defer imgFile.Close() + + outFile, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("creating output file: %w", err) + } + // Set once the snapshot itself is written. A later step failing (the cloud + // upload) must not take a finished local backup down with it. + snapshotComplete := false + + defer func() { + closeErr := outFile.Close() + if closeErr != nil && retErr == nil { + retErr = fmt.Errorf("closing output file: %w", closeErr) + } + // Only discard a snapshot that never finished, or one whose close + // failed and may therefore be truncated. + if retErr != nil && (!snapshotComplete || closeErr != nil) { + os.Remove(outputPath) + } + }() + + ctx.Info("Computing checksum and compressing...") + + checksum, err := snapshot.Backup(outFile, imgFile, opts.Name, imgInfo.Size(), target.Filesystem) + if err != nil { + return err + } + + outInfo, err := outFile.Stat() + if err != nil { + return fmt.Errorf("stat output file: %w", err) + } + + snapshotComplete = true + + duration := time.Since(start) + ratio := float64(outInfo.Size()) / float64(imgInfo.Size()) * 100 + + ctx.Info("Backup complete") + ctx.Info(" Original size: %s", formatBytes(imgInfo.Size())) + ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(outInfo.Size()), ratio) + ctx.Info(" Checksum: %s", checksum) + ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) + ctx.Info(" Snapshot: %s", outputPath) + + if opts.Cloud { + updateID, err := uploadSnapshotToCloud(ctx, opts.DataPath, target, outputPath, cloudSnapshotDetails{ + Pin: opts.Pin, + InUse: inUse, + ImageSize: imgInfo.Size(), + ImageChecksum: checksum, + CompressedSize: outInfo.Size(), + }) + if err != nil { + // The local snapshot is finished and valid; say where it is rather + // than leaving the operator thinking they got nothing. + ctx.Warn("Upload failed, but the local snapshot is intact at %s", outputPath) + return fmt.Errorf("uploading snapshot to miren.cloud: %w", err) + } + ctx.Info(" Uploaded: %s", updateID) + } + + return nil +} + +// cloudSnapshotDetails carries what the sidecar records about the image the +// snapshot was taken from. It mirrors what diskio.ImageSnapshotter writes, so a +// restore sees the same fields whichever path produced the update. +type cloudSnapshotDetails struct { + Pin string + ImageSize int64 + ImageChecksum string + CompressedSize int64 + // InUse records that the image was attached when it was read, so the + // snapshot is a smear across the read rather than a point-in-time copy. + InUse bool +} + +// uploadSnapshotToCloud sends a finished snapshot file to miren.cloud as a +// loop_image update for the disk's volume. +// +// The cluster's service account key lives in the registration this server wrote +// at enrolment, which is why this has to run on the server alongside the data +// directory. +func uploadSnapshotToCloud(ctx *Context, dataPath string, target *snapshot.BackupTarget, snapshotPath string, details cloudSnapshotDetails) (string, error) { + if target.CloudVolumeID == "" { + return "", fmt.Errorf("disk %q is not registered with miren.cloud yet; "+ + "registration happens on the next disk reconcile, so try again shortly", target.Name) + } + + reg, err := registration.LoadRegistration(filepath.Join(dataPath, "server")) + if err != nil { + return "", fmt.Errorf("loading cluster registration: %w", err) + } + if reg == nil || reg.Status != "approved" || reg.PrivateKey == "" { + return "", fmt.Errorf("this cluster is not registered with miren.cloud") + } + + keyPair, err := cloudauth.LoadKeyPairFromPEM(reg.PrivateKey) + if err != nil { + return "", fmt.Errorf("parsing cluster key: %w", err) + } + + cloudURL := reg.CloudURL + if cloudURL == "" { + cloudURL = coordinate.DefaultCloudURL + } + + authClient, err := cloudauth.NewAuthClient(cloudURL, keyPair) + if err != nil { + return "", fmt.Errorf("creating cloud auth client: %w", err) + } + + file, err := os.Open(snapshotPath) + if err != nil { + return "", fmt.Errorf("opening snapshot: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return "", fmt.Errorf("stat snapshot: %w", err) + } + + ctx.Info("Uploading to %s...", cloudURL) + + updates := diskio.NewCloudUpdatesClient(ctx.Log, cloudURL, authClient) + // ctx is the command context, so Ctrl-C interrupts the upload rather than + // leaving it running to the end of a multi-gigabyte image. + return updates.Upload(ctx, target.CloudVolumeID, diskio.UploadRequest{ + Kind: diskio.KindLoopImage, + OrderingKey: fmt.Sprintf("%016x", time.Now().UnixNano()), + SnapshotName: details.Pin, + Metadata: map[string]any{ + "compression": "zstd", + "format": "miren-snapshot", + "filesystem": target.Filesystem, + "image_size": details.ImageSize, + "image_sha256": details.ImageChecksum, + "compressed_size": details.CompressedSize, + // Records that this was taken from a live disk, so the image is a + // smear across the read rather than a point-in-time copy. + "was_attached": details.InUse, + }, + }, file, info.Size()) +} diff --git a/cli/commands/debug_disk_list_deleted.go b/cli/commands/debug_disk_list_deleted.go new file mode 100644 index 000000000..435e5ad79 --- /dev/null +++ b/cli/commands/debug_disk_list_deleted.go @@ -0,0 +1,94 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/components/diskio" +) + +// DebugDiskListDeleted reads the soft-delete holding area directly, without +// going through the server. +// +// Break-glass; `miren disk list-deleted` is the supported command. +func DebugDiskListDeleted(ctx *Context, opts struct { + FormatOptions + ConfigCentric + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` +}) error { + diskDataPath := filepath.Join(opts.DataPath, "disk-data") + if _, err := os.Stat(diskDataPath); err != nil { + return fmt.Errorf("data path %s not found — this command must be run on the server", diskDataPath) + } + + entries, err := diskio.ListDeletedVolumes(diskDataPath) + if err != nil { + return fmt.Errorf("listing deleted volumes: %w", err) + } + + retentionDays := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + + if opts.IsJSON() { + type deletedDiskJSON struct { + DiskName string `json:"disk_name"` + VolumeID string `json:"volume_id"` + SizeGB int64 `json:"size_gb"` + Filesystem string `json:"filesystem"` + DeletedAt string `json:"deleted_at"` + ExpiresAt string `json:"expires_at"` + RetentionDays int `json:"retention_days"` + } + + // Non-nil, so an empty result marshals as [] rather than null and + // callers can iterate it without a special case. + items := make([]deletedDiskJSON, 0, len(entries)) + for _, e := range entries { + meta := e.Metadata + expiresAt := meta.DeletedAt.Add(time.Duration(retentionDays) * 24 * time.Hour) + items = append(items, deletedDiskJSON{ + DiskName: meta.DiskName, + VolumeID: meta.VolumeID, + SizeGB: meta.SizeGb, + Filesystem: meta.Filesystem, + DeletedAt: meta.DeletedAt.Format(time.RFC3339), + ExpiresAt: expiresAt.Format(time.RFC3339), + RetentionDays: retentionDays, + }) + } + + return PrintJSON(items) + } + + if len(entries) == 0 { + ctx.Info("No deleted disks found") + return nil + } + + ctx.Info("Deleted disks available for recovery:") + ctx.Info("") + + for _, e := range entries { + meta := e.Metadata + age := time.Since(meta.DeletedAt) + remaining := time.Duration(retentionDays)*24*time.Hour - age + + ctx.Info("Name: %s", meta.DiskName) + ctx.Info(" Volume ID: %s", meta.VolumeID) + ctx.Info(" Size: %d GB", meta.SizeGb) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Deleted: %s (%s ago)", meta.DeletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) + if remaining > 0 { + ctx.Info(" Expires in: %s", remaining.Truncate(time.Minute)) + } else { + ctx.Info(" Expires in: imminent (past retention period)") + } + ctx.Info("") + } + + ctx.Info("To recover: miren disk undelete --name ") + ctx.Info(" or, if the server's RPC listener is down: miren debug disk undelete --name ") + + return nil +} diff --git a/cli/commands/debug_disk_restore.go b/cli/commands/debug_disk_restore.go new file mode 100644 index 000000000..e5917de3d --- /dev/null +++ b/cli/commands/debug_disk_restore.go @@ -0,0 +1,152 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/snapshot" +) + +// DebugDiskRestore restores a disk by writing its image directly, without going +// through the server. +// +// Break-glass, for when the server's RPC listener is down; `miren disk restore` +// is the supported command. Note that this path cannot see whether the image is +// currently loop-attached, so it can rename a restored image over a live one and +// report success having changed nothing. `miren disk restore` refuses that case. +func DebugDiskRestore(ctx *Context, opts struct { + ConfigCentric + Snapshot string `short:"s" long:"snapshot" description:"Path to snapshot file" required:"true"` + Name string `short:"n" long:"name" description:"Disk name to restore to (default: original name from snapshot)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + Force bool `short:"f" long:"force" description:"Overwrite existing disk image without confirmation"` +}) (retErr error) { + if _, err := os.Stat(opts.DataPath); err != nil { + return fmt.Errorf("data path %s not found — disk restore must be run on the server", opts.DataPath) + } + + snapFile, err := os.Open(opts.Snapshot) + if err != nil { + return fmt.Errorf("opening snapshot file: %w", err) + } + defer snapFile.Close() + + meta, err := snapshot.ReadHeader(snapFile) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + + diskName := opts.Name + if diskName == "" { + diskName = meta.Name + } + + ctx.Info("Restoring from snapshot: %s", opts.Snapshot) + ctx.Info(" Original disk: %s", meta.Name) + ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Created: %s", meta.Timestamp.Format(time.RFC3339)) + ctx.Info(" Checksum: %s", meta.Checksum) + ctx.Info(" Target disk: %s", diskName) + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + ec := entityserver.NewClient(ctx.Log, eac) + resolver := diskresolve.New(eac, ec) + + target, err := snapshot.PrepareRestore(ctx, resolver, diskName, opts.DataPath, + snapshot.WithCreator(resolver, meta.SizeBytes, meta.Filesystem), + ) + if err != nil { + return err + } + + // If the disk was freshly created and anything fails, clean up the + // RESTORING disk entity so it doesn't become a zombie. + if target.Created && target.Cleanup != nil { + defer func() { + if retErr != nil { + ctx.Warn("Cleaning up disk entity after failed restore") + if cerr := target.Cleanup(ctx); cerr != nil { + ctx.Warn("Failed to clean up disk entity: %v", cerr) + } + } + }() + } + + if !target.Created { + if _, err := os.Stat(target.ImagePath); err == nil { + if !opts.Force { + return fmt.Errorf("disk image already exists at %s — use --force to overwrite", target.ImagePath) + } + ctx.Warn("Overwriting existing disk image at %s", target.ImagePath) + } else if !os.IsNotExist(err) { + return fmt.Errorf("checking existing image at %s: %w", target.ImagePath, err) + } + } + + ctx.Info("Restoring to: %s", target.ImagePath) + + start := time.Now() + + if err := os.MkdirAll(filepath.Dir(target.ImagePath), 0o755); err != nil { + return fmt.Errorf("creating image directory: %w", err) + } + + tmpPath := target.ImagePath + ".restore.tmp" + outFile, err := os.Create(tmpPath) + if err != nil { + return fmt.Errorf("creating temp image file: %w", err) + } + defer func() { + outFile.Close() + if retErr != nil { + os.Remove(tmpPath) + } + }() + + if err := outFile.Truncate(meta.SizeBytes); err != nil { + return fmt.Errorf("truncating image file: %w", err) + } + + ctx.Info("Decompressing...") + + if err := snapshot.RestoreImage(outFile, snapFile, meta); err != nil { + return err + } + + if err := outFile.Close(); err != nil { + return fmt.Errorf("closing restored image: %w", err) + } + + if err := os.Rename(tmpPath, target.ImagePath); err != nil { + return fmt.Errorf("moving restored image into place: %w", err) + } + + if target.Finalize != nil { + ctx.Info("Finalizing disk entities...") + if err := target.Finalize(ctx); err != nil { + return fmt.Errorf("finalizing restore: %w", err) + } + } + + duration := time.Since(start) + + ctx.Info("Restore complete") + ctx.Info(" Disk: %s", diskName) + ctx.Info(" Image: %s", target.ImagePath) + ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) + ctx.Info(" Checksum: verified") + ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) + + return nil +} diff --git a/cli/commands/debug_disk_undelete.go b/cli/commands/debug_disk_undelete.go new file mode 100644 index 000000000..99477b033 --- /dev/null +++ b/cli/commands/debug_disk_undelete.go @@ -0,0 +1,219 @@ +package commands + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/idgen" +) + +// DebugDiskUndelete recovers a deleted disk by moving its data back and +// recreating its entities directly, without going through the server. +// +// Break-glass, for when the server's RPC listener is down; `miren disk +// undelete` is the supported command. It therefore has to run on the server and +// keeps --data-path. +func DebugDiskUndelete(ctx *Context, opts struct { + ConfigCentric + Name string `short:"n" long:"name" description:"Disk name to undelete" required:"true"` + VolumeID string `short:"V" long:"volume-id" description:"Volume ID to restore (when multiple deleted disks share a name)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` +}) error { + diskDataPath := filepath.Join(opts.DataPath, "disk-data") + if _, err := os.Stat(diskDataPath); err != nil { + return fmt.Errorf("data path %s not found — disk undelete must be run on the server", diskDataPath) + } + + entries, err := diskio.ListDeletedVolumes(diskDataPath) + if err != nil { + return fmt.Errorf("listing deleted volumes: %w", err) + } + + // Filter by name + var matches []diskio.DeletedVolumeEntry + for _, e := range entries { + if e.Metadata.DiskName == opts.Name { + if opts.VolumeID == "" || e.Metadata.VolumeID == opts.VolumeID { + matches = append(matches, e) + } + } + } + + if len(matches) == 0 { + return fmt.Errorf("no deleted disk found with name %q", opts.Name) + } + + if len(matches) > 1 { + ctx.Info("Multiple deleted disks found with name %q:", opts.Name) + for _, m := range matches { + age := time.Since(m.Metadata.DeletedAt).Truncate(time.Minute) + ctx.Info(" Volume ID: %s (deleted %s ago, %d GB, %s)", + m.Metadata.VolumeID, age, m.Metadata.SizeGb, m.Metadata.Filesystem) + } + return fmt.Errorf("specify --volume-id to select which disk to restore") + } + + entry := matches[0] + meta := entry.Metadata + + ctx.Info("Restoring deleted disk:") + ctx.Info(" Name: %s", meta.DiskName) + ctx.Info(" Size: %d GB", meta.SizeGb) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Deleted: %s", meta.DeletedAt.Format(time.RFC3339)) + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + ec := entityserver.NewClient(ctx.Log, eac) + resolver := diskresolve.New(eac, ec) + + // Check if a disk with this name already exists + if _, err := resolver.FindDisk(context.Background(), meta.DiskName); err == nil { + return fmt.Errorf("a disk named %q already exists — rename or delete it before restoring", meta.DiskName) + } + + // Normalize filesystem + filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + + var fs storage_v1alpha.DiskFilesystem + switch filesystem { + case "ext4": + fs = storage_v1alpha.EXT4 + case "xfs": + fs = storage_v1alpha.XFS + case "btrfs": + fs = storage_v1alpha.BTRFS + default: + fs = storage_v1alpha.EXT4 + } + + // Create the disk entity in RESTORING state + diskId := idgen.GenNS("disk") + disk := &storage_v1alpha.Disk{ + Name: meta.DiskName, + SizeGb: meta.SizeGb, + Filesystem: fs, + Status: storage_v1alpha.RESTORING, + } + + diskEntityId, err := ec.Create(context.Background(), diskId, disk) + if err != nil { + return fmt.Errorf("creating disk entity: %w", err) + } + + ctx.Info("Created disk entity: %s", diskEntityId) + + // Move the volume directory back + volId := meta.VolumeID + destPath := filepath.Join(diskDataPath, "volumes", volId) + + if err := os.Rename(entry.Path, destPath); err != nil { + // Clean up the disk entity we just created + if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk entity: %v", derr) + } + return fmt.Errorf("moving volume back to %s: %w", destPath, err) + } + + // Deferred rollback: if we don't reach the final commit, move the + // volume back to deleted-volumes (with its metadata intact) and clean + // up any entities we created. + committed := false + defer func() { + if committed { + return + } + if rerr := os.Rename(destPath, entry.Path); rerr != nil { + ctx.Warn("Failed to move volume back to deleted-volumes: %v", rerr) + } + if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk entity: %v", derr) + } + }() + + // Find the node ID + nodeId, err := resolver.FindNodeId(context.Background()) + if err != nil { + ctx.Warn("Failed to find node ID, using stored value: %v", err) + nodeId = meta.NodeID.Id() + } + + imagePath := filepath.Join(destPath, "disk.img") + + // Verify the disk image actually exists before creating entities + if _, err := os.Stat(imagePath); err != nil { + return fmt.Errorf("disk image not found at %s — deleted volume may be corrupted", imagePath) + } + + // Create disk_volume entity + volEntityId := entity.Id("disk_volume/" + volId) + vol := &storage_v1alpha.DiskVolume{ + Name: meta.DiskName, + DiskId: diskEntityId, + VolumeId: volId, + SizeGb: meta.SizeGb, + Filesystem: filesystem, + VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY. The runner's DiskVolumeController drives + // the restored volume through the same mount-then-READY ordering as a + // fresh create (skipping the reimage, since disk.img is already in + // place). Advertising DV_READY here would let a lease bind against a + // volume that isn't registered in the runner's in-memory state yet, + // which lands the lease in a terminal FAILED (MIR-1469). + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, + } + + _, err = eac.Create(context.Background(), entity.New( + entity.DBId, volEntityId, + vol.Encode, + ).Attrs()) + if err != nil { + return fmt.Errorf("creating disk_volume entity: %w", err) + } + + // Transition disk to PROVISIONING (not PROVISIONED). The DiskController + // promotes it to PROVISIONED only once the disk_volume actually reaches + // DV_READY — i.e. after the volume is registered and mounted on the node — + // so the disk never reports "provisioned" (and thus leasable) ahead of a + // real mount. + _, err = eac.Patch(context.Background(), []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volId), + }, 0) + if err != nil { + // Also clean up the volume entity + if _, derr := eac.Delete(context.Background(), string(volEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk_volume entity: %v", derr) + } + return fmt.Errorf("updating disk to provisioned: %w", err) + } + + // All entities committed — remove leftover metadata and disarm rollback + os.Remove(filepath.Join(destPath, "metadata.json")) + committed = true + + ctx.Info("Disk restored successfully") + ctx.Info(" Disk ID: %s", diskEntityId) + ctx.Info(" Volume ID: %s", volId) + ctx.Info(" Image: %s", imagePath) + + return nil +} diff --git a/cli/commands/disk_backup.go b/cli/commands/disk_backup.go index 86901c8c4..114b9da77 100644 --- a/cli/commands/disk_backup.go +++ b/cli/commands/disk_backup.go @@ -1,59 +1,61 @@ package commands import ( + "errors" "fmt" + "io" "os" - "path/filepath" "time" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/components/coordinate" - "miren.dev/runtime/components/diskio" - "miren.dev/runtime/pkg/cloudauth" - "miren.dev/runtime/pkg/registration" - "miren.dev/runtime/pkg/snapshot" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/rpc/stream" ) -// DiskBackup backs up a disk to a compressed snapshot file. +// DiskBackup backs up a disk, driving the server over RPC. +// +// There is one path here whether you run this on the server host or on your +// laptop, so there is no local-versus-remote detection and no failure mode where +// that detection guesses wrong. The image bytes and the cluster's miren.cloud +// key both stay on the server; see RFD-108. func DiskBackup(ctx *Context, opts struct { ConfigCentric - Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` - Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` - Cloud bool `long:"cloud" description:"Also upload the snapshot to miren.cloud as a restore point"` - Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` + Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` + Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` + Cloud bool `long:"cloud" description:"Upload the snapshot to miren.cloud as a restore point instead of writing a local file"` + Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` }) (retErr error) { - if _, err := os.Stat(opts.DataPath); err != nil { - return fmt.Errorf("data path %s not found — disk backup must be run on the server", opts.DataPath) + // Checked before the client is built, so a flag mistake reports the flag + // rather than whatever the network had to say about reaching the cluster. + if opts.Pin != "" && !opts.Cloud { + return fmt.Errorf("--pin names a restore point in miren.cloud, so it only applies with --cloud") } - client, err := ctx.RPCClient("entities") + client, err := ctx.RPCClient(diskBackupService) if err != nil { return err } + dc := disk_v1alpha.NewDiskBackupClient(client) - eac := entityserver_v1alpha.NewEntityAccessClient(client) - resolver := newEntityDiskResolver(eac, nil) + start := time.Now() + progress := diskProgress(ctx) - target, err := snapshot.PrepareBackup(ctx, resolver, opts.Name, opts.DataPath) + transferID, err := newTransferID() if err != nil { return err } - imgInfo, err := os.Stat(target.ImagePath) - if err != nil { - return fmt.Errorf("disk image not found at %s: %w", target.ImagePath, err) - } + if opts.Cloud { + ctx.Info("Backing up disk %q to miren.cloud", opts.Name) - if target.IsAttached { - // Nothing here freezes the filesystem or takes a copy-on-write clone, so - // this is a sequential read of a file the loop device is still writing. - // The head and tail of the image come from different moments, which is - // weaker than the power-loss state fsck and Postgres recovery are built - // for. Say so plainly: the operator is the one deciding this is safe. - ctx.Warn("Disk is attached and may be written during the backup.") - ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") - ctx.Warn("Detach the disk first for a backup you can rely on.") + // No resume here: the bytes go from the server straight to the cloud, + // so nothing is riding on this connection but the request itself. + res, err := dc.Backup(ctx, opts.Name, true, opts.Pin, nil, progress, transferID, 0) + if err != nil { + return err + } + reportBackup(ctx, res.Result(), time.Since(start)) + ctx.Info(" Restore point: %s", res.Result().RestorePointId()) + return nil } outputPath := opts.Output @@ -61,156 +63,96 @@ func DiskBackup(ctx *Context, opts struct { outputPath = fmt.Sprintf("%s-%s.miren.zst", opts.Name, time.Now().Format("20060102-150405")) } - ctx.Info("Backing up disk %q (%s)", opts.Name, formatBytes(imgInfo.Size())) - ctx.Info("Image: %s", target.ImagePath) - ctx.Info("Output: %s", outputPath) - - start := time.Now() - - imgFile, err := os.Open(target.ImagePath) - if err != nil { - return fmt.Errorf("opening image file: %w", err) - } - defer imgFile.Close() - - outFile, err := os.Create(outputPath) + // Deliberately not os.Create. This command cleans up a snapshot that never + // finished, so truncating an existing file and then failing would destroy a + // good backup to produce nothing, which is the one outcome a backup command + // must not have. + outFile, err := os.OpenFile(outputPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("%s already exists — choose another --output path, or move it aside", outputPath) + } return fmt.Errorf("creating output file: %w", err) } - // Set once the snapshot itself is written. A later step failing (the cloud - // upload) must not take a finished local backup down with it. - snapshotComplete := false + // Discard a snapshot that never finished. A truncated .miren.zst that looks + // like a backup is worse than no file at all. + complete := false defer func() { closeErr := outFile.Close() if closeErr != nil && retErr == nil { retErr = fmt.Errorf("closing output file: %w", closeErr) } - // Only discard a snapshot that never finished, or one whose close - // failed and may therefore be truncated. - if retErr != nil && (!snapshotComplete || closeErr != nil) { + if retErr != nil || !complete { os.Remove(outputPath) } }() - ctx.Info("Computing checksum and compressing...") - - checksum, err := snapshot.Backup(outFile, imgFile, opts.Name, imgInfo.Size(), target.Filesystem) - if err != nil { - return err - } - - outInfo, err := outFile.Stat() - if err != nil { - return fmt.Errorf("stat output file: %w", err) - } - - snapshotComplete = true + ctx.Info("Backing up disk %q", opts.Name) + ctx.Info("Output: %s", outputPath) - duration := time.Since(start) - ratio := float64(outInfo.Size()) / float64(imgInfo.Size()) * 100 + var result *disk_v1alpha.BackupResult - ctx.Info("Backup complete") - ctx.Info(" Original size: %s", formatBytes(imgInfo.Size())) - ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(outInfo.Size()), ratio) - ctx.Info(" Checksum: %s", checksum) - ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) - ctx.Info(" Snapshot: %s", outputPath) + err = runTransfer(ctx, "Backup", func(try int) error { + // What is durably in the file is the resume point, and it is the + // client's to know: it is the only end that can see how much of the + // stream actually reached disk. + offset, oerr := syncedSize(outFile) + if oerr != nil { + return oerr + } - if opts.Cloud { - updateID, err := uploadSnapshotToCloud(ctx, opts.DataPath, target, outputPath, cloudSnapshotDetails{ - Pin: opts.Pin, - ImageSize: imgInfo.Size(), - ImageChecksum: checksum, - CompressedSize: outInfo.Size(), - }) - if err != nil { - // The local snapshot is finished and valid; say where it is rather - // than leaving the operator thinking they got nothing. - ctx.Warn("Upload failed, but the local snapshot is intact at %s", outputPath) - return fmt.Errorf("uploading snapshot to miren.cloud: %w", err) + res, berr := dc.Backup(ctx, opts.Name, false, "", stream.ServeWriter(ctx, outFile), progress, transferID, offset) + if berr != nil { + return berr } - ctx.Info(" Uploaded: %s", updateID) + result = res.Result() + return nil + }) + if err != nil { + return err } + complete = true + reportBackup(ctx, result, time.Since(start)) + ctx.Info(" Snapshot: %s", outputPath) return nil } -// cloudSnapshotDetails carries what the sidecar records about the image the -// snapshot was taken from. It mirrors what diskio.ImageSnapshotter writes, so a -// restore sees the same fields whichever path produced the update. -type cloudSnapshotDetails struct { - Pin string - ImageSize int64 - ImageChecksum string - CompressedSize int64 -} - -// uploadSnapshotToCloud sends a finished snapshot file to miren.cloud as a -// loop_image update for the disk's volume. +// syncedSize flushes what has been written and reports how much of it is +// durably on disk. // -// The cluster's service account key lives in the registration this server wrote -// at enrolment, which is why this has to run on the server alongside the data -// directory. -func uploadSnapshotToCloud(ctx *Context, dataPath string, target *snapshot.BackupTarget, snapshotPath string, details cloudSnapshotDetails) (string, error) { - if target.CloudVolumeID == "" { - return "", fmt.Errorf("disk %q is not registered with miren.cloud yet; "+ - "registration happens on the next disk reconcile, so try again shortly", target.Name) - } - - reg, err := registration.LoadRegistration(filepath.Join(dataPath, "server")) - if err != nil { - return "", fmt.Errorf("loading cluster registration: %w", err) - } - if reg == nil || reg.Status != "approved" || reg.PrivateKey == "" { - return "", fmt.Errorf("this cluster is not registered with miren.cloud") - } - - keyPair, err := cloudauth.LoadKeyPairFromPEM(reg.PrivateKey) +// This is the number the server is told to continue from, so it has to be what +// survived rather than what was handed to the kernel: asking to resume past +// bytes that a crash could still take back would leave a hole in the snapshot. +func syncedSize(f *os.File) (int64, error) { + if err := f.Sync(); err != nil { + return 0, fmt.Errorf("flushing snapshot: %w", err) + } + info, err := f.Stat() if err != nil { - return "", fmt.Errorf("parsing cluster key: %w", err) + return 0, fmt.Errorf("stat snapshot: %w", err) } - - cloudURL := reg.CloudURL - if cloudURL == "" { - cloudURL = coordinate.DefaultCloudURL - } - - authClient, err := cloudauth.NewAuthClient(cloudURL, keyPair) - if err != nil { - return "", fmt.Errorf("creating cloud auth client: %w", err) - } - - file, err := os.Open(snapshotPath) - if err != nil { - return "", fmt.Errorf("opening snapshot: %w", err) - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - return "", fmt.Errorf("stat snapshot: %w", err) + // Writes go on at the end, so the offset has to follow the length: a retry + // that left the file position short would overwrite good bytes. + if _, err := f.Seek(info.Size(), io.SeekStart); err != nil { + return 0, fmt.Errorf("seeking snapshot: %w", err) } + return info.Size(), nil +} - ctx.Info("Uploading to %s...", cloudURL) - - updates := diskio.NewCloudUpdatesClient(ctx.Log, cloudURL, authClient) - // ctx is the command context, so Ctrl-C interrupts the upload rather than - // leaving it running to the end of a multi-gigabyte image. - return updates.Upload(ctx, target.CloudVolumeID, diskio.UploadRequest{ - Kind: diskio.KindLoopImage, - OrderingKey: fmt.Sprintf("%016x", time.Now().UnixNano()), - SnapshotName: details.Pin, - Metadata: map[string]any{ - "compression": "zstd", - "format": "miren-snapshot", - "filesystem": target.Filesystem, - "image_size": details.ImageSize, - "image_sha256": details.ImageChecksum, - "compressed_size": details.CompressedSize, - // Records that this was taken from a live disk, so the image is a - // smear across the read rather than a point-in-time copy. - "was_attached": target.IsAttached, - }, - }, file, info.Size()) +func reportBackup(ctx *Context, res *disk_v1alpha.BackupResult, took time.Duration) { + ctx.Info("Backup complete") + if res == nil { + return + } + ctx.Info(" Original size: %s", formatBytes(res.ImageSizeBytes())) + if res.ImageSizeBytes() > 0 { + ratio := float64(res.CompressedSizeBytes()) / float64(res.ImageSizeBytes()) * 100 + ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(res.CompressedSizeBytes()), ratio) + } else { + ctx.Info(" Compressed size: %s", formatBytes(res.CompressedSizeBytes())) + } + ctx.Info(" Checksum: %s", res.Checksum()) + ctx.Info(" Duration: %s", took.Truncate(time.Millisecond)) } diff --git a/cli/commands/disk_list_deleted.go b/cli/commands/disk_list_deleted.go index 1d3943cc9..ec01ab8b6 100644 --- a/cli/commands/disk_list_deleted.go +++ b/cli/commands/disk_list_deleted.go @@ -1,32 +1,29 @@ package commands import ( - "fmt" - "os" - "path/filepath" "time" - "miren.dev/runtime/components/diskio" + "miren.dev/runtime/api/disk/disk_v1alpha" ) -// DiskListDeleted lists disks that have been soft-deleted and are available -// for recovery via disk undelete. +// DiskListDeleted lists disks whose data is still recoverable, asking the +// server rather than reading its data directory, so it works from anywhere. func DiskListDeleted(ctx *Context, opts struct { FormatOptions ConfigCentric - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` }) error { - diskDataPath := filepath.Join(opts.DataPath, "disk-data") - if _, err := os.Stat(diskDataPath); err != nil { - return fmt.Errorf("data path %s not found — this command must be run on the server", diskDataPath) + client, err := ctx.RPCClient(diskBackupService) + if err != nil { + return err } - entries, err := diskio.ListDeletedVolumes(diskDataPath) + res, err := disk_v1alpha.NewDiskBackupClient(client).ListDeleted(ctx) if err != nil { - return fmt.Errorf("listing deleted volumes: %w", err) + return err } - retentionDays := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + disks := res.Disks() + retentionDays := int(res.RetentionDays()) if opts.IsJSON() { type deletedDiskJSON struct { @@ -39,17 +36,17 @@ func DiskListDeleted(ctx *Context, opts struct { RetentionDays int `json:"retention_days"` } - var items []deletedDiskJSON - for _, e := range entries { - meta := e.Metadata - expiresAt := meta.DeletedAt.Add(time.Duration(retentionDays) * 24 * time.Hour) + // Non-nil, so an empty result marshals as [] rather than null and + // callers can iterate it without a special case. + items := make([]deletedDiskJSON, 0, len(disks)) + for _, d := range disks { items = append(items, deletedDiskJSON{ - DiskName: meta.DiskName, - VolumeID: meta.VolumeID, - SizeGB: meta.SizeGb, - Filesystem: meta.Filesystem, - DeletedAt: meta.DeletedAt.Format(time.RFC3339), - ExpiresAt: expiresAt.Format(time.RFC3339), + DiskName: d.DiskName(), + VolumeID: d.VolumeId(), + SizeGB: d.SizeGb(), + Filesystem: d.Filesystem(), + DeletedAt: rfc3339(d.HasDeletedAt(), d.DeletedAt()), + ExpiresAt: rfc3339(d.HasExpiresAt(), d.ExpiresAt()), RetentionDays: retentionDays, }) } @@ -57,7 +54,7 @@ func DiskListDeleted(ctx *Context, opts struct { return PrintJSON(items) } - if len(entries) == 0 { + if len(disks) == 0 { ctx.Info("No deleted disks found") return nil } @@ -65,25 +62,34 @@ func DiskListDeleted(ctx *Context, opts struct { ctx.Info("Deleted disks available for recovery:") ctx.Info("") - for _, e := range entries { - meta := e.Metadata - age := time.Since(meta.DeletedAt) - remaining := time.Duration(retentionDays)*24*time.Hour - age - - ctx.Info("Name: %s", meta.DiskName) - ctx.Info(" Volume ID: %s", meta.VolumeID) - ctx.Info(" Size: %d GB", meta.SizeGb) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Deleted: %s (%s ago)", meta.DeletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) - if remaining > 0 { - ctx.Info(" Expires in: %s", remaining.Truncate(time.Minute)) + for _, d := range disks { + deletedAt := goTime(d.HasDeletedAt(), d.DeletedAt()) + expiresAt := goTime(d.HasExpiresAt(), d.ExpiresAt()) + + ctx.Info("Name: %s", d.DiskName()) + ctx.Info(" Volume ID: %s", d.VolumeId()) + ctx.Info(" Size: %d GB", d.SizeGb()) + ctx.Info(" Filesystem: %s", d.Filesystem()) + + if deletedAt.IsZero() { + ctx.Info(" Deleted: unknown") } else { + age := time.Since(deletedAt) + ctx.Info(" Deleted: %s (%s ago)", deletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) + } + + switch { + case expiresAt.IsZero(): + ctx.Info(" Expires in: unknown") + case time.Until(expiresAt) > 0: + ctx.Info(" Expires in: %s", time.Until(expiresAt).Truncate(time.Minute)) + default: ctx.Info(" Expires in: imminent (past retention period)") } ctx.Info("") } - ctx.Info("To restore: miren disk undelete --name ") + ctx.Info("To recover: miren disk undelete --name ") return nil } diff --git a/cli/commands/disk_progress.go b/cli/commands/disk_progress.go new file mode 100644 index 000000000..2034a1d35 --- /dev/null +++ b/cli/commands/disk_progress.go @@ -0,0 +1,106 @@ +package commands + +import ( + "fmt" + "os" + "time" + + "golang.org/x/term" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/progress/upload" + "miren.dev/runtime/pkg/rpc/standard" + "miren.dev/runtime/pkg/rpc/stream" +) + +const diskBackupService = "dev.miren.runtime/disk-backup" + +// goTime converts a wire timestamp, treating an absent one as the zero time so +// callers can say "unknown" rather than print 1970. +func goTime(present bool, ts *standard.Timestamp) time.Time { + if !present || ts == nil { + return time.Time{} + } + return time.Unix(ts.Seconds(), int64(ts.Nanoseconds())) +} + +// rfc3339 renders a wire timestamp for JSON output, empty when absent. +func rfc3339(present bool, ts *standard.Timestamp) string { + t := goTime(present, ts) + if t.IsZero() { + return "" + } + return t.Format(time.RFC3339) +} + +// diskProgress renders the server's progress events for backup and restore. +// +// Disk images are large enough that a silent wait is indistinguishable from a +// hang, so transfer events get a line that updates in place on a terminal. Off a +// terminal the same events are printed one per line and throttled, so a log does +// not fill with thousands of near-identical rows. +func diskProgress(ctx *Context) stream.SendStream[*disk_v1alpha.Progress] { + tty := term.IsTerminal(int(os.Stdout.Fd())) + var lastLine time.Time + inPlace := false + + // Clear a partially written in-place line before printing anything else, so + // a message never lands on top of a progress bar. + clear := func() { + if inPlace { + fmt.Fprint(os.Stdout, "\r\033[K") + inPlace = false + } + } + + return stream.Callback(func(p *disk_v1alpha.Progress) error { + switch p.Update().Which() { + case "message": + clear() + ctx.Info("%s", p.Update().Message()) + + case "warning": + clear() + ctx.Warn("%s", p.Update().Warning()) + + case "error": + clear() + ctx.Warn("%s", p.Update().Error()) + + case "transfer": + t := p.Update().Transfer() + if t == nil { + return nil + } + line := formatTransfer(t) + + if tty { + fmt.Fprintf(os.Stdout, "\r\033[K %s", line) + inPlace = true + return nil + } + // One line per half second is enough to show life in a log. + if now := time.Now(); now.Sub(lastLine) >= 500*time.Millisecond { + lastLine = now + ctx.Info(" %s", line) + } + } + return nil + }) +} + +func formatTransfer(t *disk_v1alpha.Transfer) string { + moved := upload.FormatBytes(t.Done()) + speed := upload.FormatSpeed(float64(t.BytesPerSecond())) + + if t.Total() <= 0 { + return fmt.Sprintf("%s at %s", moved, speed) + } + + pct := float64(t.Done()) / float64(t.Total()) * 100 + out := fmt.Sprintf("%s of %s (%.0f%%) at %s", + moved, upload.FormatBytes(t.Total()), pct, speed) + if eta := t.EtaSeconds(); eta > 0 { + out += fmt.Sprintf(", %s left", upload.FormatDuration(time.Duration(eta)*time.Second)) + } + return out +} diff --git a/cli/commands/disk_resolver.go b/cli/commands/disk_resolver.go deleted file mode 100644 index 63533b40a..000000000 --- a/cli/commands/disk_resolver.go +++ /dev/null @@ -1,292 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - compute "miren.dev/runtime/api/compute/compute_v1alpha" - "miren.dev/runtime/api/entityserver" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/api/storage/storage_v1alpha" - "miren.dev/runtime/pkg/entity" - "miren.dev/runtime/pkg/idgen" - "miren.dev/runtime/pkg/snapshot" -) - -// entityDiskResolver implements snapshot.DiskResolver using the entity -// access RPC client. -type entityDiskResolver struct { - eac *entityserver_v1alpha.EntityAccessClient - ec *entityserver.Client -} - -func newEntityDiskResolver(eac *entityserver_v1alpha.EntityAccessClient, ec *entityserver.Client) *entityDiskResolver { - return &entityDiskResolver{eac: eac, ec: ec} -} - -func (r *entityDiskResolver) FindDisk(ctx context.Context, name string) (*snapshot.DiskState, error) { - ref := entity.Ref(entity.EntityKind, storage_v1alpha.KindDisk) - results, err := r.eac.List(ctx, ref) - if err != nil { - return nil, fmt.Errorf("listing disks: %w", err) - } - - var matches []snapshot.DiskState - for _, e := range results.Values() { - var disk storage_v1alpha.Disk - disk.Decode(e.Entity()) - if disk.Name == name { - matches = append(matches, snapshot.DiskState{ - ID: string(disk.ID), - Name: disk.Name, - Status: string(disk.Status), - Filesystem: strings.TrimPrefix(string(disk.Filesystem), "filesystem."), - }) - } - } - - switch len(matches) { - case 0: - return nil, fmt.Errorf("disk %q not found", name) - case 1: - return &matches[0], nil - default: - return nil, fmt.Errorf("multiple disks found with name %q (%d matches)", name, len(matches)) - } -} - -func (r *entityDiskResolver) FindVolume(ctx context.Context, diskID string) (*snapshot.VolumeState, error) { - resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskVolumeDiskIdId, entity.Id(diskID))) - if err != nil { - return nil, fmt.Errorf("listing disk volumes: %w", err) - } - - values := resp.Values() - if len(values) == 0 { - return nil, fmt.Errorf("no disk volume found for disk %s", diskID) - } - if len(values) > 1 { - return nil, fmt.Errorf("multiple disk volumes found for disk %s (%d matches)", diskID, len(values)) - } - - var vol storage_v1alpha.DiskVolume - vol.Decode(values[0].Entity()) - return &snapshot.VolumeState{ - VolumeID: vol.VolumeId, - // Where the disk controller records the volume's id in miren.cloud - // once it has registered it there. - CloudVolumeID: vol.CloudVolumeId, - ImagePath: vol.ImagePath, - }, nil -} - -// CreateDiskAndVolume creates a new disk entity in RESTORING state so the disk -// controller ignores it while restore writes the image. The returned -// RestoreTarget includes a Finalize callback that creates the disk_volume -// entity and transitions the disk to PROVISIONED. -func (r *entityDiskResolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { - sizeGb := sizeBytes / (1 << 30) - if sizeGb == 0 { - sizeGb = 1 - } - - // Normalize filesystem string — strip enum prefix if present - filesystem = strings.TrimPrefix(strings.ToLower(filesystem), "filesystem.") - - var fs storage_v1alpha.DiskFilesystem - switch filesystem { - case "ext4": - fs = storage_v1alpha.EXT4 - case "xfs": - fs = storage_v1alpha.XFS - case "btrfs": - fs = storage_v1alpha.BTRFS - default: - fs = storage_v1alpha.EXT4 - } - - diskId := idgen.GenNS("disk") - volId := idgen.GenNS("disk-vol") - imagePath := filepath.Join(dataPath, "disk-data", "volumes", volId, "disk.img") - - disk := &storage_v1alpha.Disk{ - Name: name, - SizeGb: sizeGb, - Filesystem: fs, - Status: storage_v1alpha.RESTORING, - } - - diskEntityId, err := r.ec.Create(ctx, diskId, disk) - if err != nil { - return nil, fmt.Errorf("creating disk entity: %w", err) - } - - nodeId, err := r.findNodeId(ctx) - if err != nil { - return nil, fmt.Errorf("finding node: %w", err) - } - - return &snapshot.RestoreTarget{ - Name: name, - ImagePath: imagePath, - Created: true, - Cleanup: func(cctx context.Context) error { - // The restore's own context is already cancelled when the - // operator interrupts the restore, and the rollback is exactly - // what has to run then. Detach so cleanup is not defeated by - // the failure it is cleaning up after. - cctx = context.WithoutCancel(cctx) - - // Finalize is the last fallible step of a restore, so cleanup - // only ever runs when it did not complete — and its - // disk_volume Create is its last write. Nothing owns the image - // at this point, and the restore already renamed it into - // place, so the temp-file removal in disk_restore.go is a - // no-op for it. Remove it before touching the disk: a - // PROVISIONED disk still carrying its VolumeId can be - // self-healed into a disk_volume by the controller, and there - // is no reason to let that adopt an image on its way to being - // torn down. Tolerate "not present" — the rename may never - // have happened. - imageErr := os.Remove(imagePath) - if os.IsNotExist(imageErr) { - imageErr = nil - } - - // Transition the disk to DELETING rather than deleting the - // entity outright. A direct Delete bypasses the disk - // controller's DELETING-driven handleDeletion path, which is - // the only writer of disk_volume.desired_state=DV_ABSENT; once - // the disk is gone, a disk_volume the controller self-healed - // from the PROVISIONED disk has no reaper and is left - // reconciled by the coordinator as a live mount / volume - // directory / phantom cloud volume. Marking the disk DELETING - // keeps it alive to drive that existing contract, which tears - // the disk_volume down via the coordinator's - // DiskVolumeController. Idempotent: patching an already-DELETING - // disk is a no-op. - _, err := r.eac.Patch(cctx, []entity.Attr{ - entity.Ref(entity.DBId, diskEntityId), - entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), - }, 0) - if err != nil { - return fmt.Errorf("transitioning disk to deleting during cleanup: %w", err) - } - - // Reported only once the authoritative step is done, so a - // failure to reclaim the image never costs us the disk - // rollback — a leftover image is disk space, a stuck - // RESTORING disk blocks every same-name retry. - if imageErr != nil { - return fmt.Errorf("removing restored image during cleanup: %w", imageErr) - } - return nil - }, - Finalize: func(fctx context.Context) error { - vol := &storage_v1alpha.DiskVolume{ - Name: name, - DiskId: diskEntityId, - VolumeId: volId, - SizeGb: sizeGb, - Filesystem: filesystem, - VolumeMode: detectVolumeMode(), - DesiredState: storage_v1alpha.DV_PRESENT, - ActualState: storage_v1alpha.DV_READY, - ImagePath: imagePath, - NodeId: nodeId, - } - - // Transition the disk to PROVISIONED before creating the - // disk_volume. These are two independent, non-transactional - // writes, so the order matters: if the disk_volume were - // created first and the Patch then failed, the deferred - // Cleanup would orphan the committed disk_volume by deleting - // its parent disk out from under it. Patching first means a - // Create failure leaves no disk_volume behind, and a surviving - // PROVISIONED disk drives the existing DELETING-based cleanup - // and self-healing paths. - _, err := r.eac.Patch(fctx, []entity.Attr{ - entity.Ref(entity.DBId, diskEntityId), - entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisionedId), - entity.String(storage_v1alpha.DiskVolumeIdId, volId), - }, 0) - if err != nil { - return fmt.Errorf("updating disk to provisioned: %w", err) - } - - _, err = r.eac.Create(fctx, entity.New( - entity.DBId, entity.Id("disk_volume/"+volId), - vol.Encode, - ).Attrs()) - if err != nil { - return fmt.Errorf("creating disk_volume entity: %w", err) - } - - return nil - }, - }, nil -} - -// findNodeId finds the coordinator node. Stateful sandboxes (those with -// disk volumes) run on the coordinator, so disk_volume entities must -// reference it. -func (r *entityDiskResolver) findNodeId(ctx context.Context) (entity.Id, error) { - resp, err := r.eac.List(ctx, entity.Ref(entity.EntityKind, compute.KindNode)) - if err != nil { - return "", fmt.Errorf("listing nodes: %w", err) - } - - values := resp.Values() - if len(values) == 0 { - return "", fmt.Errorf("no nodes found") - } - - // If there's only one node, use it. - if len(values) == 1 { - return entity.Id(values[0].Entity().Id()), nil - } - - // Multiple nodes — find the coordinator (role=coordinator constraint). - for _, v := range values { - var node compute.Node - node.Decode(v.Entity()) - if role, _ := node.Constraints.Get("role"); role == "coordinator" { - return node.ID, nil - } - } - - return "", fmt.Errorf("multiple nodes found but none has role=coordinator") -} - -func (r *entityDiskResolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.LeaseState, error) { - resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskLeaseDiskIdId, entity.Id(diskID))) - if err != nil { - return nil, fmt.Errorf("listing disk leases: %w", err) - } - - var leases []snapshot.LeaseState - for _, e := range resp.Values() { - var lease storage_v1alpha.DiskLease - lease.Decode(e.Entity()) - leases = append(leases, snapshot.LeaseState{ - ID: string(lease.ID), - Status: string(lease.Status), - }) - } - - return leases, nil -} - -func detectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { - if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { - return storage_v1alpha.VM_ACCELERATOR - } - if _, err := exec.LookPath("lbdctl"); err == nil { - return storage_v1alpha.VM_ACCELERATOR - } - return storage_v1alpha.VM_UNIVERSAL -} diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index 6c9f90ae4..05d5041cc 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -2,144 +2,210 @@ package commands import ( "fmt" + "io" "os" - "path/filepath" + "time" - "miren.dev/runtime/api/entityserver" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/progress/upload" + "miren.dev/runtime/pkg/rpc/stream" "miren.dev/runtime/pkg/snapshot" + "miren.dev/runtime/pkg/ui" ) -// DiskRestore restores a disk from a compressed snapshot file. +// DiskRestore restores a disk, driving the server over RPC. +// +// Either from a local snapshot file, which is streamed up, or from a restore +// point in miren.cloud, which the server downloads itself. Restoring a disk this +// cluster has never seen creates it, so this works on a freshly built host with +// nothing on it — which is the whole point, since disaster recovery means the +// original host is gone. func DiskRestore(ctx *Context, opts struct { ConfigCentric - Snapshot string `short:"s" long:"snapshot" description:"Path to snapshot file" required:"true"` - Name string `short:"n" long:"name" description:"Disk name to restore to (default: original name from snapshot)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` - Force bool `short:"f" long:"force" description:"Overwrite existing disk image without confirmation"` -}) (retErr error) { - if _, err := os.Stat(opts.DataPath); err != nil { - return fmt.Errorf("data path %s not found — disk restore must be run on the server", opts.DataPath) + Snapshot string `short:"s" long:"snapshot" description:"Path to a snapshot file to restore from"` + Name string `short:"n" long:"name" description:"Disk name to restore to (default: the name recorded in the snapshot)"` + FromCloud bool `long:"from-cloud" description:"Restore from a miren.cloud restore point"` + RestorePoint string `long:"restore-point" description:"Restore point to use (implies --from-cloud; default: the newest)"` + Force bool `short:"f" long:"force" description:"Overwrite an existing disk image"` +}) error { + client, err := ctx.RPCClient(diskBackupService) + if err != nil { + return err } + dc := disk_v1alpha.NewDiskBackupClient(client) - snapFile, err := os.Open(opts.Snapshot) - if err != nil { - return fmt.Errorf("opening snapshot file: %w", err) + fromCloud := opts.FromCloud || opts.RestorePoint != "" + if opts.Snapshot == "" && !fromCloud { + return fmt.Errorf("restore needs either --snapshot to read a local file or --from-cloud to fetch a restore point") } - defer snapFile.Close() + if opts.Snapshot != "" && fromCloud { + return fmt.Errorf("pass either --snapshot or --from-cloud, not both") + } + + start := time.Now() + progress := diskProgress(ctx) + + if fromCloud { + if opts.Name == "" { + return fmt.Errorf("--name says which disk's restore points to look at, so it is required with --from-cloud") + } + + point := opts.RestorePoint + if point == "" { + point, err = pickRestorePoint(ctx, dc, opts.Name) + if err != nil { + return err + } + } - meta, err := snapshot.ReadHeader(snapFile) + ctx.Info("Restoring disk %q from restore point %s", opts.Name, point) + + // No resume here: the server fetches from the cloud over its own + // connection, so nothing rides on this one but the request. + res, err := dc.Restore(ctx, opts.Name, point, nil, opts.Force, progress, "", 0) + if err != nil { + return err + } + reportRestore(ctx, res.Result(), time.Since(start)) + return nil + } + + snapFile, err := os.Open(opts.Snapshot) if err != nil { - return fmt.Errorf("reading snapshot header: %w", err) + return fmt.Errorf("opening snapshot: %w", err) } + defer snapFile.Close() - diskName := opts.Name - if diskName == "" { - diskName = meta.Name + name := opts.Name + if name == "" { + // The snapshot records the disk it came from, so restoring one back + // where it belongs needs no --name. The server reads the header again + // for the size and filesystem; this read is only to answer "which + // disk", which the client has to know before it can ask. + meta, err := snapshot.ReadHeader(snapFile) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + name = meta.Name + if name == "" { + return fmt.Errorf("%s records no disk name, so pass --name", opts.Snapshot) + } + if _, err := snapFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewinding snapshot: %w", err) + } } - ctx.Info("Restoring from snapshot: %s", opts.Snapshot) - ctx.Info(" Original disk: %s", meta.Name) - ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Created: %s", meta.Timestamp.Format(time.RFC3339)) - ctx.Info(" Checksum: %s", meta.Checksum) - ctx.Info(" Target disk: %s", diskName) + ctx.Info("Restoring disk %q from %s", name, opts.Snapshot) - client, err := ctx.RPCClient("entities") + transferID, err := newTransferID() if err != nil { return err } - eac := entityserver_v1alpha.NewEntityAccessClient(client) - ec := entityserver.NewClient(ctx.Log, eac) - resolver := newEntityDiskResolver(eac, ec) + var result *disk_v1alpha.RestoreResult - target, err := snapshot.PrepareRestore(ctx, resolver, diskName, opts.DataPath, - snapshot.WithCreator(resolver, meta.SizeBytes, meta.Filesystem), - ) - if err != nil { - return err - } + err = runTransfer(ctx, "Restore", func(try int) error { + // The server is the only end that knows how much of the upload it + // durably has, so ask rather than assume. An id it has never seen + // reports zero, which makes the first attempt and a retry identical. + off, oerr := dc.TransferOffset(ctx, transferID) + if oerr != nil { + return oerr + } + offset := off.ReceivedBytes() - // If the disk was freshly created and anything fails, clean up the - // RESTORING disk entity so it doesn't become a zombie. - if target.Created && target.Cleanup != nil { - defer func() { - if retErr != nil { - ctx.Warn("Cleaning up disk entity after failed restore") - if cerr := target.Cleanup(ctx); cerr != nil { - ctx.Warn("Failed to clean up disk entity: %v", cerr) - } - } - }() - } + if _, serr := snapFile.Seek(offset, io.SeekStart); serr != nil { + return fmt.Errorf("seeking snapshot to %d: %w", offset, serr) + } - if !target.Created { - if _, err := os.Stat(target.ImagePath); err == nil { - if !opts.Force { - return fmt.Errorf("disk image already exists at %s — use --force to overwrite", target.ImagePath) - } - ctx.Warn("Overwriting existing disk image at %s", target.ImagePath) - } else if !os.IsNotExist(err) { - return fmt.Errorf("checking existing image at %s: %w", target.ImagePath, err) + // readerOnly, not snapFile itself: ServeReader closes what it is given + // once it reaches the end of the stream, and a retry after a fully + // delivered upload has to seek this same file again. Ownership stays + // with the defer above. + res, rerr := dc.Restore(ctx, name, "", + stream.ServeReader(ctx, readerOnly{snapFile}, stream.WithBulkBatching()), + opts.Force, progress, transferID, offset) + if rerr != nil { + return rerr } + result = res.Result() + return nil + }) + if err != nil { + return err } - ctx.Info("Restoring to: %s", target.ImagePath) + reportRestore(ctx, result, time.Since(start)) + return nil +} - start := time.Now() +// restorePointItem adapts a restore point to the shared picker. +type restorePointItem struct { + point *disk_v1alpha.RestorePoint +} - if err := os.MkdirAll(filepath.Dir(target.ImagePath), 0o755); err != nil { - return fmt.Errorf("creating image directory: %w", err) +func (r restorePointItem) ID() string { return r.point.Id() } + +func (r restorePointItem) Row() []string { + when := "unknown" + if t := goTime(r.point.HasCreatedAt(), r.point.CreatedAt()); !t.IsZero() { + when = t.Format("2006-01-02 15:04:05") + } + return []string{ + when, + upload.FormatBytes(r.point.SizeBytes()), + r.point.Name(), + r.point.Id(), } +} - tmpPath := target.ImagePath + ".restore.tmp" - outFile, err := os.Create(tmpPath) +// pickRestorePoint lists what is available and asks, so an operator recovering a +// disk does not have to already know a restore point id. +func pickRestorePoint(ctx *Context, dc *disk_v1alpha.DiskBackupClient, name string) (string, error) { + res, err := dc.ListBackups(ctx, name) if err != nil { - return fmt.Errorf("creating temp image file: %w", err) + return "", err } - defer func() { - outFile.Close() - if retErr != nil { - os.Remove(tmpPath) - } - }() - if err := outFile.Truncate(meta.SizeBytes); err != nil { - return fmt.Errorf("truncating image file: %w", err) + points := res.Points() + if len(points) == 0 { + return "", fmt.Errorf("disk %q has no restore points in miren.cloud", name) } - ctx.Info("Decompressing...") - - if err := snapshot.RestoreImage(outFile, snapFile, meta); err != nil { - return err + // The server returns newest first. On a non-interactive run take that one: + // it is what a recovery almost always wants, and there is nobody to ask. + if !ui.IsInteractive() { + ctx.Info("Using the newest restore point (%s of %d)", points[0].Id(), len(points)) + return points[0].Id(), nil } - if err := outFile.Close(); err != nil { - return fmt.Errorf("closing restored image: %w", err) + items := make([]ui.PickerItem, 0, len(points)) + for _, p := range points { + items = append(items, restorePointItem{point: p}) } - if err := os.Rename(tmpPath, target.ImagePath); err != nil { - return fmt.Errorf("moving restored image into place: %w", err) + chosen, err := ui.RunPicker(items, + ui.WithTitle(fmt.Sprintf("Restore point for %q", name)), + ui.WithHeaders([]string{"Taken", "Size", "Pinned as", "ID"}), + ) + if err != nil { + return "", err } - - if target.Finalize != nil { - ctx.Info("Finalizing disk entities...") - if err := target.Finalize(ctx); err != nil { - return fmt.Errorf("finalizing restore: %w", err) - } + if chosen == nil { + return "", fmt.Errorf("no restore point selected") } + return chosen.ID(), nil +} - duration := time.Since(start) - +func reportRestore(ctx *Context, res *disk_v1alpha.RestoreResult, took time.Duration) { ctx.Info("Restore complete") - ctx.Info(" Disk: %s", diskName) - ctx.Info(" Image: %s", target.ImagePath) - ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) - ctx.Info(" Checksum: verified") - ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) - - return nil + if res == nil { + return + } + if res.Created() { + ctx.Info(" Created disk: %s", res.Disk()) + } + ctx.Info(" Restored size: %s", formatBytes(res.ImageSizeBytes())) + ctx.Info(" Duration: %s", took.Truncate(time.Millisecond)) } diff --git a/cli/commands/disk_transfer.go b/cli/commands/disk_transfer.go new file mode 100644 index 000000000..1bdd04d05 --- /dev/null +++ b/cli/commands/disk_transfer.go @@ -0,0 +1,112 @@ +package commands + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "time" + + "miren.dev/runtime/pkg/cond" +) + +const ( + // transferAttempts is how many times a backup or restore will pick itself + // up after the connection drops. + // + // This is not a tight retry loop around a flaky call — each attempt resumes + // from where the last one stopped, so the work already done is kept and the + // cost of another attempt is only the time to reconnect. Six covers a + // cluster uplink that flaps repeatedly during one long transfer, which is + // the case that motivated resume in the first place. + transferAttempts = 6 + + // transferRetryDelay is the wait before picking a transfer back up. A + // cluster's link to the cloud can take up to a minute to come back + // (RFD-101), so this backs off toward that rather than hammering. + transferRetryDelay = 5 * time.Second + transferRetryMax = 60 * time.Second +) + +// readerOnly hides a reader's Close from whoever it is handed to. +// +// The stream helper closes what it reads from once it reaches the end, which is +// right when it owns the reader and wrong when the caller still needs it. A +// retry has to seek the same file again, and an upload that got all the way to +// the end before the server rejected it is exactly when that happens: the file +// would already be closed and every later attempt would fail on the seek rather +// than on anything real. +// +// Embedding only io.Reader is what does the hiding: the wrapper has Read and +// nothing else, so a type assertion for io.Closer finds nothing to call. +type readerOnly struct { + io.Reader +} + +// newTransferID names one backup or restore so an interrupted one can be +// resumed. +// +// The client picks it, not the server: an id the server handed back would be +// lost along with the call that carried it, which is precisely the call that +// failed. +func newTransferID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("generating a transfer id: %w", err) + } + return hex.EncodeToString(b), nil +} + +// resumable reports whether an error is worth another attempt. +// +// A refusal is not. The server declining to overwrite a mounted disk, or +// reporting that no such disk exists, means the same thing on every attempt, +// and retrying it six times only makes the operator wait longer to read it. +func resumable(err error) bool { + if err == nil { + return false + } + + var validation cond.ErrValidationFailure + var notFound cond.ErrNotFound + return !errors.As(err, &validation) && !errors.As(err, ¬Found) +} + +// runTransfer calls attempt until it succeeds, the error turns out not to be +// worth retrying, or the attempts run out. +// +// Each attempt is expected to resume rather than restart: this loop exists to +// survive a dropped connection, not to paper over a call that fails the same +// way every time. +func runTransfer(ctx *Context, what string, attempt func(try int) error) error { + delay := transferRetryDelay + + for try := 1; ; try++ { + err := attempt(try) + if err == nil { + return nil + } + if !resumable(err) || try >= transferAttempts { + return err + } + // A cancelled command is the operator's decision, not a failure to + // retry around. + if ctx.Err() != nil { + return err + } + + ctx.Warn("%s was interrupted: %v", what, err) + ctx.Info("Picking up where it stopped in %s (attempt %d of %d)", delay, try+1, transferAttempts) + + select { + case <-time.After(delay): + case <-ctx.Done(): + return err + } + + if delay *= 2; delay > transferRetryMax { + delay = transferRetryMax + } + } +} diff --git a/cli/commands/disk_transfer_test.go b/cli/commands/disk_transfer_test.go new file mode 100644 index 000000000..f81eb9353 --- /dev/null +++ b/cli/commands/disk_transfer_test.go @@ -0,0 +1,128 @@ +package commands + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" +) + +// Retrying a refusal six times only makes the operator wait longer to read a +// message that will say the same thing every time. +func TestRefusalsAreNotRetried(t *testing.T) { + refusals := []error{ + cond.ValidationFailure("disk-backup", `disk "data" is in use`), + cond.NotFound("disk", "data"), + fmt.Errorf("wrapped: %w", cond.ValidationFailure("disk-backup", "no cloud")), + } + for _, err := range refusals { + assert.False(t, resumable(err), "should not retry: %v", err) + } +} + +// A dropped connection is exactly what resume exists for. +func TestTransportFailuresAreRetried(t *testing.T) { + assert.True(t, resumable(errors.New("connection reset by peer"))) + assert.True(t, resumable(errors.New("the cluster's link to the cloud dropped"))) +} + +func TestNilIsNotRetried(t *testing.T) { + assert.False(t, resumable(nil)) +} + +// Two transfers must not collide in the server's staging directory. +func TestTransferIDsAreDistinct(t *testing.T) { + seen := map[string]bool{} + for i := 0; i < 100; i++ { + id, err := newTransferID() + require.NoError(t, err) + require.NotEmpty(t, id) + assert.False(t, seen[id], "transfer ids must not repeat") + seen[id] = true + } +} + +// The id names a file on the server, which validates it — so the ids we +// generate have to be ones it accepts. +func TestTransferIDsUseOnlySafeCharacters(t *testing.T) { + id, err := newTransferID() + require.NoError(t, err) + + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + default: + t.Fatalf("transfer id %q contains %q, which the server rejects", id, r) + } + } +} + +// closeCountingFile stands in for the snapshot file a restore uploads from, +// recording whether anything closed it out from under us. +type closeCountingFile struct { + *os.File + closes int +} + +func (f *closeCountingFile) Close() error { + f.closes++ + return f.File.Close() +} + +// A retry has to be able to seek the snapshot again, and the case that catches +// this is the one where the upload fully succeeded and the server then rejected +// it: the stream reaches EOF, closes what it was reading from, and every later +// attempt fails on the seek rather than on anything real. +// +// readerOnly is what prevents that, by handing the stream something with no +// Close to find. +func TestReaderOnlyHidesCloseFromTheStream(t *testing.T) { + path := filepath.Join(t.TempDir(), "snap.miren.zst") + require.NoError(t, os.WriteFile(path, []byte("a snapshot's worth of bytes"), 0600)) + + f, err := os.Open(path) + require.NoError(t, err) + tracked := &closeCountingFile{File: f} + defer tracked.File.Close() + + // What the stream helper does when it reaches the end: close the reader if + // it can. Wrapped, there is nothing to close. + var wrapped io.Reader = readerOnly{tracked} + _, isCloser := wrapped.(io.Closer) + assert.False(t, isCloser, "the stream must not be able to close the caller's file") + + // Read it to the end, the way a completed upload does. + _, err = io.ReadAll(wrapped) + require.NoError(t, err) + assert.Zero(t, tracked.closes, "reading to EOF must not have closed the file") + + // And a retry can still rewind and re-send it. + _, err = tracked.Seek(0, io.SeekStart) + require.NoError(t, err, "a retry after a complete upload must still be able to seek") + + again, err := io.ReadAll(readerOnly{tracked}) + require.NoError(t, err) + assert.Equal(t, "a snapshot's worth of bytes", string(again)) +} + +// The unwrapped file is the shape that caused the bug, so pin the difference: +// handed the file directly, the stream helper would find a Closer to call. +func TestAnUnwrappedFileWouldBeClosedByTheStream(t *testing.T) { + path := filepath.Join(t.TempDir(), "snap.miren.zst") + require.NoError(t, os.WriteFile(path, []byte("bytes"), 0600)) + + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + var plain io.Reader = f + _, isCloser := plain.(io.Closer) + assert.True(t, isCloser, + "an *os.File is a Closer, which is why it has to be wrapped before it is streamed") +} diff --git a/cli/commands/disk_undelete.go b/cli/commands/disk_undelete.go index e40674848..9df461f2c 100644 --- a/cli/commands/disk_undelete.go +++ b/cli/commands/disk_undelete.go @@ -1,213 +1,36 @@ package commands import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "miren.dev/runtime/api/entityserver" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/api/storage/storage_v1alpha" - "miren.dev/runtime/components/diskio" - "miren.dev/runtime/pkg/entity" - "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/api/disk/disk_v1alpha" ) -// DiskUndelete restores a recently deleted disk from the soft-delete holding area. +// DiskUndelete recovers a recently deleted disk, asking the server to move the +// data back and recreate the entities, so it works from anywhere. func DiskUndelete(ctx *Context, opts struct { ConfigCentric Name string `short:"n" long:"name" description:"Disk name to undelete" required:"true"` - VolumeID string `short:"V" long:"volume-id" description:"Volume ID to restore (when multiple deleted disks share a name)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + VolumeID string `short:"V" long:"volume-id" description:"Volume ID to recover (when several deleted disks share a name)"` }) error { - diskDataPath := filepath.Join(opts.DataPath, "disk-data") - if _, err := os.Stat(diskDataPath); err != nil { - return fmt.Errorf("data path %s not found — disk undelete must be run on the server", diskDataPath) - } - - entries, err := diskio.ListDeletedVolumes(diskDataPath) - if err != nil { - return fmt.Errorf("listing deleted volumes: %w", err) - } - - // Filter by name - var matches []diskio.DeletedVolumeEntry - for _, e := range entries { - if e.Metadata.DiskName == opts.Name { - if opts.VolumeID == "" || e.Metadata.VolumeID == opts.VolumeID { - matches = append(matches, e) - } - } - } - - if len(matches) == 0 { - return fmt.Errorf("no deleted disk found with name %q", opts.Name) - } - - if len(matches) > 1 { - ctx.Info("Multiple deleted disks found with name %q:", opts.Name) - for _, m := range matches { - age := time.Since(m.Metadata.DeletedAt).Truncate(time.Minute) - ctx.Info(" Volume ID: %s (deleted %s ago, %d GB, %s)", - m.Metadata.VolumeID, age, m.Metadata.SizeGb, m.Metadata.Filesystem) - } - return fmt.Errorf("specify --volume-id to select which disk to restore") - } - - entry := matches[0] - meta := entry.Metadata - - ctx.Info("Restoring deleted disk:") - ctx.Info(" Name: %s", meta.DiskName) - ctx.Info(" Size: %d GB", meta.SizeGb) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Deleted: %s", meta.DeletedAt.Format(time.RFC3339)) - - client, err := ctx.RPCClient("entities") + client, err := ctx.RPCClient(diskBackupService) if err != nil { return err } - eac := entityserver_v1alpha.NewEntityAccessClient(client) - ec := entityserver.NewClient(ctx.Log, eac) - resolver := newEntityDiskResolver(eac, ec) - - // Check if a disk with this name already exists - if _, err := resolver.FindDisk(context.Background(), meta.DiskName); err == nil { - return fmt.Errorf("a disk named %q already exists — rename or delete it before restoring", meta.DiskName) - } - - // Normalize filesystem - filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + ctx.Info("Recovering deleted disk %q", opts.Name) - var fs storage_v1alpha.DiskFilesystem - switch filesystem { - case "ext4": - fs = storage_v1alpha.EXT4 - case "xfs": - fs = storage_v1alpha.XFS - case "btrfs": - fs = storage_v1alpha.BTRFS - default: - fs = storage_v1alpha.EXT4 - } - - // Create the disk entity in RESTORING state - diskId := idgen.GenNS("disk") - disk := &storage_v1alpha.Disk{ - Name: meta.DiskName, - SizeGb: meta.SizeGb, - Filesystem: fs, - Status: storage_v1alpha.RESTORING, - } - - diskEntityId, err := ec.Create(context.Background(), diskId, disk) + res, err := disk_v1alpha.NewDiskBackupClient(client).Undelete(ctx, opts.Name, opts.VolumeID) if err != nil { - return fmt.Errorf("creating disk entity: %w", err) - } - - ctx.Info("Created disk entity: %s", diskEntityId) - - // Move the volume directory back - volId := meta.VolumeID - destPath := filepath.Join(diskDataPath, "volumes", volId) - - if err := os.Rename(entry.Path, destPath); err != nil { - // Clean up the disk entity we just created - if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk entity: %v", derr) - } - return fmt.Errorf("moving volume back to %s: %w", destPath, err) - } - - // Deferred rollback: if we don't reach the final commit, move the - // volume back to deleted-volumes (with its metadata intact) and clean - // up any entities we created. - committed := false - defer func() { - if committed { - return - } - if rerr := os.Rename(destPath, entry.Path); rerr != nil { - ctx.Warn("Failed to move volume back to deleted-volumes: %v", rerr) - } - if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk entity: %v", derr) - } - }() - - // Find the node ID - nodeId, err := resolver.findNodeId(context.Background()) - if err != nil { - ctx.Warn("Failed to find node ID, using stored value: %v", err) - nodeId = meta.NodeID.Id() - } - - imagePath := filepath.Join(destPath, "disk.img") - - // Verify the disk image actually exists before creating entities - if _, err := os.Stat(imagePath); err != nil { - return fmt.Errorf("disk image not found at %s — deleted volume may be corrupted", imagePath) - } - - // Create disk_volume entity - volEntityId := entity.Id("disk_volume/" + volId) - vol := &storage_v1alpha.DiskVolume{ - Name: meta.DiskName, - DiskId: diskEntityId, - VolumeId: volId, - SizeGb: meta.SizeGb, - Filesystem: filesystem, - VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), - DesiredState: storage_v1alpha.DV_PRESENT, - // Start PENDING, not READY. The runner's DiskVolumeController drives - // the restored volume through the same mount-then-READY ordering as a - // fresh create (skipping the reimage, since disk.img is already in - // place). Advertising DV_READY here would let a lease bind against a - // volume that isn't registered in the runner's in-memory state yet, - // which lands the lease in a terminal FAILED (MIR-1469). - ActualState: storage_v1alpha.DV_PENDING, - ImagePath: imagePath, - NodeId: nodeId, - } - - _, err = eac.Create(context.Background(), entity.New( - entity.DBId, volEntityId, - vol.Encode, - ).Attrs()) - if err != nil { - return fmt.Errorf("creating disk_volume entity: %w", err) - } - - // Transition disk to PROVISIONING (not PROVISIONED). The DiskController - // promotes it to PROVISIONED only once the disk_volume actually reaches - // DV_READY — i.e. after the volume is registered and mounted on the node — - // so the disk never reports "provisioned" (and thus leasable) ahead of a - // real mount. - _, err = eac.Patch(context.Background(), []entity.Attr{ - entity.Ref(entity.DBId, diskEntityId), - entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), - entity.String(storage_v1alpha.DiskVolumeIdId, volId), - }, 0) - if err != nil { - // Also clean up the volume entity - if _, derr := eac.Delete(context.Background(), string(volEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk_volume entity: %v", derr) - } - return fmt.Errorf("updating disk to provisioned: %w", err) + return err } - // All entities committed — remove leftover metadata and disarm rollback - os.Remove(filepath.Join(destPath, "metadata.json")) - committed = true - + out := res.Result() ctx.Info("Disk restored successfully") - ctx.Info(" Disk ID: %s", diskEntityId) - ctx.Info(" Volume ID: %s", volId) - ctx.Info(" Image: %s", imagePath) + if out == nil { + return nil + } + ctx.Info(" Disk ID: %s", out.DiskId()) + ctx.Info(" Volume ID: %s", out.VolumeId()) + ctx.Info(" Image: %s", out.ImagePath()) return nil } diff --git a/components/coordinate/runner_endpoints.go b/components/coordinate/runner_endpoints.go index d83cf485f..8e38598c8 100644 --- a/components/coordinate/runner_endpoints.go +++ b/components/coordinate/runner_endpoints.go @@ -6,10 +6,14 @@ import ( "fmt" "path/filepath" + "miren.dev/runtime/api/disk/disk_v1alpha" + aes "miren.dev/runtime/api/entityserver" "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/api/sqlitebackup/sqlitebackup_v1alpha" "miren.dev/runtime/api/telemetry/telemetry_v1alpha" + "miren.dev/runtime/components/diskio" "miren.dev/runtime/pkg/rpc" + disksrv "miren.dev/runtime/servers/disk" runnerserver "miren.dev/runtime/servers/runner" sqlitebackupsrv "miren.dev/runtime/servers/sqlitebackup" telemetrysrv "miren.dev/runtime/servers/telemetry" @@ -47,6 +51,28 @@ func (c *RunnerEndpoints) Start(context.Context) error { } server.ExposeValue(rpc.ServiceSqliteBackup, sqlitebackup_v1alpha.AdaptSqliteBackup(backup)) + // Disk backup and restore, driven by a client that need not be on this host + // (RFD-108). The image bytes and the cloud key stay here; everything else is + // the client's orchestration. + // + // It belongs with the other endpoints a host needs before workload control + // starts, for the same reason they do: recovering a disk is something you + // do to a cluster that is not yet running anything. + // + // The updates client is nil when the cluster has no cloud registration, + // which leaves the local-file backup path working and refuses the cloud + // ones with an explanation. + var diskUpdates diskio.CloudUpdatesClient + if c.authClient != nil { + cloudURL := c.CloudAuth.CloudURL + if cloudURL == "" { + cloudURL = DefaultCloudURL + } + diskUpdates = diskio.NewCloudUpdatesClient(c.Log, cloudURL, c.authClient) + } + diskBackup := disksrv.NewServer(c.Log, c.eac, aes.NewClient(c.Log, c.eac), c.DataPath, diskUpdates) + server.ExposeValue("dev.miren.runtime/disk-backup", disk_v1alpha.AdaptDiskBackup(diskBackup)) + runnerReg := runnerserver.NewRegistrationServer(runnerserver.RegistrationServerConfig{ Log: c.Log, Authority: c.authority, diff --git a/components/diskio/disk_ops.go b/components/diskio/disk_ops.go index 7e27783f1..43e15e4e2 100644 --- a/components/diskio/disk_ops.go +++ b/components/diskio/disk_ops.go @@ -21,6 +21,24 @@ type ActiveMount struct { MountPath string } +// LoopBacking describes what a loop device is attached to. +// +// Path and Deleted are reported separately because the two questions callers +// ask are genuinely different. "Is anything holding a file under my volumes +// directory?" wants both, so a stale device can be reclaimed. "Is this image +// already attached, so I should mount that device instead of attaching a second +// one?" wants only live attachments — a loop holding an unlinked inode is not +// this image, it is the file that used to be at this path, and mounting it +// serves data the operator replaced. +type LoopBacking struct { + // Path is the backing file as the kernel reports it. + Path string + // Deleted reports that the backing inode has been unlinked. The file lives + // on because the loop device holds a reference, but nothing can reach it by + // name any more: something renamed or removed the path out from under it. + Deleted bool +} + // DiskMountOps abstracts OS operations for disk mount management. // This interface enables testing without requiring actual loop device or mount operations. type DiskMountOps interface { @@ -34,7 +52,7 @@ type DiskMountOps interface { // FindAllLoopBackings returns a map of loop device path to the backing // file currently attached to it, for every loop device in the kernel. // Used by boot-time orphan reconciliation to find stale attachments. - FindAllLoopBackings() (map[string]string, error) + FindAllLoopBackings() (map[string]LoopBacking, error) LbdAttach(ctx context.Context, imagePath, logDir string) (devicePath string, err error) LbdDetach(ctx context.Context, devicePath string) error Mount(device, mountPath, filesystem string, readOnly bool) error diff --git a/components/diskio/disk_ops_darwin.go b/components/diskio/disk_ops_darwin.go index 2c8050b22..1b50ea28b 100644 --- a/components/diskio/disk_ops_darwin.go +++ b/components/diskio/disk_ops_darwin.go @@ -60,7 +60,7 @@ func (s *stubDiskMountOps) FindLoopByBacking(_ string) (string, error) { return "", nil } -func (s *stubDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (s *stubDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { return nil, nil } diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 0ea0c5dc8..72637179c 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -166,7 +166,8 @@ func (r *realDiskMountOps) LoopDetach(devicePath string) error { } // FindLoopByBacking walks /sys/block/loop*/loop/backing_file and returns the -// loop device path currently backing imagePath, or "" if none is attached. +// loop device path currently backing the file at imagePath, or "" if none is +// attached. // // The kernel never deletes a stale loop device just because miren restarted, // so finding a match here means a previous miren (or an uncleanly shut down @@ -174,6 +175,15 @@ func (r *realDiskMountOps) LoopDetach(devicePath string) error { // produce two loop devices with independent, incoherent page caches and // corrupt the filesystem. Callers should reuse the returned device, fail // loudly, or detach it explicitly. +// +// A loop whose backing inode has been unlinked is deliberately not a match, +// even though the kernel still reports this exact path for it. That device is +// holding the file that used to be here, not the one here now — after a restore +// wrote a new image and renamed it into place, say. Treating it as a match is +// how stale data gets mounted over good data: the caller adopts the device, +// finds the old filesystem on it, decides no formatting is needed, and serves +// the contents the operator just replaced. Use FindAllLoopBackings, which +// reports the distinction, to find such devices and reclaim them. func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { absPath, err := filepath.Abs(imagePath) if err != nil { @@ -185,7 +195,7 @@ func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { return "", err } for dev, backing := range all { - if backing == absPath { + if backing.Path == absPath && !backing.Deleted { return dev, nil } } @@ -195,13 +205,13 @@ func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { // FindAllLoopBackings walks /sys/block/loop*/loop/backing_file and returns // a map of loop device path → backing file path for every loop device in // the kernel. Devices that race with a concurrent detach are skipped. -func (r *realDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (r *realDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { entries, err := filepath.Glob("/sys/block/loop*/loop/backing_file") if err != nil { return nil, fmt.Errorf("failed to glob loop backing files: %w", err) } - result := make(map[string]string, len(entries)) + result := make(map[string]LoopBacking, len(entries)) for _, entry := range entries { data, err := os.ReadFile(entry) if err != nil { @@ -212,19 +222,44 @@ func (r *realDiskMountOps) FindAllLoopBackings() (map[string]string, error) { return nil, fmt.Errorf("failed to read %s: %w", entry, err) } - backing := strings.TrimSpace(string(data)) - // The kernel appends " (deleted)" when the backing inode is - // unlinked; strip it so callers can compare against a live path. - backing = strings.TrimSuffix(backing, " (deleted)") - // entry is /sys/block/loopN/loop/backing_file — extract loopN. loopName := filepath.Base(filepath.Dir(filepath.Dir(entry))) - result["/dev/"+loopName] = backing + result["/dev/"+loopName] = parseLoopBacking(string(data)) } return result, nil } +// deletedBackingSuffix is what the kernel appends to a loop device's backing +// file path once that file has been unlinked. +const deletedBackingSuffix = " (deleted)" + +// parseLoopBacking reads one backing_file line. +// +// The deleted marker is reported rather than stripped: to a caller comparing +// against a live path, a deleted backing that still reads as that path is a +// trap, because the file at the path now is a different file entirely. +// +// What the kernel actually does, confirmed against a live loop device: it +// tracks the backing inode's current name, so renaming the file simply changes +// what this reports, with no marker. The marker appears only once the inode has +// no name left. That is exactly what a restore does to the image it replaces — +// renaming a new image over the old path unlinks the old inode — which is how a +// loop device ends up reporting a path whose file it is no longer holding. +// +// The marker is ambiguous for a file whose own name ends in " (deleted)", and +// the kernel offers no way to tell the two apart. Reading it as deleted is the +// safe way to be wrong: the cost is attaching a second loop device rather than +// adopting one, where the other way round serves stale data. +func parseLoopBacking(raw string) LoopBacking { + path := strings.TrimSpace(raw) + deleted := strings.HasSuffix(path, deletedBackingSuffix) + return LoopBacking{ + Path: strings.TrimSuffix(path, deletedBackingSuffix), + Deleted: deleted, + } +} + func (r *realDiskMountOps) LbdAttach(ctx context.Context, imagePath, logDir string) (string, error) { cmd := exec.CommandContext(ctx, "lbdctl", "add", "--json", imagePath, "--log-dir", logDir) output, err := cmd.CombinedOutput() diff --git a/components/diskio/disk_ops_linux_test.go b/components/diskio/disk_ops_linux_test.go new file mode 100644 index 000000000..f58d0c4c3 --- /dev/null +++ b/components/diskio/disk_ops_linux_test.go @@ -0,0 +1,56 @@ +package diskio + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseLoopBacking(t *testing.T) { + cases := []struct { + name string + raw string + want LoopBacking + }{ + { + name: "a live backing file", + raw: "/var/lib/miren/disk-data/volumes/vol-1/disk.img\n", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/disk.img"}, + }, + { + // What the kernel reports once the file has lost its last name. + // The path is where it was when that happened. + name: "a backing file that has been unlinked", + raw: "/var/lib/miren/disk-data/volumes/vol-1/disk.img (deleted)\n", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/disk.img", Deleted: true}, + }, + { + // A renamed file is followed to its new name with no marker: the + // kernel tracks the inode, and the inode still has a name. Only + // losing its last name marks it. + name: "a backing file that was renamed rather than removed", + raw: "/var/lib/miren/disk-data/volumes/vol-1/renamed.img", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/renamed.img"}, + }, + { + name: "an unlinked file whose own name ended in the marker", + raw: "/tmp/weird (deleted) (deleted)", + want: LoopBacking{Path: "/tmp/weird (deleted)", Deleted: true}, + }, + { + // Genuinely ambiguous — a live file named "weird (deleted)" reads + // identically to a deleted file named "weird", and the kernel gives + // no way to tell. Read as deleted, which is the safe way to be + // wrong: it costs an extra loop device rather than stale data. + name: "the ambiguous case is read as deleted", + raw: "/tmp/weird (deleted)", + want: LoopBacking{Path: "/tmp/weird", Deleted: true}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, parseLoopBacking(tc.raw)) + }) + } +} diff --git a/components/diskio/disk_ops_mock_test.go b/components/diskio/disk_ops_mock_test.go index 117b1cd9e..23dd4203b 100644 --- a/components/diskio/disk_ops_mock_test.go +++ b/components/diskio/disk_ops_mock_test.go @@ -90,6 +90,11 @@ type mockDiskMountOps struct { mountDevices map[string]string // mount path → device, for FindMounts formattedDevs map[string]string formatCalls []diskMockFormat + // deletedBacking marks image paths whose loop device holds an unlinked + // inode — the kernel still reports the path, but the file there now is a + // different file. Set it to model a ghost loop. + deletedBacking map[string]bool + // loopBacking maps imagePath → existing loop device, for FindLoopByBacking. // Tests populate this to simulate a pre-existing attachment. loopBacking map[string]string @@ -191,19 +196,21 @@ func (m *mockDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { if m.loopBacking == nil { return "", nil } + // A device holding an unlinked inode is not this image, so it is not a + // match — the same rule the real implementation follows. + if m.deletedBacking[imagePath] { + return "", nil + } return m.loopBacking[imagePath], nil } -func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { if m.findLoopErr != nil { return nil, m.findLoopErr } - if m.loopBacking == nil { - return map[string]string{}, nil - } - result := make(map[string]string, len(m.loopBacking)) + result := make(map[string]LoopBacking, len(m.loopBacking)) for imagePath, dev := range m.loopBacking { - result[dev] = imagePath + result[dev] = LoopBacking{Path: imagePath, Deleted: m.deletedBacking[imagePath]} } return result, nil } diff --git a/components/diskio/disk_volume_controller.go b/components/diskio/disk_volume_controller.go index f5cf496e7..2745efa4f 100644 --- a/components/diskio/disk_volume_controller.go +++ b/components/diskio/disk_volume_controller.go @@ -421,13 +421,37 @@ func (c *DiskVolumeController) deleteVolume(ctx context.Context, volume *storage c.log.Info("deleting disk volume", "entity_id", entityId) + volState := c.state.GetVolume(entityId) + if volState == nil { + c.log.Warn("volume not found in state", "entity_id", entityId) + // Restore and undelete can put an image in place before the volume's + // first reconcile. Keep the same recovery window as a tracked volume. + if volume.ActualState == storage_v1alpha.DV_PENDING { + volumePath := c.getVolumePath(entityId) + if c.ops.VolumePathExists(volumePath) { + imagePath := filepath.Join(volumePath, "disk.img") + dev, err := c.mntOps.FindLoopByBacking(imagePath) + if err != nil { + return fmt.Errorf("checking pending volume backing: %w", err) + } + if dev != "" { + return fmt.Errorf("pending volume image %s is still attached to %s", imagePath, dev) + } + pending := &VolumeState{EntityId: entityId, VolumeId: localVolumeID(volume), DiskPath: volumePath} + if err := c.softDeleteVolume(ctx, volume, pending); err != nil { + // Without local state this may be the only copy of an undeleted + // disk. Never fall back to hard deletion if the move fails. + return fmt.Errorf("soft-deleting untracked pending volume %s: %w", volumePath, err) + } + } + } + } + if err := c.updateVolumeState(ctx, volume.ID, storage_v1alpha.DV_DELETING, "", ""); err != nil { c.log.Warn("failed to update volume state to deleting", "error", err) } - volState := c.state.GetVolume(entityId) if volState == nil { - c.log.Warn("volume not found in state", "entity_id", entityId) if err := c.updateVolumeState(ctx, volume.ID, storage_v1alpha.DV_DELETED, "", ""); err != nil { c.log.Warn("failed to update volume state to deleted", "error", err) } @@ -648,6 +672,12 @@ func (c *DiskVolumeController) ensureVolumeMount(ctx context.Context, entityId s ) devicePath = existing } else { + // A loop device may still be holding the file that used to be at this + // path — after a restore replaced the image, say. It is not adopted, + // because it would serve the old contents, but it does leak a device + // until the orphan sweep reclaims it, so say so. + c.warnAboutStaleLoop(entityId, imagePath) + // No existing loop. Any stale volState.DevicePath is meaningless // (the kernel has no loop backing this image), so we don't touch // it — the loop index it names may have been reallocated to some @@ -810,6 +840,32 @@ func (c *DiskVolumeController) Shutdown() { } } +// warnAboutStaleLoop reports a loop device still holding a file that used to be +// at imagePath. +// +// Best effort, and never fatal: this is only here to make a leaked device +// visible. The mount that follows attaches the live image on its own device and +// is correct either way, and the orphan sweep reclaims the leftover. +func (c *DiskVolumeController) warnAboutStaleLoop(entityId, imagePath string) { + abs, err := filepath.Abs(imagePath) + if err != nil { + return + } + backings, err := c.mntOps.FindAllLoopBackings() + if err != nil { + return + } + for dev, backing := range backings { + if backing.Deleted && backing.Path == abs { + c.log.Warn("a loop device still holds the file that used to be at this path", + "entity_id", entityId, + "image_path", imagePath, + "device", dev, + ) + } + } +} + // reconcileOrphanKernelState runs once at boot and tears down any kernel // loop devices or mounts that are rooted in miren's volumes directory but // that no longer correspond to a known volume in local state. @@ -874,14 +930,20 @@ func (c *DiskVolumeController) reconcileOrphanKernelState() { } // Only touch loops backing files inside miren's volumes dir. // Anything else is not ours to manage. - if !strings.HasPrefix(backing, volumesDir+string(filepath.Separator)) && - !strings.HasPrefix(backing, volumesDir+"/") { + // + // A deleted backing counts: the path it reports is where the file used + // to be, which is still enough to say the device is ours, and a device + // holding an unlinked image is exactly the kind of leftover this sweep + // exists to reclaim. + if !strings.HasPrefix(backing.Path, volumesDir+string(filepath.Separator)) && + !strings.HasPrefix(backing.Path, volumesDir+"/") { continue } c.log.Warn("orphan sweep: detaching stale loop device", "device", dev, - "backing_file", backing, + "backing_file", backing.Path, + "backing_deleted", backing.Deleted, ) if err := c.mntOps.LoopDetach(dev); err != nil { c.log.Warn("orphan sweep: LoopDetach failed", diff --git a/components/diskio/disk_volume_controller_test.go b/components/diskio/disk_volume_controller_test.go index 3caeb219e..420cd4d58 100644 --- a/components/diskio/disk_volume_controller_test.go +++ b/components/diskio/disk_volume_controller_test.go @@ -560,6 +560,114 @@ func TestDiskVolumeControllerDeleteNotInState(t *testing.T) { assert.Equal(t, storage_v1alpha.DV_DELETED, updated.ActualState) } +func TestDiskVolumeControllerDeleteUntrackedPendingImage(t *testing.T) { + for _, tc := range []struct { + name string + attached bool + moveErr bool + }{ + {name: "unattached"}, + {name: "attached", attached: true}, + {name: "move failure", moveErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + state := NewState() + ops := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + volumePath := filepath.Join(dataPath, "volumes", "vol-pending") + imagePath := filepath.Join(volumePath, "disk.img") + require.NoError(t, os.MkdirAll(volumePath, 0755)) + require.NoError(t, os.WriteFile(imagePath, []byte("recovered data"), 0600)) + ops.existingPaths[volumePath] = true + if tc.attached { + mntOps.loopBacking = make(map[string]string) + mntOps.loopBacking[imagePath] = "/dev/loop7" + } + if tc.moveErr { + ops.moveDirErr = errors.New("disk I/O failure") + } + + vc := NewDiskVolumeController(testutils.TestLogger(t), dataPath, compute.NewNodeId("test-node-1"), state, ops, mntOps) + vc.SetEAC(es.EAC) + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-pending", + NodeId: compute.NewNodeId("test-node-1").Id(), + DesiredState: storage_v1alpha.DV_ABSENT, + ActualState: storage_v1alpha.DV_PENDING, + } + createDiskVolumeEntity(ctx, t, es, vol) + + err := vc.reconcileVolume(ctx, vol) + resp, getErr := es.EAC.Get(ctx, string(vol.ID)) + require.NoError(t, getErr) + var updated storage_v1alpha.DiskVolume + updated.Decode(resp.Entity().Entity()) + if tc.attached { + require.ErrorContains(t, err, "still attached") + assert.Empty(t, ops.removedDirs) + assert.Empty(t, ops.movedDirs) + assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) + } else if tc.moveErr { + require.ErrorContains(t, err, "disk I/O failure") + assert.Empty(t, ops.removedDirs) + assert.Empty(t, ops.movedDirs) + assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) + data, readErr := os.ReadFile(imagePath) + require.NoError(t, readErr) + assert.Equal(t, "recovered data", string(data)) + } else { + require.NoError(t, err) + assert.Empty(t, ops.removedDirs) + require.Len(t, ops.movedDirs, 1) + assert.Equal(t, volumePath, ops.movedDirs[0].src) + assert.Equal(t, filepath.Join(dataPath, "deleted-volumes", "vol-pending"), ops.movedDirs[0].dst) + assert.Equal(t, storage_v1alpha.DV_DELETED, updated.ActualState) + } + }) + } +} + +func TestDiskVolumeControllerSoftDeletesUntrackedUndelete(t *testing.T) { + ctx := t.Context() + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + volumePath := filepath.Join(dataPath, "volumes", "vol-recovered") + require.NoError(t, os.MkdirAll(volumePath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(volumePath, "disk.img"), []byte("recovered data"), 0600)) + + log := testutils.TestLogger(t) + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId("test-node-1"), NewState(), NewRealDiskVolumeOps(log), newMockDiskMountOps()) + vc.SetEAC(es.EAC) + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-recovered", + Name: "recovered", + VolumeId: "vol-recovered", + NodeId: compute.NewNodeId("test-node-1").Id(), + DesiredState: storage_v1alpha.DV_ABSENT, + ActualState: storage_v1alpha.DV_PENDING, + } + createDiskVolumeEntity(ctx, t, es, vol) + require.NoError(t, vc.reconcileVolume(ctx, vol)) + + deletedPath := filepath.Join(dataPath, "deleted-volumes", "vol-recovered") + data, err := os.ReadFile(filepath.Join(deletedPath, "disk.img")) + require.NoError(t, err) + assert.Equal(t, "recovered data", string(data)) + meta, err := LoadDeletedVolumeMetadata(deletedPath) + require.NoError(t, err) + assert.Equal(t, "recovered", meta.DiskName) + assert.Equal(t, "vol-recovered", meta.VolumeID) + _, err = os.Stat(volumePath) + assert.True(t, os.IsNotExist(err)) +} + func TestDiskVolumeControllerUniversalMountAtCreation(t *testing.T) { ctx := t.Context() log := testutils.TestLogger(t) @@ -1080,3 +1188,115 @@ func TestDiskVolumeControllerAcceleratorNoMountAtCreation(t *testing.T) { require.NotNil(t, volState) assert.False(t, volState.Mounted) } + +// TestDiskVolumeControllerDoesNotAdoptADeletedBacking is the counterpart to the +// adoption test above, for the case where the loop device is holding a file +// that is no longer at that path. +// +// The kernel reports such a device's backing_file as the original path with +// " (deleted)" appended. Matching on the path alone makes it look like the +// image is already attached, so the controller adopts it, finds the old +// filesystem, decides no formatting is needed, and mounts the contents that +// were replaced — after a restore wrote a new image and renamed it into place, +// exactly the data the operator had just discarded. +// +// The live image must get its own loop device instead. +func TestDiskVolumeControllerDoesNotAdoptADeletedBacking(t *testing.T) { + ctx := t.Context() + log := testutils.TestLogger(t) + + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + nodeId := "test-node-1" + state := NewState() + volOps := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + + volPath := filepath.Join(dataPath, "volumes", "vol-ghost") + mountPath := filepath.Join(dataPath, "vol-ghost") + imagePath := filepath.Join(volPath, "disk.img") + + state.SetVolume("disk_volume/vol-ghost", &VolumeState{ + EntityId: "disk_volume/vol-ghost", + VolumeId: "vol-ghost", + DiskPath: volPath, + SizeBytes: 10 * 1024 * 1024 * 1024, + Filesystem: "ext4", + Mode: storage_v1alpha.VM_UNIVERSAL, + Mounted: false, + MountPath: mountPath, + }) + volOps.existingPaths[volPath] = true + + // A loop device still pinning the image that used to be at this path. + const ghostLoopDev = "/dev/loop9" + mntOps.loopBacking = map[string]string{imagePath: ghostLoopDev} + mntOps.deletedBacking = map[string]bool{imagePath: true} + mntOps.formattedDevs[ghostLoopDev] = "ext4" + + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId(nodeId), state, volOps, mntOps) + vc.SetEAC(es.EAC) + + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-ghost", + NodeId: compute.NewNodeId(nodeId).Id(), + SizeGb: 10, + Filesystem: "ext4", + VolumeMode: storage_v1alpha.VM_UNIVERSAL, + DesiredState: storage_v1alpha.DV_PRESENT, + ActualState: storage_v1alpha.DV_READY, + } + createDiskVolumeEntity(ctx, t, es, vol) + + require.NoError(t, vc.ReconcileWithEntities(ctx)) + + // The live image gets a loop of its own rather than the ghost. + require.Len(t, mntOps.attachedLoops, 1, + "the live image must be attached rather than the device holding the deleted one") + + require.Len(t, mntOps.mounts, 1) + assert.NotEqual(t, ghostLoopDev, mntOps.mounts[0].device, + "mounting the device that holds the deleted inode serves the data the operator replaced") + + volState := state.GetVolume("disk_volume/vol-ghost") + require.NotNil(t, volState) + assert.True(t, volState.Mounted) + assert.NotEqual(t, ghostLoopDev, volState.DevicePath) +} + +// TestDiskVolumeControllerOrphanSweepReclaimsDeletedBackings is the other half +// of the deleted-backing distinction. +// +// Adoption must ignore a loop holding an unlinked inode, but the sweep must +// still reclaim it: a device pinning a file that no longer has a name is +// precisely the leftover this sweep exists for, and it is what keeps the +// non-adoption above from leaking a device forever. +func TestDiskVolumeControllerOrphanSweepReclaimsDeletedBackings(t *testing.T) { + ctx := t.Context() + log := testutils.TestLogger(t) + + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + nodeId := "test-node-1" + state := NewState() + volOps := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + + ghostImage := filepath.Join(dataPath, "volumes", "vol-ghost", "disk.img") + const ghostLoopDev = "/dev/loop11" + + mntOps.loopBacking = map[string]string{ghostImage: ghostLoopDev} + mntOps.deletedBacking = map[string]bool{ghostImage: true} + + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId(nodeId), state, volOps, mntOps) + vc.SetEAC(es.EAC) + + require.NoError(t, vc.ReconcileWithEntities(ctx)) + + assert.Contains(t, mntOps.detachedLoops, ghostLoopDev, + "a loop holding an unlinked image under the volumes dir must still be reclaimed") +} diff --git a/components/diskio/image_snapshot.go b/components/diskio/image_snapshot.go index 3b058e1d6..da3d9d1cb 100644 --- a/components/diskio/image_snapshot.go +++ b/components/diskio/image_snapshot.go @@ -53,6 +53,14 @@ type SnapshotRequest struct { Filesystem string // SnapshotName optionally pins this as a named restore point. SnapshotName string + // InUse records that the image was attached when it was read, so the + // snapshot is a smear across the read rather than a point-in-time copy. + // Worth knowing later: it is the difference between a restore point you can + // rely on and one you are choosing to gamble on. + // + // The caller decides, because "attached" is a question about kernel state + // that the snapshotter has no handle on. + InUse bool // LeaseNonce is required when the volume has an active lease. LeaseNonce string // StagingDir is where the compressed snapshot is written before upload. @@ -60,33 +68,47 @@ type SnapshotRequest struct { StagingDir string } -// Snapshot compresses the image and uploads it, returning the cloud's update ID. -func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (string, error) { +// SnapshotResult describes the restore point a Snapshot produced. The sizes and +// checksum are what a caller reports back to an operator, so they come from the +// snapshot that was actually uploaded rather than being recomputed. +type SnapshotResult struct { + // UpdateID is the cloud's identifier for the uploaded restore point. + UpdateID string + // ImageSize is the uncompressed size of the image that was read. + ImageSize int64 + // CompressedSize is the size of the uploaded snapshot. + CompressedSize int64 + // Checksum is the SHA-256 of the uncompressed image, hex encoded. + Checksum string +} + +// Snapshot compresses the image and uploads it as a restore point. +func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (*SnapshotResult, error) { if s.updates == nil { - return "", fmt.Errorf("no cloud updates client configured") + return nil, fmt.Errorf("no cloud updates client configured") } if req.VolumeID == "" { - return "", fmt.Errorf("volume ID is required") + return nil, fmt.Errorf("volume ID is required") } info, err := os.Stat(req.ImagePath) if err != nil { - return "", fmt.Errorf("stat image: %w", err) + return nil, fmt.Errorf("stat image: %w", err) } staged, checksum, err := s.stage(req, info) if err != nil { - return "", err + return nil, err } defer os.Remove(staged.Name()) defer staged.Close() stagedInfo, err := staged.Stat() if err != nil { - return "", fmt.Errorf("stat staged snapshot: %w", err) + return nil, fmt.Errorf("stat staged snapshot: %w", err) } if _, err := staged.Seek(0, io.SeekStart); err != nil { - return "", fmt.Errorf("rewind staged snapshot: %w", err) + return nil, fmt.Errorf("rewind staged snapshot: %w", err) } // The ordering key must sort against its siblings and match the cloud's @@ -105,11 +127,12 @@ func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (s "image_size": info.Size(), "image_sha256": checksum, "compressed_size": stagedInfo.Size(), + "was_attached": req.InUse, }, LeaseNonce: req.LeaseNonce, }, staged, stagedInfo.Size()) if err != nil { - return "", fmt.Errorf("upload image snapshot: %w", err) + return nil, fmt.Errorf("upload image snapshot: %w", err) } s.log.Info("uploaded volume image snapshot", @@ -119,7 +142,12 @@ func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (s "image_size", info.Size(), "compressed_size", stagedInfo.Size(), ) - return updateID, nil + return &SnapshotResult{ + UpdateID: updateID, + ImageSize: info.Size(), + CompressedSize: stagedInfo.Size(), + Checksum: checksum, + }, nil } // stage writes a compressed snapshot to a temp file. diff --git a/components/diskio/image_snapshot_test.go b/components/diskio/image_snapshot_test.go index 4393429d0..6060e5774 100644 --- a/components/diskio/image_snapshot_test.go +++ b/components/diskio/image_snapshot_test.go @@ -29,7 +29,7 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { content := bytes.Repeat([]byte("disk contents "), 512) snapshotter, fake, imagePath := newSnapshotFixture(t, content) - updateID, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ + res, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ VolumeID: "vol-1", ImagePath: imagePath, Name: "data", @@ -37,7 +37,10 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { SnapshotName: "pre-migration", }) require.NoError(t, err) - assert.Equal(t, "volup-fake", updateID) + assert.Equal(t, "volup-fake", res.UpdateID) + assert.Equal(t, int64(len(content)), res.ImageSize) + assert.NotZero(t, res.CompressedSize) + assert.NotEmpty(t, res.Checksum) require.Len(t, fake.uploads, 1) up := fake.uploads[0] @@ -59,6 +62,29 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { assert.Equal(t, "data", meta.Name) assert.Equal(t, int64(len(content)), meta.SizeBytes) assert.Equal(t, meta.Checksum, up.Request.Metadata["image_sha256"]) + + // The fixture did not say the image was in use, so the restore point does + // not claim it was. + assert.Equal(t, false, up.Request.Metadata["was_attached"]) +} + +// A snapshot taken from a live image is a smear across the read rather than a +// point-in-time copy, and the restore point has to say so — it is the +// difference between one you can rely on and one you are gambling on. +func TestImageSnapshotRecordsThatTheImageWasInUse(t *testing.T) { + snapshotter, fake, imagePath := newSnapshotFixture(t, []byte("written while we read")) + + _, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ + VolumeID: "vol-1", + ImagePath: imagePath, + Name: "data", + Filesystem: "ext4", + InUse: true, + }) + require.NoError(t, err) + + require.Len(t, fake.uploads, 1) + assert.Equal(t, true, fake.uploads[0].Request.Metadata["was_attached"]) } // Every snapshot is a deliberate act, so two in a row both upload. There is no diff --git a/controllers/integration/mock_ops.go b/controllers/integration/mock_ops.go index 4a6d1e84d..3f1cd79d0 100644 --- a/controllers/integration/mock_ops.go +++ b/controllers/integration/mock_ops.go @@ -94,10 +94,10 @@ func (m *mockDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { return "", nil } -func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]string, error) { - result := make(map[string]string, len(m.loopDevices)) +func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]diskio.LoopBacking, error) { + result := make(map[string]diskio.LoopBacking, len(m.loopDevices)) for img, dev := range m.loopDevices { - result[dev] = img + result[dev] = diskio.LoopBacking{Path: img} } return result, nil } diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index e691066f6..3a3f407d9 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -125,6 +125,7 @@ "command/debug-ctr-nuke", "command/debug-deploy-events", "command/debug-disk", + "command/debug-disk-backup", "command/debug-disk-create", "command/debug-disk-delete", "command/debug-disk-lease", @@ -133,8 +134,11 @@ "command/debug-disk-lease-release", "command/debug-disk-lease-status", "command/debug-disk-list", + "command/debug-disk-list-deleted", "command/debug-disk-mounts", + "command/debug-disk-restore", "command/debug-disk-status", + "command/debug-disk-undelete", "command/debug-entity", "command/debug-entity-create", "command/debug-entity-delete", diff --git a/docs/docs/addons.md b/docs/docs/addons.md index fe45a7fbc..4ecf638c6 100644 --- a/docs/docs/addons.md +++ b/docs/docs/addons.md @@ -390,7 +390,7 @@ Miren provisions PostgreSQL, injects `DATABASE_URL`, and starts your app once th :::info[Early version] Addon backup and restore uses the general-purpose disk backup system. We plan to add addon-aware backup commands in a future release that will simplify this workflow. For now, the steps below work reliably for PostgreSQL addon data. -The `disk backup` and `disk restore` commands must be run directly on the server (via SSH or `miren ssh`), not from your local machine. Remote backup support is planned. +The `disk backup` and `disk restore` commands run from anywhere you can reach the cluster, including your own machine. The server does the work; the snapshot is streamed to or from you. ::: Each PostgreSQL addon stores its data on a Miren disk. You can back up and restore this disk using the `miren disk backup` and `miren disk restore` commands. @@ -409,19 +409,23 @@ Addon disks are named with a `pg-` prefix. For dedicated (`small`) addons, the n ### Creating a Backup -Back up the disk to a compressed snapshot file. This must be run on the server: +Back up the disk to a compressed snapshot file: - + ```miren miren disk backup -n ``` -This creates a timestamped `.miren.zst` file in the current directory. If the disk is currently in use, the backup will be crash-consistent (safe for PostgreSQL, which uses write-ahead logging). +This creates a timestamped `.miren.zst` file in the current directory. + +:::warning[Detach the disk first] +Nothing freezes the filesystem while the snapshot is read, so backing up a disk that is currently attached reads it as it is being written. The head and tail of the image come from different moments, which is weaker than the power-loss state PostgreSQL's write-ahead log recovery is built for. The command warns you when it detects this. Stop the services using the disk for a backup you can rely on. +::: Example: - + ```miren miren disk backup -n pg-pg-myapp-sCZDabc123-data # Output: pg-pg-myapp-sCZDabc123-data-20260324-120000.miren.zst @@ -430,7 +434,7 @@ miren disk backup -n pg-pg-myapp-sCZDabc123-data You can specify a custom output path with `-o`: - + ```miren miren disk backup -n pg-pg-myapp-sCZDabc123-data -o /backups/myapp-db.miren.zst ``` @@ -438,9 +442,9 @@ miren disk backup -n pg-pg-myapp-sCZDabc123-data -o /backups/myapp-db.miren.zst ### Restoring from a Backup -To restore from a backup, provide the snapshot file. This must also be run on the server: +To restore from a backup, provide the snapshot file: - + ```miren miren disk restore -s ``` @@ -448,28 +452,47 @@ miren disk restore -s The restore procedure recreates the disk with the original name. If the disk already exists, use `--force` to overwrite: - + ```miren miren disk restore -s myapp-db.miren.zst --force ``` -To restore to a different disk name: +:::warning[Restoring over a disk in use is refused] +Restoring on top of a disk your app is using is not supported today, and the command stops rather than pretending. A disk in use is held open by the kernel, so writing a new image into it would leave the running database on the old data while reporting success. +::: + +There is currently no way to release a disk while keeping the app configured for it, so there is no sequence that makes an in-place restore work. Disk-backed services run at fixed concurrency, `num_instances = 0` is rejected, and scaling a pool to zero by hand is reconciled straight back to one. Even during that brief window the disk stays mounted. - +The working recovery is to restore into a **new** disk and move the app across. Restore into a new disk name: + + ```miren -miren disk restore -s myapp-db.miren.zst -n new-disk-name +miren disk restore -s myapp-db.miren.zst -n myapp-db-restored ``` -After restoring, restart your app to pick up the restored data: +Then point the service at it in `.miren/app.toml` and deploy: + +```toml +[[services.db.disks]] +name = "myapp-db-restored" +mount_path = "/var/lib/postgresql/data" +size_gb = 20 +``` ```miren -miren app restart myapp +miren deploy ``` +The one case that does restore in place is the one where nothing is holding the disk: a rebuilt host whose disk image is gone. There the image is absent rather than mounted, so `miren disk restore` writes it and the app picks it up on the next start. + +:::note[`--from-cloud` restores into the disk the point came from] +A cloud restore point is looked up through its own disk's cloud volume, so `--from-cloud` needs that disk to already exist and be registered. It cannot restore into a new disk name, which means the move-the-app-across path above needs a local snapshot file. Keep one for a disk you would want to recover onto a running cluster. +::: + ### Backup Recommendations - **Schedule regular backups** for production databases, especially before destructive operations like `addon destroy` diff --git a/docs/docs/command/debug-disk-backup.md b/docs/docs/command/debug-disk-backup.md new file mode 100644 index 000000000..e87c2e171 --- /dev/null +++ b/docs/docs/command/debug-disk-backup.md @@ -0,0 +1,35 @@ +--- +title: "miren debug disk backup" +sidebar_label: "debug disk backup" +description: "Back up a disk by reading its image directly (break-glass)" +--- + +# miren debug disk backup + +Back up a disk by reading its image directly (break-glass) + +## Usage + +```bash +miren debug disk backup [flags] +``` + +## Flags + +- `--cloud` — Also upload the snapshot to miren.cloud as a restore point +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--name, -n` — Disk name to backup +- `--output, -o` — Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst) +- `--pin` — Name the uploaded restore point, pinning it against cleanup + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](./debug-disk.md) diff --git a/docs/docs/command/debug-disk-list-deleted.md b/docs/docs/command/debug-disk-list-deleted.md new file mode 100644 index 000000000..d360ac2f7 --- /dev/null +++ b/docs/docs/command/debug-disk-list-deleted.md @@ -0,0 +1,33 @@ +--- +title: "miren debug disk list-deleted" +sidebar_label: "debug disk list-deleted" +description: "Read the soft-delete holding area directly (break-glass)" +--- + +# miren debug disk list-deleted + +Read the soft-delete holding area directly (break-glass) + +## Usage + +```bash +miren debug disk list-deleted [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--format` — Output format (text, json) (default: `text`) +- `--json` — Shorthand for --format json + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](./debug-disk.md) diff --git a/docs/docs/command/debug-disk-restore.md b/docs/docs/command/debug-disk-restore.md new file mode 100644 index 000000000..d4522b87b --- /dev/null +++ b/docs/docs/command/debug-disk-restore.md @@ -0,0 +1,34 @@ +--- +title: "miren debug disk restore" +sidebar_label: "debug disk restore" +description: "Restore a disk by writing its image directly (break-glass)" +--- + +# miren debug disk restore + +Restore a disk by writing its image directly (break-glass) + +## Usage + +```bash +miren debug disk restore [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--force, -f` — Overwrite existing disk image without confirmation +- `--name, -n` — Disk name to restore to (default: original name from snapshot) +- `--snapshot, -s` — Path to snapshot file + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](./debug-disk.md) diff --git a/docs/docs/command/debug-disk-undelete.md b/docs/docs/command/debug-disk-undelete.md new file mode 100644 index 000000000..ebd5d5271 --- /dev/null +++ b/docs/docs/command/debug-disk-undelete.md @@ -0,0 +1,33 @@ +--- +title: "miren debug disk undelete" +sidebar_label: "debug disk undelete" +description: "Recover a deleted disk by moving its data directly (break-glass)" +--- + +# miren debug disk undelete + +Recover a deleted disk by moving its data directly (break-glass) + +## Usage + +```bash +miren debug disk undelete [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--name, -n` — Disk name to undelete +- `--volume-id, -V` — Volume ID to restore (when multiple deleted disks share a name) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](./debug-disk.md) diff --git a/docs/docs/command/debug-disk.md b/docs/docs/command/debug-disk.md index f1eb8884d..90a2322be 100644 --- a/docs/docs/command/debug-disk.md +++ b/docs/docs/command/debug-disk.md @@ -55,6 +55,7 @@ miren debug disk [flags] ## Subcommands +- [`miren debug disk backup`](./debug-disk-backup.md) — Back up a disk by reading its image directly (break-glass) - [`miren debug disk create`](./debug-disk-create.md) — Create a disk entity for testing - [`miren debug disk delete`](./debug-disk-delete.md) — Delete a disk entity - [`miren debug disk lease`](./debug-disk-lease.md) — Create a disk lease for testing @@ -63,8 +64,11 @@ miren debug disk [flags] - [`miren debug disk lease-release`](./debug-disk-lease-release.md) — Release a disk lease - [`miren debug disk lease-status`](./debug-disk-lease-status.md) — Show detailed status of a disk lease - [`miren debug disk list`](./debug-disk-list.md) — List all disk entities +- [`miren debug disk list-deleted`](./debug-disk-list-deleted.md) — Read the soft-delete holding area directly (break-glass) - [`miren debug disk mounts`](./debug-disk-mounts.md) — List all mounted disks from /proc/mounts +- [`miren debug disk restore`](./debug-disk-restore.md) — Restore a disk by writing its image directly (break-glass) - [`miren debug disk status`](./debug-disk-status.md) — Show status of a disk entity +- [`miren debug disk undelete`](./debug-disk-undelete.md) — Recover a deleted disk by moving its data directly (break-glass) ## See also diff --git a/docs/docs/command/disk-backup.md b/docs/docs/command/disk-backup.md index 00704f2a9..a3bdfac47 100644 --- a/docs/docs/command/disk-backup.md +++ b/docs/docs/command/disk-backup.md @@ -16,10 +16,9 @@ miren disk backup [flags] ## Flags -- `--cloud` — Also upload the snapshot to miren.cloud as a restore point +- `--cloud` — Upload the snapshot to miren.cloud as a restore point instead of writing a local file - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--name, -n` — Disk name to backup - `--output, -o` — Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst) - `--pin` — Name the uploaded restore point, pinning it against cleanup diff --git a/docs/docs/command/disk-list-deleted.md b/docs/docs/command/disk-list-deleted.md index 99ef3d28a..abec4fe89 100644 --- a/docs/docs/command/disk-list-deleted.md +++ b/docs/docs/command/disk-list-deleted.md @@ -18,7 +18,6 @@ miren disk list-deleted [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--format` — Output format (text, json) (default: `text`) - `--json` — Shorthand for --format json diff --git a/docs/docs/command/disk-restore.md b/docs/docs/command/disk-restore.md index c05226632..2be5590aa 100644 --- a/docs/docs/command/disk-restore.md +++ b/docs/docs/command/disk-restore.md @@ -18,10 +18,11 @@ miren disk restore [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) -- `--force, -f` — Overwrite existing disk image without confirmation -- `--name, -n` — Disk name to restore to (default: original name from snapshot) -- `--snapshot, -s` — Path to snapshot file +- `--force, -f` — Overwrite an existing disk image +- `--from-cloud` — Restore from a miren.cloud restore point +- `--name, -n` — Disk name to restore to (default: the name recorded in the snapshot) +- `--restore-point` — Restore point to use (implies --from-cloud; default: the newest) +- `--snapshot, -s` — Path to a snapshot file to restore from ## Global Options diff --git a/docs/docs/command/disk-undelete.md b/docs/docs/command/disk-undelete.md index b29b123ad..4b13fef89 100644 --- a/docs/docs/command/disk-undelete.md +++ b/docs/docs/command/disk-undelete.md @@ -18,9 +18,8 @@ miren disk undelete [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--name, -n` — Disk name to undelete -- `--volume-id, -V` — Volume ID to restore (when multiple deleted disks share a name) +- `--volume-id, -V` — Volume ID to recover (when several deleted disks share a name) ## Global Options diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 59e48f72b..896ed1162 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -321,6 +321,7 @@ These commands are intended for advanced debugging and troubleshooting. They may | [`miren debug ctr nuke`](./command/debug-ctr-nuke.md) | Nuke a containerd namespace | | [`miren debug deploy-events`](./command/debug-deploy-events.md) | Render a 'miren deploy --format jsonl' stream as readable output | | [`miren debug disk`](./command/debug-disk.md) | Disk entity debug commands | +| [`miren debug disk backup`](./command/debug-disk-backup.md) | Back up a disk by reading its image directly (break-glass) | | [`miren debug disk create`](./command/debug-disk-create.md) | Create a disk entity for testing | | [`miren debug disk delete`](./command/debug-disk-delete.md) | Delete a disk entity | | [`miren debug disk lease`](./command/debug-disk-lease.md) | Create a disk lease for testing | @@ -329,8 +330,11 @@ These commands are intended for advanced debugging and troubleshooting. They may | [`miren debug disk lease-release`](./command/debug-disk-lease-release.md) | Release a disk lease | | [`miren debug disk lease-status`](./command/debug-disk-lease-status.md) | Show detailed status of a disk lease | | [`miren debug disk list`](./command/debug-disk-list.md) | List all disk entities | +| [`miren debug disk list-deleted`](./command/debug-disk-list-deleted.md) | Read the soft-delete holding area directly (break-glass) | | [`miren debug disk mounts`](./command/debug-disk-mounts.md) | List all mounted disks from /proc/mounts | +| [`miren debug disk restore`](./command/debug-disk-restore.md) | Restore a disk by writing its image directly (break-glass) | | [`miren debug disk status`](./command/debug-disk-status.md) | Show status of a disk entity | +| [`miren debug disk undelete`](./command/debug-disk-undelete.md) | Recover a deleted disk by moving its data directly (break-glass) | | [`miren debug entity`](./command/debug-entity.md) | Entity store debug commands | | [`miren debug entity create`](./command/debug-entity-create.md) | Create a new entity | | [`miren debug entity delete`](./command/debug-entity-delete.md) | Delete an entity | diff --git a/docs/docs/disks.md b/docs/docs/disks.md index 92000bf97..a1b34667e 100644 --- a/docs/docs/disks.md +++ b/docs/docs/disks.md @@ -98,7 +98,7 @@ If any of your environment variables reference `/miren/data/local`, Miren will a ## Miren Disks :::note[Backups] -Miren Disks live on your server. Back up important data with `miren disk backup` and restore it with `miren disk restore`. Cloud backup is on the [roadmap](#roadmap-cloud-backup--sync). +Miren Disks live on your server. Back up important data with `miren disk backup` and restore it with `miren disk restore`, from your own machine or from the server. Add `--cloud` to store the backup in Miren Cloud instead of a local file. ::: Miren Disks provide managed persistent storage for your applications. Disks are provisioned with a specific size and filesystem, support exclusive leasing for data consistency, and persist across app restarts and redeployments. @@ -309,7 +309,7 @@ Your server must have the mkfs tools to format the disk types. We're building toward cloud-connected storage for Miren Disks. Here's what's planned: -- **Remote backup & restore** (next up): Trigger backups of your disks to Miren Cloud and restore them on any cluster. This extends the existing local backup/restore functionality to work remotely. +- **Remote backup & restore** (shipped): `miren disk backup` and `miren disk restore` run from any machine that can reach the cluster. Add `--cloud` to store a backup in Miren Cloud as a restore point, and `--from-cloud` to restore from one. - **Automatic cloud sync**: Background replication of disk data to Miren Cloud, enabling seamless portability across clusters. We'll update this page and the [changelog](https://miren.md/changelog) as these capabilities land. diff --git a/hack/e2e_disk/main.go b/hack/e2e_disk/main.go index 6659f9d0c..676468dd8 100644 --- a/hack/e2e_disk/main.go +++ b/hack/e2e_disk/main.go @@ -67,13 +67,14 @@ func main() { check(os.WriteFile(filepath.Join(diskPath, "disk.img"), imageData, 0644), "write image") snapshotter := diskio.NewImageSnapshotter(log, updates) - imageUpdateID, err := snapshotter.Snapshot(ctx, diskio.SnapshotRequest{ + imageSnap, err := snapshotter.Snapshot(ctx, diskio.SnapshotRequest{ VolumeID: volumeID, ImagePath: filepath.Join(diskPath, "disk.img"), Name: "e2e-disk", Filesystem: "ext4", }) check(err, "snapshot image") + imageUpdateID := imageSnap.UpdateID fmt.Printf("✓ uploaded loop_image snapshot %s\n", imageUpdateID) // --- list both kinds back --- diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go new file mode 100644 index 000000000..f688b15ec --- /dev/null +++ b/pkg/diskresolve/resolver.go @@ -0,0 +1,390 @@ +package diskresolve + +import ( + "context" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "strings" + + compute "miren.dev/runtime/api/compute/compute_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/snapshot" +) + +// Resolver implements snapshot.DiskResolver using the entity +// access RPC client. +type Resolver struct { + eac *entityserver_v1alpha.EntityAccessClient + ec *entityserver.Client +} + +func New(eac *entityserver_v1alpha.EntityAccessClient, ec *entityserver.Client) *Resolver { + return &Resolver{eac: eac, ec: ec} +} + +func (r *Resolver) FindDisk(ctx context.Context, name string) (*snapshot.DiskState, error) { + ref := entity.Ref(entity.EntityKind, storage_v1alpha.KindDisk) + results, err := r.eac.List(ctx, ref) + if err != nil { + return nil, fmt.Errorf("listing disks: %w", err) + } + + var matches []snapshot.DiskState + for _, e := range results.Values() { + var disk storage_v1alpha.Disk + disk.Decode(e.Entity()) + if disk.Name == name { + matches = append(matches, snapshot.DiskState{ + ID: string(disk.ID), + Name: disk.Name, + Status: string(disk.Status), + Filesystem: strings.TrimPrefix(string(disk.Filesystem), "filesystem."), + }) + } + } + + switch len(matches) { + case 0: + // Typed, so callers can tell "no such disk" from "the lookup failed". + // PrepareRestore creates a disk on the first and must not on the second. + return nil, snapshot.DiskNotFoundError{Name: name} + case 1: + return &matches[0], nil + default: + return nil, fmt.Errorf("multiple disks found with name %q (%d matches)", name, len(matches)) + } +} + +func (r *Resolver) FindVolume(ctx context.Context, diskID string) (*snapshot.VolumeState, error) { + resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskVolumeDiskIdId, entity.Id(diskID))) + if err != nil { + return nil, fmt.Errorf("listing disk volumes: %w", err) + } + + values := resp.Values() + if len(values) == 0 { + return nil, fmt.Errorf("no disk volume found for disk %s", diskID) + } + if len(values) > 1 { + return nil, fmt.Errorf("multiple disk volumes found for disk %s (%d matches)", diskID, len(values)) + } + + var vol storage_v1alpha.DiskVolume + vol.Decode(values[0].Entity()) + return &snapshot.VolumeState{ + VolumeID: vol.VolumeId, + // Where the disk controller records the volume's id in miren.cloud + // once it has registered it there. + CloudVolumeID: vol.CloudVolumeId, + ImagePath: vol.ImagePath, + }, nil +} + +// CreateDiskAndVolume creates a new disk entity in RESTORING state so the disk +// controller ignores it while restore writes the image. The returned +// RestoreTarget includes a Finalize callback that creates the disk_volume +// entity and hands the disk to the controller as PROVISIONING, which promotes +// it once the volume is ready. +func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { + sizeGb, err := diskSizeGb(sizeBytes) + if err != nil { + return nil, err + } + + // Normalize filesystem string — strip enum prefix if present + filesystem = strings.TrimPrefix(strings.ToLower(filesystem), "filesystem.") + fs := ParseFilesystem(filesystem) + + diskId := idgen.GenNS("disk") + volId := idgen.GenNS("disk-vol") + imagePath := filepath.Join(dataPath, "disk-data", "volumes", volId, "disk.img") + + disk := &storage_v1alpha.Disk{ + Name: name, + SizeGb: sizeGb, + Filesystem: fs, + Status: storage_v1alpha.RESTORING, + } + + diskEntityId, err := r.ec.Create(ctx, diskId, disk) + if err != nil { + return nil, fmt.Errorf("creating disk entity: %w", err) + } + + nodeId, err := r.FindNodeId(ctx) + if err != nil { + // The disk is already committed but the caller is about to get an + // error and no RestoreTarget, so it has no Cleanup to call. Unwind it + // here or it sits in RESTORING forever, and the next restore of the + // same name finds that disk, skips creating one, and fails looking for + // the volume it never got. + if uerr := r.abandonDisk(ctx, diskEntityId); uerr != nil { + return nil, fmt.Errorf( + "finding node: %w (the half-created disk could not be unwound either, so %s is stuck in restoring: %v)", + err, name, uerr) + } + return nil, fmt.Errorf("finding node: %w", err) + } + + // Set once Finalize has committed the disk_volume, which is the moment the + // image stops being ours to delete. Finalize and Cleanup are called in turn + // by the restore handler on one goroutine, so a plain bool is enough. + var volumeCommitted bool + + return &snapshot.RestoreTarget{ + Name: name, + ImagePath: imagePath, + Created: true, + Cleanup: func(cctx context.Context) error { + // The restore's own context is already cancelled when the + // operator interrupts the restore, and the rollback is exactly + // what has to run then. Detach so cleanup is not defeated by + // the failure it is cleaning up after. + cctx = context.WithoutCancel(cctx) + + // Transition the disk to DELETING rather than deleting the + // entity outright. A direct Delete bypasses the disk + // controller's DELETING-driven handleDeletion path, which is + // the only writer of disk_volume.desired_state=DV_ABSENT; a + // disk_volume whose disk is gone has no reaper and is left + // reconciled by the coordinator as a live mount / volume + // directory / phantom cloud volume. Marking the disk DELETING + // keeps it alive to drive that existing contract, which tears + // the disk_volume down via the coordinator's + // DiskVolumeController. Idempotent: patching an already-DELETING + // disk is a no-op. + // + _, err := r.eac.Patch(cctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), + }, 0) + if err != nil { + return fmt.Errorf("transitioning disk to deleting during cleanup: %w", err) + } + + // Once the disk_volume exists, the image belongs to the teardown + // above and not to us. That teardown is asynchronous: the patch + // only marks the disk, and the controller unmounts, detaches and + // soft-deletes the volume directory some time later. Removing the + // image here would race it, and losing that race unlinks a file + // whose loop device is still attached, which leaves the kernel + // holding an inode nobody can reach instead of releasing it. + // + // Waiting for the teardown instead would mean polling the entity + // store from an error path that is already handling a failure. + // Letting the controller finish the job it already does is both + // simpler and the thing that cannot race. + if volumeCommitted { + return nil + } + + // No volume was ever created, which is the common case: most + // cleanups run before Finalize. Nothing has opened this image, so + // there is no teardown to wait for and no loop device to strand. + // Tolerate "not present" — the restore may never have renamed the + // image into place. + if imageErr := os.Remove(imagePath); imageErr != nil && !os.IsNotExist(imageErr) { + // Reported only once the authoritative step is done, so a + // failure to reclaim the image never costs us the disk + // rollback — a leftover image is disk space, a stuck + // RESTORING disk blocks every same-name retry. + return fmt.Errorf("removing restored image during cleanup: %w", imageErr) + } + return nil + }, + Finalize: func(fctx context.Context) error { + vol := &storage_v1alpha.DiskVolume{ + Name: name, + DiskId: diskEntityId, + VolumeId: volId, + SizeGb: sizeGb, + Filesystem: filesystem, + VolumeMode: DetectVolumeMode(), + + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY, and let the DiskVolumeController + // drive it the rest of the way. The image is already written + // and formatted, so it mounts what was restored rather than + // reimaging. + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, + } + + // The volume goes in first, and the disk moves to PROVISIONING + // rather than PROVISIONED. Both halves of that matter, and the + // reason is a window rather than a preference. + // + // DiskController ignores a RESTORING disk, so nothing watches + // this disk until the patch below. Announcing PROVISIONED first + // would end that truce while there was still no disk_volume to + // find, and handleProvisioned answers a provisioned disk with no + // volume by provisioning a blank one. The restore's own volume + // then lands beside it: two DV_READY volumes on one disk, both + // mounted, and the next backup fails on the ambiguity. + // + // This way the volume is already there when the disk becomes + // visible, and the controller promotes it to PROVISIONED once the + // volume reports ready. It is the same handshake a recovered disk + // uses, so there is one path through this transition instead of + // two. + _, err := r.eac.Create(fctx, entity.New( + entity.DBId, entity.Id("disk_volume/"+volId), + vol.Encode, + ).Attrs()) + if err != nil { + return fmt.Errorf("creating disk_volume entity: %w", err) + } + // From here the volume owns the image, so Cleanup must leave it to + // the controller's teardown rather than unlinking it itself. + volumeCommitted = true + + // A failure here leaves a RESTORING disk with a committed volume, + // which Cleanup handles: it marks the disk DELETING, and the + // controller tears the volume down from there. + _, err = r.eac.Patch(fctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volId), + }, 0) + if err != nil { + return fmt.Errorf("updating disk to provisioning: %w", err) + } + + return nil + }, + }, nil +} + +// FindNodeId finds the coordinator node. Stateful sandboxes (those with +// disk volumes) run on the coordinator, so disk_volume entities must +// reference it. +func (r *Resolver) FindNodeId(ctx context.Context) (entity.Id, error) { + resp, err := r.eac.List(ctx, entity.Ref(entity.EntityKind, compute.KindNode)) + if err != nil { + return "", fmt.Errorf("listing nodes: %w", err) + } + + values := resp.Values() + if len(values) == 0 { + return "", fmt.Errorf("no nodes found") + } + + // If there's only one node, use it. + if len(values) == 1 { + return entity.Id(values[0].Entity().Id()), nil + } + + // Multiple nodes — find the coordinator (role=coordinator constraint). + for _, v := range values { + var node compute.Node + node.Decode(v.Entity()) + if role, _ := node.Constraints.Get("role"); role == "coordinator" { + return node.ID, nil + } + } + + return "", fmt.Errorf("multiple nodes found but none has role=coordinator") +} + +func (r *Resolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.LeaseState, error) { + resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskLeaseDiskIdId, entity.Id(diskID))) + if err != nil { + return nil, fmt.Errorf("listing disk leases: %w", err) + } + + var leases []snapshot.LeaseState + for _, e := range resp.Values() { + var lease storage_v1alpha.DiskLease + lease.Decode(e.Entity()) + leases = append(leases, snapshot.LeaseState{ + ID: string(lease.ID), + Status: string(lease.Status), + }) + } + + return leases, nil +} + +// abandonDisk marks a disk DELETING when creation gave up partway through. +// +// It is the same unwind Cleanup performs, for the window before there is a +// RestoreTarget to hang a Cleanup on. +func (r *Resolver) abandonDisk(ctx context.Context, diskEntityId entity.Id) error { + // The caller's context may already be cancelled, and unwinding is exactly + // what still has to happen when it is. + ctx = context.WithoutCancel(ctx) + + _, err := r.eac.Patch(ctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), + }, 0) + return err +} + +// gib is the unit disks are sized in. +const gib = 1 << 30 + +// diskSizeGb converts an image size into the capacity to record on the disk. +// +// It rounds up, because a disk claiming less capacity than the image it is +// about to be given is wrong on its face: a 1.5 GiB image would land on a disk +// reporting 1 GiB. +// +// The size comes from a snapshot header, which is a file the caller handed us, +// so it is checked rather than trusted. The rounding is done with a remainder +// rather than by adding gib-1 so that a size near the top of the range cannot +// overflow into a negative capacity, and the upper bound is where the +// controller's own conversion back to bytes would overflow. +func diskSizeGb(sizeBytes int64) (int64, error) { + switch { + case sizeBytes < 0: + return 0, fmt.Errorf("snapshot reports a negative image size (%d bytes)", sizeBytes) + case sizeBytes > math.MaxInt64-gib+1: + // The ceiling is the largest size whose rounded-up capacity still + // converts back to bytes without overflowing, which is the whole GiB + // just below the top of the range. + return 0, fmt.Errorf("snapshot reports an image size too large to be real (%d bytes)", sizeBytes) + } + + sizeGb := sizeBytes / gib + if sizeBytes%gib != 0 { + sizeGb++ + } + // Even an empty image gets a disk, and a disk has to be at least 1 GB. + if sizeGb == 0 { + sizeGb = 1 + } + return sizeGb, nil +} + +// ParseFilesystem maps a filesystem name onto the disk enum, tolerating the +// "filesystem." prefix the enum renders with. Anything unrecognized becomes +// ext4, which is the default a disk gets when it does not ask for one. +func ParseFilesystem(fs string) storage_v1alpha.DiskFilesystem { + switch strings.TrimPrefix(strings.ToLower(fs), "filesystem.") { + case "xfs": + return storage_v1alpha.XFS + case "btrfs": + return storage_v1alpha.BTRFS + default: + return storage_v1alpha.EXT4 + } +} + +func DetectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { + if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { + return storage_v1alpha.VM_ACCELERATOR + } + if _, err := exec.LookPath("lbdctl"); err == nil { + return storage_v1alpha.VM_ACCELERATOR + } + return storage_v1alpha.VM_UNIVERSAL +} diff --git a/cli/commands/disk_resolver_test.go b/pkg/diskresolve/resolver_test.go similarity index 58% rename from cli/commands/disk_resolver_test.go rename to pkg/diskresolve/resolver_test.go index 88639411c..4396a00b6 100644 --- a/cli/commands/disk_resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -1,8 +1,9 @@ -package commands +package diskresolve import ( "context" "fmt" + "math" "os" "path/filepath" "sync" @@ -73,17 +74,17 @@ func (c cancelAwareRPC) Call(ctx context.Context, method string, args, result an } // setupResolver builds an in-memory entity server, seeds a coordinator node so -// entityDiskResolver.findNodeId succeeds, and returns a resolver wired through +// Resolver.FindNodeId succeeds, and returns a resolver wired through // fault (or a plain eac when fault is nil). Reads in tests should use es.EAC, // which is not fault-injected. -func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, *entityDiskResolver) { +func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, *Resolver) { t.Helper() ctx := t.Context() es, cleanup := testutils.NewInMemEntityServer(t) t.Cleanup(cleanup) - // Seed a single coordinator node so entityDiskResolver.findNodeId returns one. + // Seed a single coordinator node so Resolver.FindNodeId returns one. _, err := es.Client.Create(ctx, "coordinator", &compute.Node{ApiAddress: ":8444"}) require.NoError(t, err) @@ -95,7 +96,7 @@ func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, } eac := entityserver_v1alpha.NewEntityAccessClient(cli) ec := entityserver.NewClient(testutils.TestLogger(t), eac) - return es, newEntityDiskResolver(eac, ec) + return es, New(eac, ec) } func listTestDisks(t *testing.T, ctx context.Context, eac *entityserver_v1alpha.EntityAccessClient) []storage_v1alpha.Disk { @@ -146,14 +147,14 @@ func getTestDisk(t *testing.T, ctx context.Context, eac *entityserver_v1alpha.En return d } -// TestCreateDiskAndVolume_FinalizeSuccess is the happy path: Finalize patches -// the disk to PROVISIONED and then creates the disk_volume, leaving the disk +// TestCreateDiskAndVolume_FinalizeSuccess is the happy path: Finalize creates +// the disk_volume and then moves the disk to PROVISIONING, leaving the disk // carrying its VolumeId and exactly one disk_volume referencing it. func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { ctx := t.Context() es, resolver := setupResolver(t, nil) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) require.NotNil(t, target) assert.True(t, target.Created) @@ -168,7 +169,10 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { require.NoError(t, target.Finalize(ctx)) disk := getTestDisk(t, ctx, es.EAC, diskID) - assert.Equal(t, storage_v1alpha.PROVISIONED, disk.Status) + // PROVISIONING, not PROVISIONED. DiskController promotes it once the volume + // reports ready, so the restore never hands over a disk whose volume the + // controller would go and invent for itself. + assert.Equal(t, storage_v1alpha.PROVISIONING, disk.Status) assert.NotEmpty(t, disk.VolumeId) vols := volumesForDisk(t, ctx, es.EAC, diskID) @@ -177,7 +181,7 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { assert.Equal(t, diskID, vol.DiskId) assert.Equal(t, disk.VolumeId, vol.VolumeId) assert.Equal(t, storage_v1alpha.DV_PRESENT, vol.DesiredState) - assert.Equal(t, storage_v1alpha.DV_READY, vol.ActualState) + assert.Equal(t, storage_v1alpha.DV_PENDING, vol.ActualState) assert.Equal(t, target.ImagePath, vol.ImagePath) // disk_volume.NodeId must match the coordinator node the resolver found. @@ -187,16 +191,16 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { assert.Equal(t, nodes.Values()[0].Entity().Id(), vol.NodeId) } -// TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan is the core bug -// repro, inverted: with the reordered Finalize (Patch before Create), failing -// the disk_volume Create leaves NO disk_volume committed. Under the old -// Create-first order this is exactly the window that orphaned a disk_volume. +// TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan covers the first of +// Finalize's two writes failing. The Create is what commits a volume, so its +// failure leaves nothing behind and the disk is still RESTORING, which the disk +// controller ignores. func TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan(t *testing.T) { ctx := t.Context() fault := newFaultRPC(nil, "create", 1, fmt.Errorf("simulated disk_volume create failure")) es, resolver := setupResolver(t, fault) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) @@ -207,17 +211,15 @@ func TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "creating disk_volume entity") - // No disk_volume entity was committed: the Create that would have committed - // it ran last and failed. This is the invariant the reorder restores. + // Nothing was committed: the Create is the first write now, so its failure + // leaves no volume at all. assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC), "no disk_volume must be left behind") assert.Empty(t, volumesForDisk(t, ctx, es.EAC, diskID)) - // The disk survived and reached PROVISIONED (the Patch ran first), so it can - // drive self-healing / DELETING-based cleanup instead of being deleted out - // from under a volume the coordinator may already be reconciling. + // The disk is still RESTORING, which DiskController ignores, so a restore + // that fell over here has not handed anything half-built to the controller. disk := getTestDisk(t, ctx, es.EAC, diskID) - assert.Equal(t, storage_v1alpha.PROVISIONED, disk.Status) - assert.NotEmpty(t, disk.VolumeId) + assert.Equal(t, storage_v1alpha.RESTORING, disk.Status) // Cleanup keeps the disk alive by transitioning it to DELETING, not by // deleting it outright; the disk controller's DELETING path then drives @@ -238,7 +240,7 @@ func TestCreateDiskAndVolume_FinalizePatchFailsLeavesNoOrphan(t *testing.T) { fault := newFaultRPC(nil, "patch", 1, fmt.Errorf("simulated disk patch failure")) es, resolver := setupResolver(t, fault) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) @@ -247,23 +249,57 @@ func TestCreateDiskAndVolume_FinalizePatchFailsLeavesNoOrphan(t *testing.T) { err = target.Finalize(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "updating disk to provisioned") + assert.Contains(t, err.Error(), "updating disk to provisioning") - // Patch is now the first write, so a Patch failure cannot have left a - // disk_volume behind. - assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC)) - assert.Empty(t, volumesForDisk(t, ctx, es.EAC, diskID)) + // The volume is created first now, so a Patch failure does leave one + // behind. That is not an orphan: an entity is only orphaned when nothing + // owns its teardown, and this one is still attached to a disk. + vols := volumesForDisk(t, ctx, es.EAC, diskID) + require.Len(t, vols, 1) - // Disk still exists, still RESTORING (Patch failed, Create never ran). + // The disk never advertised itself, so nothing else has acted on it. disk := getTestDisk(t, ctx, es.EAC, diskID) assert.Equal(t, storage_v1alpha.RESTORING, disk.Status) // Cleanup's Patch is the 2nd patch call; only the 1st (Finalize's) fails, - // so cleanup still transitions the disk to DELETING. + // so cleanup still transitions the disk to DELETING. That is what hands + // the volume to the controller's teardown path, which is the only writer + // of desired_state=DV_ABSENT. require.NoError(t, target.Cleanup(ctx)) disk = getTestDisk(t, ctx, es.EAC, diskID) assert.Equal(t, storage_v1alpha.DELETING, disk.Status) - assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC)) + assert.Len(t, volumesForDisk(t, ctx, es.EAC, diskID), 1, + "the volume stays attached to the deleting disk, which is what reaps it") +} + +// The restore hands the disk over already carrying its volume, and as +// PROVISIONING rather than PROVISIONED. +// +// Both halves close the same window. DiskController answers a PROVISIONED disk +// with no disk_volume by provisioning a blank one, so announcing the disk +// before its volume exists produces a second volume beside the restored one: +// two ready volumes on one disk, both mounted, and every later lookup by disk +// ambiguous. +func TestCreateDiskAndVolume_FinalizeHandsOverAVolumeItAlreadyOwns(t *testing.T) { + ctx := t.Context() + es, resolver := setupResolver(t, nil) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) + require.NoError(t, err) + require.NoError(t, target.Finalize(ctx)) + + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + + // PROVISIONING, not PROVISIONED: the controller promotes it once the + // volume reports ready, so there is one path through this transition. + assert.Equal(t, storage_v1alpha.PROVISIONING, disks[0].Status) + + vols := volumesForDisk(t, ctx, es.EAC, disks[0].ID) + require.Len(t, vols, 1, "a restore must leave exactly one volume on the disk") + assert.Equal(t, storage_v1alpha.DV_PENDING, vols[0].ActualState) + assert.Equal(t, storage_v1alpha.DV_PRESENT, vols[0].DesiredState) + assert.Equal(t, disks[0].VolumeId, vols[0].VolumeId) } // TestCreateDiskAndVolume_CleanupDoesNotHardDeleteDisk pins the key behavioral @@ -274,7 +310,7 @@ func TestCreateDiskAndVolume_CleanupDoesNotHardDeleteDisk(t *testing.T) { ctx := t.Context() es, resolver := setupResolver(t, nil) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) diskID := disks[0].ID @@ -355,7 +391,7 @@ func TestCreateDiskAndVolume_CleanupRunsAfterCancellation(t *testing.T) { // The resolver talks through a client that honours cancellation, so a // cleanup that inherited the dead context would fail its Patch. eac := entityserver_v1alpha.NewEntityAccessClient(cancelAwareRPC{Client: es.EAC.Client}) - resolver := newEntityDiskResolver(eac, entityserver.NewClient(testutils.TestLogger(t), eac)) + resolver := New(eac, entityserver.NewClient(testutils.TestLogger(t), eac)) target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) @@ -373,3 +409,143 @@ func TestCreateDiskAndVolume_CleanupRunsAfterCancellation(t *testing.T) { _, statErr := os.Stat(target.ImagePath) assert.True(t, os.IsNotExist(statErr), "the restored image must be removed, got %v", statErr) } + +// A disk that claims less capacity than the image it is given is wrong on its +// face, so the size rounds up rather than truncating. +func TestCreateDiskAndVolumeRoundsSizeUp(t *testing.T) { + ctx := t.Context() + + cases := []struct { + name string + sizeBytes int64 + wantGb int64 + }{ + {"exactly one GiB", 1 << 30, 1}, + {"a byte over one GiB", (1 << 30) + 1, 2}, + {"one and a half GiB", (1 << 30) * 3 / 2, 2}, + {"exactly two GiB", 2 << 30, 2}, + {"smaller than a GiB still gets one", 4096, 1}, + {"zero still gets one", 0, 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + es, resolver := setupResolver(t, nil) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", tc.sizeBytes, "ext4", t.TempDir()) + require.NoError(t, err) + require.NotNil(t, target) + + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, tc.wantGb, disks[0].SizeGb) + }) + } +} + +// The size comes out of a snapshot header, which is a file the caller handed +// us, so it is checked rather than trusted. Rounding by adding gib-1 would let +// a size near the top of the range overflow into a negative capacity, which the +// entity store would happily persist. +func TestDiskSizeGbRejectsSizesItCannotRepresent(t *testing.T) { + for _, tc := range []struct { + name string + sizeBytes int64 + }{ + {"negative", -1}, + {"very negative", math.MinInt64}, + {"large enough to overflow the rounding", math.MaxInt64}, + {"just past the representable ceiling", math.MaxInt64 - (1 << 30) + 2}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := diskSizeGb(tc.sizeBytes) + require.Error(t, err, "size %d should have been rejected", tc.sizeBytes) + }) + } +} + +// And the largest size it does accept round-trips back to bytes without +// overflowing, which is what the disk volume controller does with it. +func TestDiskSizeGbCeilingRoundTrips(t *testing.T) { + // The ceiling itself: exactly divisible by a GiB, so it rounds to a + // capacity that converts straight back to the same byte count. + sizeGb, err := diskSizeGb(math.MaxInt64 - (1 << 30) + 1) + require.NoError(t, err) + assert.Positive(t, sizeGb) + assert.Positive(t, sizeGb*(1<<30)/(1<<30), "the byte conversion must not overflow") +} + +// Once Finalize has committed the disk_volume, the image belongs to the +// controller's teardown, not to Cleanup. +// +// Marking the disk DELETING only starts that teardown; the unmount and detach +// happen later. Removing the image here would race them, and losing the race +// unlinks a file whose loop device is still attached, leaving the kernel +// holding an inode nothing can reach. +func TestCleanupLeavesTheImageToTeardownOnceAVolumeExists(t *testing.T) { + ctx := t.Context() + es, resolver := setupResolver(t, nil) + + dataPath := t.TempDir() + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", dataPath) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(target.ImagePath), 0700)) + require.NoError(t, os.WriteFile(target.ImagePath, []byte("restored"), 0600)) + + // Finalize commits the volume, so the image is now the teardown's business. + require.NoError(t, target.Finalize(ctx)) + require.NoError(t, target.Cleanup(ctx)) + + assert.FileExists(t, target.ImagePath, + "cleanup must not unlink an image the controller is still tearing down") + + // The disk is handed to the teardown path, which is what reclaims both. + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, storage_v1alpha.DELETING, disks[0].Status) +} + +// The common case is the opposite one: most cleanups run before Finalize, so no +// volume exists, nothing ever opened the image, and there is no teardown to +// wait for. Leaving it would just waste the space. +func TestCleanupRemovesTheImageWhenNoVolumeWasCommitted(t *testing.T) { + ctx := t.Context() + _, resolver := setupResolver(t, nil) + + dataPath := t.TempDir() + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", dataPath) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(target.ImagePath), 0700)) + require.NoError(t, os.WriteFile(target.ImagePath, []byte("restored"), 0600)) + + // No Finalize, so no volume. + require.NoError(t, target.Cleanup(ctx)) + + _, statErr := os.Stat(target.ImagePath) + assert.True(t, os.IsNotExist(statErr), + "with no volume to tear down, cleanup owns the image") +} + +// A disk is committed before the node lookup, and a lookup failure returns no +// RestoreTarget, so the caller has no Cleanup to call. Left alone the disk sits +// in RESTORING and the next restore of the same name finds it, skips creating +// one, and dies looking for the volume it never got. +func TestCreateDiskAndVolumeUnwindsWhenTheNodeLookupFails(t *testing.T) { + ctx := t.Context() + // FindNodeId lists nodes, and that is the first list this path makes. + fault := newFaultRPC(nil, "list", 1, fmt.Errorf("simulated node lookup failure")) + es, resolver := setupResolver(t, fault) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "finding node") + assert.Nil(t, target, "a failed creation must not hand back a target") + + // The disk it had already committed is on its way out rather than stuck. + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, storage_v1alpha.DELETING, disks[0].Status, + "a half-created disk must not be left in restoring, which blocks every same-name retry") +} diff --git a/pkg/snapshot/disk.go b/pkg/snapshot/disk.go index 7295a5708..63b4caf0a 100644 --- a/pkg/snapshot/disk.go +++ b/pkg/snapshot/disk.go @@ -2,17 +2,31 @@ package snapshot import ( "context" + "errors" "fmt" "path/filepath" ) const ( StatusDeleting = "DELETING" - StatusAttached = "ATTACHED" LeaseStatusBound = "BOUND" ) +// DiskNotFoundError reports that no disk answers to a name. +// +// It is a distinct type because "there is no such disk" and "I could not find +// out" call for opposite responses. Callers act on the first by creating the +// disk, or by reporting that there is nothing to recover. Acting on the second +// the same way turns a moment of entity-store trouble into a duplicate disk. +type DiskNotFoundError struct { + Name string +} + +func (e DiskNotFoundError) Error() string { + return fmt.Sprintf("disk %q not found", e.Name) +} + // DiskState holds the state of a disk entity as returned by a DiskResolver. type DiskState struct { ID string @@ -56,7 +70,6 @@ type BackupTarget struct { Name string Filesystem string ImagePath string - IsAttached bool // VolumeID is the node-local volume identifier. VolumeID string // CloudVolumeID identifies the volume in miren.cloud, for backups sent @@ -96,7 +109,6 @@ func PrepareBackup(ctx context.Context, resolver DiskResolver, name string, data Name: name, Filesystem: disk.Filesystem, ImagePath: resolveImagePath(vol, dataPath), - IsAttached: disk.Status == StatusAttached, VolumeID: vol.VolumeID, CloudVolumeID: vol.CloudVolumeID, @@ -116,10 +128,14 @@ func PrepareRestore(ctx context.Context, resolver DiskResolver, name string, dat disk, err := resolver.FindDisk(ctx, name) if err != nil { - if cfg.creator == nil { + var notFound DiskNotFoundError + if cfg.creator == nil || !errors.As(err, ¬Found) { + // Only an actual absence means "create it". A lookup that failed + // says nothing about whether the disk exists, and creating one on + // that basis is how a brief entity-store outage turns into two + // disks answering to the same name. return nil, err } - // Disk not found — create it return cfg.creator.CreateDiskAndVolume(ctx, name, cfg.sizeBytes, cfg.filesystem, dataPath) } diff --git a/pkg/snapshot/disk_test.go b/pkg/snapshot/disk_test.go index a56d306dd..ca9b68d8a 100644 --- a/pkg/snapshot/disk_test.go +++ b/pkg/snapshot/disk_test.go @@ -2,6 +2,7 @@ package snapshot import ( "context" + "errors" "fmt" "testing" @@ -54,17 +55,6 @@ func TestPrepareBackup(t *testing.T) { assert.Equal(t, "mydb", target.Name) assert.Equal(t, "ext4", target.Filesystem) assert.Equal(t, "/data/disk.img", target.ImagePath) - assert.False(t, target.IsAttached) - }) - - t.Run("attached disk sets flag", func(t *testing.T) { - r := &mockResolver{ - disk: &DiskState{ID: "d1", Name: "mydb", Status: StatusAttached, Filesystem: "ext4"}, - volume: &VolumeState{VolumeID: "v1", ImagePath: "/data/disk.img"}, - } - target, err := PrepareBackup(ctx, r, "mydb", "/var/lib/miren") - require.NoError(t, err) - assert.True(t, target.IsAttached) }) t.Run("deleting disk rejected", func(t *testing.T) { @@ -229,3 +219,41 @@ func TestPrepareRestore(t *testing.T) { assert.Equal(t, "/var/lib/miren/disk-data/volumes/vol-xyz/disk.img", target.ImagePath) }) } + +// mockCreator records whether PrepareRestore decided to create a disk. +type mockCreator struct { + called bool +} + +func (m *mockCreator) CreateDiskAndVolume(_ context.Context, name string, _ int64, _ string, _ string) (*RestoreTarget, error) { + m.called = true + return &RestoreTarget{Name: name, ImagePath: "/data/new.img", Created: true}, nil +} + +// "There is no such disk" and "I could not find out" call for opposite +// responses, and only the first one means create it. Treating a failed lookup +// as an absence is how a brief entity-store outage becomes two disks answering +// to the same name. +func TestPrepareRestoreOnlyCreatesOnAGenuineAbsence(t *testing.T) { + ctx := context.Background() + + t.Run("absent disk is created", func(t *testing.T) { + c := &mockCreator{} + r := &mockResolver{diskErr: DiskNotFoundError{Name: "mydb"}} + + target, err := PrepareRestore(ctx, r, "mydb", "/var/lib/miren", WithCreator(c, 1<<30, "ext4")) + require.NoError(t, err) + assert.True(t, c.called, "an absent disk should be created") + assert.True(t, target.Created) + }) + + t.Run("a failed lookup is reported, not papered over", func(t *testing.T) { + c := &mockCreator{} + r := &mockResolver{diskErr: errors.New("listing disks: etcd unreachable")} + + _, err := PrepareRestore(ctx, r, "mydb", "/var/lib/miren", WithCreator(c, 1<<30, "ext4")) + require.Error(t, err) + assert.Contains(t, err.Error(), "etcd unreachable") + assert.False(t, c.called, "a lookup failure must not create a disk") + }) +} diff --git a/pkg/workloadroles/roles.go b/pkg/workloadroles/roles.go index 2ea4f918c..c8d4b1bab 100644 --- a/pkg/workloadroles/roles.go +++ b/pkg/workloadroles/roles.go @@ -169,6 +169,14 @@ func clusterAdminPerms() perms { "internalhttp": set("dorequest"), "disks": set("new", "delete"), "addons": set("createinstance", "deleteinstance"), + // Backup, restore and recovery read and rewrite a disk's contents + // wholesale, so they sit with the other disk mutations rather than + // with reads — listdeleted included, since what it lists is the + // data of disks somebody deleted. + // transferoffset only reports how far an interrupted transfer got, + // but it is part of doing a backup or a restore, so it is granted + // with them rather than to every reader. + "diskbackup": set("backup", "restore", "listbackups", "listdeleted", "undelete", "transferoffset"), }, ) } diff --git a/servers/disk/backup.go b/servers/disk/backup.go new file mode 100644 index 000000000..efc34fb5d --- /dev/null +++ b/servers/disk/backup.go @@ -0,0 +1,352 @@ +package disk + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// Backup snapshots a disk, either to miren.cloud or down to the client. +func (s *Server) Backup(ctx context.Context, state *disk_v1alpha.DiskBackupBackup) error { + args := state.Args() + prog := s.newProgress(ctx, args.Progress()) + + target, err := s.prepareBackup(ctx, args.Disk()) + if err != nil { + return err + } + + // Backing up an image while its loop device is still writing gives a read + // whose head and tail come from different moments — weaker than the + // power-loss state fsck and Postgres's WAL recovery are built for. Say so + // and continue: the operator is the one deciding this is safe. + // + // Unlike restore, this check is best effort. Backup only reads, so being + // unable to tell whether the disk is in use is a reason to say less, not a + // reason to refuse. + inUse := false + if dev, err := s.liveImageDevice(target.ImagePath); err != nil { + s.log.Warn("could not tell whether disk image is in use", "disk", target.Name, "error", err) + prog.Warn("Could not tell whether %q is in use, so this backup may not be a point-in-time copy.", target.Name) + // Recorded as in use, because the honest answer is "possibly" and this + // flag only ever warns a future reader off relying on the snapshot. + inUse = true + } else if dev != "" { + inUse = true + s.log.Info("backing up a disk that is in use", "disk", target.Name, "device", dev) + prog.Warn("Disk %q is in use (%s) and may be written during the backup.", target.Name, dev) + prog.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") + prog.Warn("Detach the disk first for a backup you can rely on.") + } + + if args.ToCloud() { + return s.backupToCloud(ctx, state, prog, target, inUse) + } + return s.backupToClient(ctx, state, prog, target) +} + +func (s *Server) backupToCloud( + ctx context.Context, + state *disk_v1alpha.DiskBackupBackup, + prog progressSink, + target *snapshot.BackupTarget, + inUse bool, +) error { + if s.updates == nil { + return errNoCloud("backing up to miren.cloud") + } + if target.CloudVolumeID == "" { + return refuse( + "disk %q is not registered with miren.cloud yet, so there is nowhere to upload to — it registers on its own shortly after the disk is created", + target.Name, + ) + } + + prog.Message("Compressing and uploading %s to miren.cloud", target.Name) + + snapper := diskio.NewImageSnapshotter(s.log, s.updates) + res, err := snapper.Snapshot(ctx, diskio.SnapshotRequest{ + VolumeID: target.CloudVolumeID, + ImagePath: target.ImagePath, + Name: target.Name, + Filesystem: target.Filesystem, + SnapshotName: state.Args().Pin(), + StagingDir: s.stagingDir(target.ImagePath), + InUse: inUse, + }) + if err != nil { + return err + } + + s.log.Info("backed up disk to cloud", + "disk", target.Name, + "restore_point", res.UpdateID, + "image_size", res.ImageSize, + "compressed_size", res.CompressedSize, + ) + + out := new(disk_v1alpha.BackupResult) + out.SetImageSizeBytes(res.ImageSize) + out.SetCompressedSizeBytes(res.CompressedSize) + out.SetChecksum(res.Checksum) + out.SetRestorePointId(res.UpdateID) + state.Results().SetResult(&out) + return nil +} + +// backupToClient compresses the image and streams it down to the caller, which +// is how a cluster with no miren.cloud still produces a backup. +// +// The snapshot is staged to a file rather than compressed straight into the +// stream for two reasons. snapshot.Backup rewrites the header with the image's +// checksum once it has read the whole image, so it needs somewhere it can seek +// back to. And a staged snapshot is what makes the transfer resumable: a client +// that loses the connection at 90% comes back for the last 10% rather than +// making the server read and compress a multi-gigabyte image all over again. +func (s *Server) backupToClient( + ctx context.Context, + state *disk_v1alpha.DiskBackupBackup, + prog progressSink, + target *snapshot.BackupTarget, +) error { + args := state.Args() + + out := args.Data() + if out == nil { + return refuse("backup needs either --cloud or somewhere to write the snapshot") + } + + // Held across staging and the whole stream, not just the file opens. The + // window that matters is the one where another call could truncate the + // staged snapshot while this one is reading it. + release := s.transfers.acquire(args.TransferId()) + defer release() + + staged, meta, err := s.stageForClient(args.TransferId(), prog, target) + if err != nil { + return err + } + defer staged.Close() + + offset := args.Offset() + if offset < 0 || offset > meta.CompressedSize { + return refuse( + "this snapshot is %d bytes and the client offered to continue from %d", + meta.CompressedSize, offset, + ) + } + if _, err := staged.Seek(offset, io.SeekStart); err != nil { + return fmt.Errorf("seeking staged snapshot: %w", err) + } + + if offset > 0 { + prog.Message("Resuming at %d of %d bytes", offset, meta.CompressedSize) + } else { + prog.Message("Sending %d bytes", meta.CompressedSize) + } + + w := stream.ToWriter(ctx, out) + sent, err := io.Copy(w, s.trackReads(staged, prog, offset, meta.CompressedSize)) + if err != nil { + // The staged snapshot deliberately survives: the whole point is that + // the client can come back for the rest of it. + return fmt.Errorf("sending snapshot: %w", err) + } + // The stream belongs to the caller, so closing our writer flushes what we + // wrote without tearing their client down. + if err := w.Close(); err != nil { + return fmt.Errorf("finishing snapshot stream: %w", err) + } + if want := meta.CompressedSize - offset; sent != want { + return fmt.Errorf("sent %d bytes of the %d remaining", sent, want) + } + + // Delivered in full, so there is nothing left to resume. + s.discardStaging(args.TransferId()) + + s.log.Info("backed up disk to client", + "disk", target.Name, + "image_size", meta.ImageSize, + "compressed_size", meta.CompressedSize, + ) + + res := new(disk_v1alpha.BackupResult) + res.SetImageSizeBytes(meta.ImageSize) + res.SetCompressedSizeBytes(meta.CompressedSize) + res.SetChecksum(meta.Checksum) + state.Results().SetResult(&res) + return nil +} + +// stageForClient returns the compressed snapshot to send, compressing it only +// if this transfer has not already produced one. +// +// Reusing it is not just an optimization. A second compression pass would read +// the image as it is now, which for a live disk is not the image the client has +// already half-downloaded, and stitching the two together would produce a file +// that is the right length and not a valid snapshot of anything. +func (s *Server) stageForClient( + transferID string, + prog progressSink, + target *snapshot.BackupTarget, +) (*os.File, *stagedSnapshot, error) { + if meta, err := s.loadStaging(transferID); err != nil { + return nil, nil, err + } else if meta != nil { + path, _ := s.transferPath(transferID) + f, oerr := os.Open(path) + if oerr != nil { + return nil, nil, fmt.Errorf("reopening staged snapshot: %w", oerr) + } + return f, meta, nil + } + + s.sweepTransfers() + + img, err := os.Open(target.ImagePath) + if err != nil { + return nil, nil, fmt.Errorf("opening disk image: %w", err) + } + defer img.Close() + + info, err := img.Stat() + if err != nil { + return nil, nil, fmt.Errorf("stat disk image: %w", err) + } + + prog.Message("Compressing %s (%d bytes)", target.Name, info.Size()) + + path, err := s.transferPath(transferID) + if err != nil { + return nil, nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, nil, fmt.Errorf("creating transfer directory: %w", err) + } + + staged, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + return nil, nil, fmt.Errorf("creating staged snapshot: %w", err) + } + + checksum, err := snapshot.Backup(staged, img, target.Name, info.Size(), target.Filesystem) + if err != nil { + staged.Close() + os.Remove(path) + return nil, nil, err + } + + stagedInfo, err := staged.Stat() + if err != nil { + staged.Close() + os.Remove(path) + return nil, nil, fmt.Errorf("stat staged snapshot: %w", err) + } + + meta := &stagedSnapshot{ + Disk: target.Name, + ImageSize: info.Size(), + CompressedSize: stagedInfo.Size(), + Checksum: checksum, + } + // Written only now, so a staging file left behind by a server that died + // mid-compression has no marker and is never mistaken for a finished one. + if err := s.saveStaging(transferID, meta); err != nil { + staged.Close() + os.Remove(path) + return nil, nil, err + } + + // Compression left the file at its end. Rewind so a freshly staged snapshot + // and one reopened for a resume are handed back the same way, rather than + // leaving every caller to remember which it got. + if _, err := staged.Seek(0, io.SeekStart); err != nil { + staged.Close() + return nil, nil, fmt.Errorf("rewinding staged snapshot: %w", err) + } + + return staged, meta, nil +} + +// trackReads reports progress as bytes are pulled out of r, counting from what +// the client already has so a resumed transfer shows a bar that keeps going up +// rather than restarting. +func (s *Server) trackReads(r io.Reader, prog progressSink, done, total int64) io.Reader { + start := time.Now() + sentThisCall := int64(0) + var lastReport time.Time + + return readerFunc(func(p []byte) (int, error) { + n, err := r.Read(p) + done += int64(n) + sentThisCall += int64(n) + + // One event per 250ms. The client renders a bar from these, and a + // report per read would be thousands of RPCs for one backup. + if now := time.Now(); now.Sub(lastReport) >= 250*time.Millisecond || err == io.EOF { + lastReport = now + // Rate is measured over this attempt only. Counting bytes a + // previous attempt moved against this attempt's clock would + // report a throughput nothing achieved. + elapsed := now.Sub(start).Seconds() + var perSecond, eta int64 + if elapsed > 0 { + perSecond = int64(float64(sentThisCall) / elapsed) + } + if perSecond > 0 && total > done { + eta = (total - done) / perSecond + } + prog.Transfer(done, total, perSecond, eta) + } + return n, err + }) +} + +type readerFunc func(p []byte) (int, error) + +func (f readerFunc) Read(p []byte) (int, error) { return f(p) } + +// ListBackups returns the restore points a disk can be restored to, newest +// first. +func (s *Server) ListBackups(ctx context.Context, state *disk_v1alpha.DiskBackupListBackups) error { + args := state.Args() + + target, err := s.prepareBackup(ctx, args.Disk()) + if err != nil { + return err + } + + if s.updates == nil { + return errNoCloud("listing restore points") + } + if target.CloudVolumeID == "" { + // Not an error: a disk that has never been registered simply has no + // restore points, which is a different thing from a broken lookup. + state.Results().SetPoints(nil) + return nil + } + + updates, err := s.updates.List(ctx, target.CloudVolumeID, diskio.ListOptions{ + Kind: diskio.KindLoopImage, + Descending: true, + }) + if err != nil { + return fmt.Errorf("listing restore points for %s: %w", target.Name, err) + } + + points := make([]*disk_v1alpha.RestorePoint, 0, len(updates)) + for _, u := range updates { + points = append(points, restorePointFromUpdate(u)) + } + + state.Results().SetPoints(points) + return nil +} diff --git a/servers/disk/restore.go b/servers/disk/restore.go new file mode 100644 index 000000000..6e9bf2e46 --- /dev/null +++ b/servers/disk/restore.go @@ -0,0 +1,396 @@ +package disk + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// Restore rebuilds a disk's image, either from a restore point in miren.cloud +// or from a snapshot the client streams up. +// +// Restoring a disk this cluster has never seen creates it, which is the case +// that matters for disaster recovery: the host is gone and the whole task is to +// rebuild onto a new one. +func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRestore) (retErr error) { + args := state.Args() + prog := s.newProgress(ctx, args.Progress()) + + name := args.Disk() + if name == "" { + return refuse("disk name is required") + } + + // Two locks, because two different things are shared here. + // + // The disk name covers the entity work and the image path. A restore + // creates the disk when it does not exist, and two restores racing to the + // same new name both find it missing and both create one, or one catches + // the other's disk halfway built and fails looking for a volume that is not + // there yet. They also write the same .restore.tmp. Held for the + // whole handler, since the name is not free again until the image is + // installed. + releaseName := s.names.acquire(name) + defer releaseName() + + // The transfer id covers the uploaded snapshot, which lives in a file of + // its own and stays there until the image is installed. Different clients + // restoring to the same name bring different transfer ids, so this does not + // duplicate the lock above. A restore point needs none of it: it comes from + // the cloud and touches no transfer file. + if id := args.TransferId(); id != "" && args.RestorePoint() == "" { + release := s.transfers.acquire(id) + defer release() + } + + src, total, closeSrc, err := s.restoreSource(ctx, name, args, prog) + if err != nil { + return err + } + defer closeSrc() + + meta, err := snapshot.ReadHeader(src) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + + target, err := snapshot.PrepareRestore(ctx, s.disks, name, s.dataPath, + snapshot.WithCreator(s.disks, meta.SizeBytes, meta.Filesystem)) + if err != nil { + return err + } + + // Roll the entities back if anything below fails, but only when this + // restore is what created them. + defer func() { + if retErr == nil { + return + } + if target.Created && target.Cleanup != nil { + if cerr := target.Cleanup(ctx); cerr != nil { + s.log.Warn("failed to clean up after restore", "disk", name, "error", cerr) + } + } + // A refusal is the one failure the client will not come back from, so + // the uploaded bytes are already dead. Drop them now rather than leave + // a copy of the snapshot sitting there until the sweep, which a + // repeatedly-refused restore would do once per attempt. + s.discardRefusedUpload(args, retErr) + }() + + if err := s.refuseLiveImage(target, name); err != nil { + return err + } + + if !target.Created { + if _, err := os.Stat(target.ImagePath); err == nil && !args.Force() { + return refuse( + "disk %q already has an image at %s — pass --force to overwrite it", + name, target.ImagePath, + ) + } + } + + prog.Message("Restoring %s (%d bytes)", name, meta.SizeBytes) + + if err := s.writeImage(target.ImagePath, src, total, meta, prog); err != nil { + return err + } + + if target.Finalize != nil { + if err := target.Finalize(ctx); err != nil { + return err + } + } + + // Installed, so the uploaded copy has served its purpose. Only now: until + // the image is in place, those bytes are the only copy on this host and a + // retry would have to send them again. + if id := args.TransferId(); id != "" && args.RestorePoint() == "" { + s.discardTransfer(id) + } + + s.log.Info("restored disk", + "disk", name, + "image_size", meta.SizeBytes, + "created", target.Created, + ) + + res := new(disk_v1alpha.RestoreResult) + res.SetDisk(name) + res.SetImageSizeBytes(meta.SizeBytes) + res.SetCreated(target.Created) + state.Results().SetResult(&res) + return nil +} + +// discardRefusedUpload drops an uploaded snapshot the server has decided it +// will not use. +// +// The distinction that matters is refusal versus interruption. An interrupted +// upload is exactly what the transfer file is for and must survive; a refused +// one will never be picked up, because the client does not retry a refusal. +func (s *Server) discardRefusedUpload(args *disk_v1alpha.DiskBackupRestoreArgs, err error) { + if shouldDiscardUpload(args.TransferId(), args.RestorePoint(), err) { + s.discardTransfer(args.TransferId()) + } +} + +func shouldDiscardUpload(transferID, restorePoint string, err error) bool { + if transferID == "" || restorePoint != "" { + return false + } + var refusal cond.ErrValidationFailure + return errors.As(err, &refusal) +} + +// restoreSource resolves the snapshot to restore from and hands back a reader +// positioned at its start. +// +// The second return is the compressed size when it is known, and 0 when it is +// not, for progress reporting. +func (s *Server) restoreSource( + ctx context.Context, + name string, + args *disk_v1alpha.DiskBackupRestoreArgs, + prog progressSink, +) (io.Reader, int64, func(), error) { + if point := args.RestorePoint(); point != "" { + return s.downloadRestorePoint(ctx, name, point, prog) + } + + in := args.Data() + if in == nil { + return nil, 0, nil, refuse("restore needs either a restore point or a snapshot to read") + } + + // Land the whole snapshot on disk before touching the image. It is what + // makes an interrupted upload resumable, and it means a transfer that dies + // halfway cannot leave a half-decompressed image behind. + path, err := s.receiveSnapshot(ctx, args.TransferId(), args.Offset(), in, prog) + if err != nil { + return nil, 0, nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, 0, nil, fmt.Errorf("reopening uploaded snapshot: %w", err) + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, 0, nil, fmt.Errorf("stat uploaded snapshot: %w", err) + } + return f, info.Size(), func() { f.Close() }, nil +} + +// receiveSnapshot appends the client's bytes to this transfer's file and +// returns where it landed. +// +// The file is the checkpoint. Every byte that reaches it is a byte the client +// never has to send again, and fsync before returning is what makes that true +// across a server restart rather than only across a dropped connection. +func (s *Server) receiveSnapshot( + ctx context.Context, + transferID string, + offset int64, + in *stream.RecvStreamClient[[]byte], + prog progressSink, +) (string, error) { + if transferID == "" { + return "", refuse("uploading a snapshot needs a transfer id, so an interrupted upload can be resumed") + } + + s.sweepTransfers() + + f, err := s.openTransfer(transferID, offset) + if err != nil { + return "", err + } + defer f.Close() + + if offset > 0 { + prog.Message("Resuming upload at %d bytes", offset) + } else { + prog.Message("Reading snapshot from client") + } + + // The stream belongs to the caller; closing our reader would tear their + // client down, so leave it to them. + r := stream.ToReader(ctx, in) + + // Total is unknown — the client has not said how big its snapshot is — so + // progress shows bytes moved rather than a percentage. + written, err := io.Copy(f, s.trackReads(r, prog, offset, 0)) + + // Flush whatever did arrive before reporting either way. On the failure + // path this is the entire point: unsynced bytes would be re-sent for no + // reason, and worse, a length the client trusts might not survive a crash. + if serr := f.Sync(); serr != nil && err == nil { + err = fmt.Errorf("flushing uploaded snapshot: %w", serr) + } + if err != nil { + return "", fmt.Errorf("receiving snapshot after %d bytes: %w", written, err) + } + + return f.Name(), nil +} + +// downloadRestorePoint opens a restore point from miren.cloud. +// +// This one is not resumable here: the bytes come over the cloud's own HTTPS +// connection rather than the client's RPC session, so an interruption is +// between the server and the cloud, and re-fetching costs the operator nothing +// but time. +func (s *Server) downloadRestorePoint( + ctx context.Context, + name, point string, + prog progressSink, +) (io.Reader, int64, func(), error) { + if s.updates == nil { + return nil, 0, nil, errNoCloud("restoring from a restore point") + } + + // The restore point lives against the disk's cloud volume, so the disk has + // to already exist for this path. Creating a disk from a cloud restore + // point it has no record of is not something this can resolve. + target, err := s.prepareBackup(ctx, name) + if err != nil { + return nil, 0, nil, err + } + if target.CloudVolumeID == "" { + return nil, 0, nil, refuse("disk %q is not registered with miren.cloud, so it has no restore points", name) + } + + size := s.restorePointSize(ctx, target.CloudVolumeID, point) + + prog.Message("Downloading restore point %s", point) + + body, err := s.updates.Download(ctx, target.CloudVolumeID, point) + if err != nil { + return nil, 0, nil, fmt.Errorf("downloading restore point %s: %w", point, err) + } + return body, size, func() { body.Close() }, nil +} + +// restorePointSize looks up how big a restore point is, so the download can +// show a percentage. Best effort: a failure here costs a progress bar, not a +// restore, so it is logged rather than returned. +func (s *Server) restorePointSize(ctx context.Context, cloudVolumeID, point string) int64 { + updates, err := s.updates.List(ctx, cloudVolumeID, diskio.ListOptions{ + Kind: diskio.KindLoopImage, + Descending: true, + }) + if err != nil { + s.log.Debug("could not size restore point", "restore_point", point, "error", err) + return 0 + } + for _, u := range updates { + if u.UpdateID == point { + return u.Size + } + } + return 0 +} + +// refuseLiveImage stops a restore that would silently do nothing. +// +// See liveImageDevice: a loop device holds the image's inode, not its path, so +// writing a new image and renaming it into place leaves the running system on +// the old one. There is deliberately no --force for this. --force means +// "overwrite an existing image", and letting it also mean "write over a live +// loop device" would let an operator ask for a no-op and be told it worked. +func (s *Server) refuseLiveImage(target *snapshot.RestoreTarget, name string) error { + dev, err := s.liveImageDevice(target.ImagePath) + if err != nil { + return err + } + if dev == "" { + return nil + } + return refuse( + "disk %q is in use (%s is backing %s), and restoring it now would write an image nothing reads — "+ + "stop everything using the disk first, or restore into a new disk instead", + name, dev, target.ImagePath, + ) +} + +// writeImage decompresses a snapshot into the image path. +// +// It writes to a temp file and renames, so an interrupted restore cannot leave +// a half-written image that looks complete. +func (s *Server) writeImage(imagePath string, src io.Reader, compressedSize int64, meta *snapshot.Meta, prog progressSink) error { + if err := os.MkdirAll(filepath.Dir(imagePath), 0700); err != nil { + return fmt.Errorf("creating volume directory: %w", err) + } + + tmpPath := imagePath + ".restore.tmp" + out, err := os.Create(tmpPath) + if err != nil { + return fmt.Errorf("creating image file: %w", err) + } + + closed := false + cleanup := true + defer func() { + if !closed { + out.Close() + } + if cleanup { + os.Remove(tmpPath) + } + }() + + // Preallocate sparsely so the sparse-aware writer can seek over zero runs. + if err := out.Truncate(meta.SizeBytes); err != nil { + return fmt.Errorf("preallocating image: %w", err) + } + + if err := snapshot.RestoreImage(out, s.trackReads(src, prog, 0, compressedSize), meta); err != nil { + return err + } + + if err := out.Sync(); err != nil { + return fmt.Errorf("flushing image: %w", err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("closing image: %w", err) + } + closed = true + if err := os.Rename(tmpPath, imagePath); err != nil { + return fmt.Errorf("moving image into place: %w", err) + } + + // Syncing the file only promised its contents survive a power loss, not the + // directory entry that gives them this name. Losing the rename would leave + // the old image in place while the disk was already reported restored, and + // on a recovery path that is exactly the report you cannot have be wrong. + if err := syncDir(filepath.Dir(imagePath)); err != nil { + return err + } + + cleanup = false + return nil +} + +func syncDir(path string) error { + d, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening volume directory: %w", err) + } + defer d.Close() + + if err := d.Sync(); err != nil { + return fmt.Errorf("flushing volume directory: %w", err) + } + return nil +} diff --git a/servers/disk/resume_test.go b/servers/disk/resume_test.go new file mode 100644 index 000000000..3a21fa974 --- /dev/null +++ b/servers/disk/resume_test.go @@ -0,0 +1,215 @@ +package disk + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/snapshot" +) + +// diskImage builds content that compresses to something big enough to be worth +// resuming, and that a wrong byte would show up in. +func diskImage(size int) []byte { + b := make([]byte, size) + for i := range b { + b[i] = byte(i*7 + i/251) + } + return b +} + +// A resumed backup must reuse the snapshot it already compressed. Compressing +// again would read the image as it is now, which for a live disk is not the +// image the client already holds half of, and the two halves would not be a +// snapshot of anything. +func TestBackupResumeReusesTheStagedSnapshot(t *testing.T) { + content := diskImage(256 * 1024) + s, _, _, imagePath := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + first, meta, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, first.Close()) + require.NotZero(t, meta.CompressedSize) + + // Change the image underneath, as a live disk would. + require.NoError(t, os.WriteFile(imagePath, diskImage(256*1024+7777), 0644)) + + second, meta2, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, second.Close()) + + assert.Equal(t, *meta, *meta2, "a resumed transfer must describe the same snapshot") + + // A different transfer id is a different backup, and does see the new image. + third, meta3, err := s.stageForClient("t2", prog, target) + require.NoError(t, err) + require.NoError(t, third.Close()) + assert.NotEqual(t, meta.Checksum, meta3.Checksum) +} + +// The point of the whole exercise: bytes delivered in two pieces, split at an +// arbitrary offset, must restore to exactly the original image. +func TestBackupSplitAcrossTwoAttemptsRestoresIdentically(t *testing.T) { + content := diskImage(512 * 1024) + s, _, _, _ := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + staged, meta, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + defer staged.Close() + + // First attempt gets a third of the way and dies. + cut := meta.CompressedSize / 3 + require.Positive(t, cut) + + firstHalf := make([]byte, cut) + _, err = io.ReadFull(staged, firstHalf) + require.NoError(t, err) + + // Second attempt resumes from exactly what arrived. + resumed, meta2, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + defer resumed.Close() + require.Equal(t, meta.CompressedSize, meta2.CompressedSize) + + _, err = resumed.Seek(cut, io.SeekStart) + require.NoError(t, err) + secondHalf, err := io.ReadAll(resumed) + require.NoError(t, err) + + stitched := append(append([]byte{}, firstHalf...), secondHalf...) + require.Len(t, stitched, int(meta.CompressedSize)) + + // And what the client ends up with is a snapshot that restores to the + // original image, checksum and all. + src := bytes.NewReader(stitched) + hdr, err := snapshot.ReadHeader(src) + require.NoError(t, err) + assert.Equal(t, meta.Checksum, hdr.Checksum) + + restored := filepath.Join(t.TempDir(), "restored.img") + require.NoError(t, s.writeImage(restored, src, 0, hdr, prog)) + + got, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, got, "a resumed backup must restore byte for byte") +} + +// The mirror case: an upload that arrives in two pieces leaves the server +// holding exactly the file the client sent. +func TestUploadSplitAcrossTwoAttemptsIsReassembled(t *testing.T) { + content := diskImage(256 * 1024) + s, _, _, _ := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + // A real snapshot, so what is reassembled can be checked by restoring it. + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + staged, meta, err := s.stageForClient("src", prog, target) + require.NoError(t, err) + snapBytes, err := io.ReadAll(staged) + require.NoError(t, err) + require.NoError(t, staged.Close()) + require.Len(t, snapBytes, int(meta.CompressedSize)) + + cut := len(snapBytes) / 2 + + // First attempt delivers half, then the connection dies. + f, err := s.openTransfer("up1", 0) + require.NoError(t, err) + _, err = f.Write(snapBytes[:cut]) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, f.Close()) + + // The client asks where to pick up, and is told what actually landed. + path, err := s.transferPath("up1") + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(cut), info.Size()) + + // Second attempt sends the rest from there. + f, err = s.openTransfer("up1", int64(cut)) + require.NoError(t, err) + _, err = f.Write(snapBytes[cut:]) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, f.Close()) + + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, snapBytes, got, "the reassembled upload must match what was sent") + + // And it really is a usable snapshot. + src := bytes.NewReader(got) + hdr, err := snapshot.ReadHeader(src) + require.NoError(t, err) + + restored := filepath.Join(t.TempDir(), "restored.img") + require.NoError(t, s.writeImage(restored, src, 0, hdr, prog)) + + back, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, back) +} + +// A completed backup releases its staging, so a transfer that finished does not +// keep a copy of the disk around. +func TestFinishedBackupReleasesItsStaging(t *testing.T) { + s, _, _, _ := newTestServer(t, diskImage(64*1024)) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + staged, _, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, staged.Close()) + + path, err := s.transferPath("t1") + require.NoError(t, err) + metaPath, err := s.stagingMetaPath("t1") + require.NoError(t, err) + require.FileExists(t, path) + require.FileExists(t, metaPath) + + s.discardStaging("t1") + + _, err = os.Stat(path) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(metaPath) + assert.True(t, os.IsNotExist(err)) +} + +// An interrupted upload is exactly what the transfer file exists for and must +// survive. A refused one never will be picked up, because the client does not +// retry a refusal, so leaving it would keep a copy of the snapshot around for +// every attempt until the sweep. +func TestOnlyRefusedUploadsAreDiscarded(t *testing.T) { + interrupted := errors.New("connection reset by peer") + refused := cond.ValidationFailure("disk-backup", `disk "data" is in use`) + + assert.True(t, shouldDiscardUpload("t1", "", refused)) + assert.False(t, shouldDiscardUpload("t1", "", interrupted)) + + // A restore point came from the cloud, so there is no upload to discard. + assert.False(t, shouldDiscardUpload("t1", "point-1", refused)) + + // And nothing to discard without an id. + assert.False(t, shouldDiscardUpload("", "", refused)) +} diff --git a/servers/disk/server.go b/servers/disk/server.go new file mode 100644 index 000000000..355e02671 --- /dev/null +++ b/servers/disk/server.go @@ -0,0 +1,250 @@ +// Package disk serves the DiskBackup RPC interface: backup and restore of disk +// images, driven by a client that need not be running on the server host. +// +// Two steps genuinely have to happen here rather than on the client. Moving the +// image bytes, because the image is a file in the server's data directory. And +// talking to miren.cloud, because that authenticates as the cluster using a key +// that lives only on the server and should never reach a laptop. Everything +// else — resolving which disk the operator means, deciding which restore point +// to use, writing a snapshot to a file — is orchestration the client does. +// +// See RFD-108. +package disk + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/rpc/standard" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// diskEntities is what the handlers need from the entity store: which disk the +// operator means, how to conjure one that does not exist yet, and which node +// stateful workloads run on. +type diskEntities interface { + snapshot.DiskResolver + snapshot.DiskCreator + + FindNodeId(ctx context.Context) (entity.Id, error) +} + +// Server implements disk_v1alpha.DiskBackup. +type Server struct { + log *slog.Logger + disks diskEntities + dataPath string + + // Recovering a deleted disk rebuilds entities from what the holding area + // recorded, rather than from anything a caller supplied, so it writes to + // the entity store directly instead of through the resolver. + eac *entityserver_v1alpha.EntityAccessClient + ec *entityserver.Client + + // updates is nil on a cluster with no miren.cloud registration. The + // streaming paths still work in that case; only the cloud ones refuse. + updates diskio.CloudUpdatesClient + + // mntOps answers "is this image currently in use". It reads kernel state + // directly, so it needs no handle on the runner's controllers. + mntOps diskio.DiskMountOps + + // transfers serializes concurrent work on one transfer id, and names does + // the same for one disk name, so a check followed by a create cannot be + // overtaken between the two steps. + transfers *keyedLocks + names *keyedLocks +} + +// NewServer builds the disk backup service. updates may be nil, which means the +// cluster has no cloud to back up to. +func NewServer( + log *slog.Logger, + eac *entityserver_v1alpha.EntityAccessClient, + ec *entityserver.Client, + dataPath string, + updates diskio.CloudUpdatesClient, +) *Server { + log = log.With("module", "disk-backup") + return &Server{ + log: log, + disks: diskresolve.New(eac, ec), + dataPath: dataPath, + eac: eac, + ec: ec, + updates: updates, + mntOps: diskio.NewRealDiskMountOps(log), + transfers: newKeyedLocks(), + names: newKeyedLocks(), + } +} + +// errCategory groups this service's errors in the RPC error envelope. +const errCategory = "disk-backup" + +// refuse reports a request this server will not carry out, as opposed to one +// that failed partway. It travels as a validation failure so the client prints +// the message on its own rather than wrapping it in "remote error: generic +// unknown", which buries the sentence an operator needs to read. +func refuse(format string, args ...any) error { + return cond.ValidationFailure(errCategory, fmt.Sprintf(format, args...)) +} + +// errNoCloud is what every cloud-dependent path reports. It names the command +// that would fix it, because "no cloud configured" on its own leaves an +// operator guessing whether that is a bug or a setup step they skipped. +func errNoCloud(what string) error { + return refuse( + "%s needs a miren.cloud registration, and this cluster has none — run `miren register` first, or back up to a local file instead", + what, + ) +} + +// liveImageDevice reports the loop device currently backing an image, or "" if +// there is none. +// +// This is the check that matters before rewriting an image, and it is not the +// same question as "does a lease exist". LoopAttach hands the kernel an open +// file descriptor, so a loop device is bound to the image's inode and not to +// its path. Renaming a fresh image over an attached one therefore changes +// nothing an operator can see: the old inode survives unlinked, the mounted +// filesystem keeps reading and writing it, and the restore reports success +// having accomplished nothing. +// +// A lease check does not cover this. Universal-mode volumes are mounted by the +// volume controller whether or not anything has leased them, and releasing the +// last lease deliberately leaves the mount up. +func (s *Server) liveImageDevice(imagePath string) (string, error) { + dev, err := s.mntOps.FindLoopByBacking(imagePath) + if err != nil { + // Fail closed. Not knowing whether the image is in use is not the same + // as knowing it is idle, and guessing wrong here loses data silently. + return "", fmt.Errorf("checking whether %s is in use: %w", imagePath, err) + } + return dev, nil +} + +// progressSink adapts the client's progress stream to something the handlers +// can call without checking for nil or caring about send failures. A client +// that has stopped listening should not fail an in-flight backup. +type progressSink struct { + send func(*disk_v1alpha.Progress) +} + +func (p progressSink) Message(format string, args ...any) { + up := new(disk_v1alpha.Progress) + up.Update().SetMessage(fmt.Sprintf(format, args...)) + p.send(up) +} + +// Warn sends a warning to the client. Callers log the underlying condition +// themselves, once, rather than having this emit a server log line per line of +// operator-facing text. +func (p progressSink) Warn(format string, args ...any) { + up := new(disk_v1alpha.Progress) + up.Update().SetWarning(fmt.Sprintf(format, args...)) + p.send(up) +} + +func (p progressSink) Transfer(done, total, perSecond, etaSeconds int64) { + t := new(disk_v1alpha.Transfer) + t.SetDone(done) + t.SetTotal(total) + t.SetBytesPerSecond(perSecond) + t.SetEtaSeconds(etaSeconds) + + up := new(disk_v1alpha.Progress) + up.Update().SetTransfer(t) + p.send(up) +} + +// newProgress wraps a client's progress stream, tolerating a caller that did +// not supply one. +func (s *Server) newProgress(ctx context.Context, out *stream.SendStreamClient[*disk_v1alpha.Progress]) progressSink { + return progressSink{ + send: func(up *disk_v1alpha.Progress) { + if out == nil { + return + } + if _, err := out.Send(ctx, up); err != nil { + s.log.Debug("dropping progress event", "error", err) + } + }, + } +} + +// prepareBackup resolves a disk to an image on this host. +func (s *Server) prepareBackup(ctx context.Context, name string) (*snapshot.BackupTarget, error) { + if name == "" { + return nil, refuse("disk name is required") + } + return snapshot.PrepareBackup(ctx, s.disks, name, s.dataPath) +} + +// restorePointFromUpdate converts a cloud update into the restore point a +// client picks from. +// +// The cloud's UpdateInfo carries no timestamp, so it is recovered from the +// ordering key, whose format is fixed per update kind: loop_image keys are Unix +// nanoseconds in 16 hex digits. +func restorePointFromUpdate(u diskio.UpdateInfo) *disk_v1alpha.RestorePoint { + rp := new(disk_v1alpha.RestorePoint) + rp.SetId(u.UpdateID) + rp.SetSizeBytes(u.Size) + rp.SetName(u.SnapshotName) + rp.SetMode(modeForKind(u.Kind)) + if ts := timestamp(orderingKeyTime(u)); ts != nil { + rp.SetCreatedAt(ts) + } + return rp +} + +func modeForKind(kind string) string { + switch diskio.UpdateKind(kind) { + case diskio.KindLoopImage: + return "universal" + case diskio.KindLBDLog: + return "accelerator" + default: + return kind + } +} + +func orderingKeyTime(u diskio.UpdateInfo) time.Time { + if diskio.UpdateKind(u.Kind) != diskio.KindLoopImage { + return time.Time{} + } + var nanos int64 + if _, err := fmt.Sscanf(u.OrderingKey, "%016x", &nanos); err != nil { + return time.Time{} + } + return time.Unix(0, nanos) +} + +func timestamp(t time.Time) *standard.Timestamp { + if t.IsZero() { + return nil + } + ts := new(standard.Timestamp) + ts.SetSeconds(t.Unix()) + ts.SetNanoseconds(int32(t.Nanosecond())) + return ts +} + +// stagingDir is where a snapshot is compressed before it is uploaded or +// streamed. It sits beside the image so the write stays on the same filesystem +// as the data it came from, which is the one sized for it. +func (s *Server) stagingDir(imagePath string) string { + return filepath.Dir(imagePath) +} diff --git a/servers/disk/server_test.go b/servers/disk/server_test.go new file mode 100644 index 000000000..192417713 --- /dev/null +++ b/servers/disk/server_test.go @@ -0,0 +1,323 @@ +package disk + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/snapshot" +) + +// fakeDisks stands in for the entity store. +type fakeDisks struct { + disk *snapshot.DiskState + volume *snapshot.VolumeState + leases []snapshot.LeaseState + created *snapshot.RestoreTarget + nodeErr error +} + +func (f *fakeDisks) FindDisk(_ context.Context, name string) (*snapshot.DiskState, error) { + if f.disk == nil { + return nil, fmt.Errorf("disk %q not found", name) + } + return f.disk, nil +} + +func (f *fakeDisks) FindVolume(context.Context, string) (*snapshot.VolumeState, error) { + if f.volume == nil { + return nil, errors.New("no volume") + } + return f.volume, nil +} + +func (f *fakeDisks) FindLeases(context.Context, string) ([]snapshot.LeaseState, error) { + return f.leases, nil +} + +func (f *fakeDisks) FindNodeId(context.Context) (entity.Id, error) { + if f.nodeErr != nil { + return "", f.nodeErr + } + return entity.Id("node/n1"), nil +} + +func (f *fakeDisks) CreateDiskAndVolume(context.Context, string, int64, string, string) (*snapshot.RestoreTarget, error) { + if f.created == nil { + return nil, errors.New("creation not configured") + } + return f.created, nil +} + +// fakeUpdates stands in for miren.cloud. +type fakeUpdates struct { + uploaded []diskio.UploadRequest + bodies [][]byte + list []diskio.UpdateInfo + download map[string][]byte + + listErr error + downloadErr error +} + +func (f *fakeUpdates) Upload(_ context.Context, _ string, req diskio.UploadRequest, body io.Reader, size int64) (string, error) { + data, err := io.ReadAll(body) + if err != nil { + return "", err + } + if int64(len(data)) != size { + return "", fmt.Errorf("declared %d bytes but body is %d", size, len(data)) + } + f.uploaded = append(f.uploaded, req) + f.bodies = append(f.bodies, data) + return "volup-test", nil +} + +func (f *fakeUpdates) List(context.Context, string, diskio.ListOptions) ([]diskio.UpdateInfo, error) { + return f.list, f.listErr +} + +func (f *fakeUpdates) Download(_ context.Context, _, updateID string) (io.ReadCloser, error) { + if f.downloadErr != nil { + return nil, f.downloadErr + } + data, ok := f.download[updateID] + if !ok { + return nil, fmt.Errorf("no such update %q", updateID) + } + return io.NopCloser(newReader(data)), nil +} + +// fakeMountOps reports whether an image is loop-attached. +type fakeMountOps struct { + diskio.DiskMountOps + device string + err error +} + +func (f fakeMountOps) FindLoopByBacking(string) (string, error) { return f.device, f.err } + +// newTestServer builds a server over a temp data directory holding one disk +// image, and returns the server, its fakes, and the image path. +func newTestServer(t *testing.T, content []byte) (*Server, *fakeDisks, *fakeUpdates, string) { + t.Helper() + + dataPath := t.TempDir() + volDir := filepath.Join(dataPath, "disk-data", "volumes", "vol-1") + require.NoError(t, os.MkdirAll(volDir, 0700)) + imagePath := filepath.Join(volDir, "disk.img") + require.NoError(t, os.WriteFile(imagePath, content, 0644)) + + disks := &fakeDisks{ + disk: &snapshot.DiskState{ + ID: "disk/1", Name: "data", Status: "PROVISIONED", Filesystem: "ext4", + }, + volume: &snapshot.VolumeState{ + VolumeID: "vol-1", CloudVolumeID: "cloud-vol-1", ImagePath: imagePath, + }, + } + updates := &fakeUpdates{download: map[string][]byte{}} + + s := &Server{ + log: slog.Default(), + disks: disks, + dataPath: dataPath, + updates: updates, + mntOps: fakeMountOps{}, + + transfers: newKeyedLocks(), + names: newKeyedLocks(), + } + return s, disks, updates, imagePath +} + +func TestPrepareBackupRequiresAName(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + _, err := s.prepareBackup(context.Background(), "") + require.Error(t, err) + assert.Contains(t, err.Error(), "disk name is required") +} + +func TestPrepareBackupResolvesTheImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + assert.Equal(t, imagePath, target.ImagePath) + assert.Equal(t, "cloud-vol-1", target.CloudVolumeID) +} + +// A cluster with no cloud must say so rather than failing obscurely, and it +// must point at the thing that would fix it. +func TestNoCloudErrorNamesTheRemedy(t *testing.T) { + err := errNoCloud("backing up to miren.cloud") + assert.Contains(t, err.Error(), "backing up to miren.cloud") + assert.Contains(t, err.Error(), "miren register") + assert.Contains(t, err.Error(), "local file") +} + +// Backup of an unregistered disk is what makes an air-gapped cluster work, so +// resolution must succeed and simply report no cloud volume. +func TestPrepareBackupToleratesAnUnregisteredDisk(t *testing.T) { + s, disks, _, imagePath := newTestServer(t, []byte("hello")) + disks.volume.CloudVolumeID = "" + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + assert.Equal(t, imagePath, target.ImagePath) + assert.Empty(t, target.CloudVolumeID) +} + +func TestRestorePointCarriesATimestampDecodedFromTheOrderingKey(t *testing.T) { + when := time.Unix(0, 1_700_000_000_123_456_789) + + rp := restorePointFromUpdate(diskio.UpdateInfo{ + UpdateID: "u-1", + Kind: string(diskio.KindLoopImage), + OrderingKey: fmt.Sprintf("%016x", when.UnixNano()), + Size: 4096, + SnapshotName: "pre-migration", + }) + + assert.Equal(t, "u-1", rp.Id()) + assert.Equal(t, int64(4096), rp.SizeBytes()) + assert.Equal(t, "pre-migration", rp.Name()) + assert.Equal(t, "universal", rp.Mode()) + require.True(t, rp.HasCreatedAt()) + assert.Equal(t, when.Unix(), rp.CreatedAt().Seconds()) + assert.Equal(t, int32(when.Nanosecond()), rp.CreatedAt().Nanoseconds()) +} + +// An accelerator-mode segment has a TAI64N key, which is not Unix nanoseconds. +// Reporting a garbage date would be worse than reporting none. +func TestRestorePointOmitsATimestampItCannotDecode(t *testing.T) { + rp := restorePointFromUpdate(diskio.UpdateInfo{ + UpdateID: "u-2", + Kind: string(diskio.KindLBDLog), + OrderingKey: "400000005f5e1000", + }) + + assert.Equal(t, "accelerator", rp.Mode()) + assert.False(t, rp.HasCreatedAt()) +} + +func TestLiveImageDeviceReportsAnAttachedLoop(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{device: "/dev/loop3"} + + dev, err := s.liveImageDevice(imagePath) + require.NoError(t, err) + assert.Equal(t, "/dev/loop3", dev) +} + +// Not knowing whether an image is in use is not the same as knowing it is idle. +func TestLiveImageDeviceFailsClosed(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{err: errors.New("sysfs unreadable")} + + _, err := s.liveImageDevice(imagePath) + require.Error(t, err) + assert.Contains(t, err.Error(), "sysfs unreadable") +} + +// The refusal is the whole point of the check: renaming an image over a live +// loop device is silently a no-op, so a restore that "succeeded" would have +// changed nothing. +func TestRestoreRefusesALiveImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{device: "/dev/loop7"} + + err := s.refuseLiveImage(&snapshot.RestoreTarget{ImagePath: imagePath}, "data") + require.Error(t, err) + assert.Contains(t, err.Error(), "/dev/loop7") + assert.Contains(t, err.Error(), "nothing reads") +} + +func TestRestoreAllowsAnIdleImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + + require.NoError(t, s.refuseLiveImage(&snapshot.RestoreTarget{ImagePath: imagePath}, "data")) +} + +// A backup written by this server must be restorable by it, byte for byte. +func TestWriteImageRoundTripsASnapshot(t *testing.T) { + content := make([]byte, 128*1024) + for i := range content { + content[i] = byte(i % 251) + } + + s, _, _, imagePath := newTestServer(t, content) + + staged := filepath.Join(t.TempDir(), "snap.miren.zst") + out, err := os.Create(staged) + require.NoError(t, err) + + img, err := os.Open(imagePath) + require.NoError(t, err) + defer img.Close() + + _, err = snapshot.Backup(out, img, "data", int64(len(content)), "ext4") + require.NoError(t, err) + require.NoError(t, out.Close()) + + src, err := os.Open(staged) + require.NoError(t, err) + defer src.Close() + + meta, err := snapshot.ReadHeader(src) + require.NoError(t, err) + + restored := filepath.Join(t.TempDir(), "restored.img") + prog := s.newProgress(context.Background(), nil) + require.NoError(t, s.writeImage(restored, src, 0, meta, prog)) + + got, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, got) +} + +// An interrupted restore must not leave something that looks like a finished +// image. +func TestWriteImageLeavesNothingBehindOnACorruptSnapshot(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + restored := filepath.Join(t.TempDir(), "restored.img") + prog := s.newProgress(context.Background(), nil) + + meta := &snapshot.Meta{Name: "data", SizeBytes: 4096, Checksum: "deadbeef"} + err := s.writeImage(restored, newReader([]byte("not a zstd stream")), 0, meta, prog) + require.Error(t, err) + + _, statErr := os.Stat(restored) + assert.True(t, os.IsNotExist(statErr), "a failed restore must not leave an image behind") + + _, tmpErr := os.Stat(restored + ".restore.tmp") + assert.True(t, os.IsNotExist(tmpErr), "a failed restore must not leave its temp file behind") +} + +func newReader(b []byte) io.Reader { return &sliceReader{b: b} } + +type sliceReader struct { + b []byte + i int +} + +func (r *sliceReader) Read(p []byte) (int, error) { + if r.i >= len(r.b) { + return 0, io.EOF + } + n := copy(p, r.b[r.i:]) + r.i += n + return n, nil +} diff --git a/servers/disk/transfer.go b/servers/disk/transfer.go new file mode 100644 index 000000000..348cbd688 --- /dev/null +++ b/servers/disk/transfer.go @@ -0,0 +1,312 @@ +package disk + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" +) + +const ( + // transfersDir holds the partial snapshots of transfers that are still in + // flight, beside the volumes they belong to so they land on the filesystem + // sized for disk data. + transfersDir = "transfers" + + // transferTTL is how long an abandoned transfer's bytes are kept. Long + // enough to outlive an outage and a night's sleep, short enough that a + // client that never comes back does not hoard a multi-gigabyte file + // forever. + transferTTL = 24 * time.Hour +) + +// keyedLocks serializes work on any one key: a transfer id, or a disk name. +// +// The transfer files are shared mutable state, and nothing else guards them. +// Two calls carrying the same id can otherwise both pass openTransfer's offset +// check and interleave their appends, or one can truncate a staged snapshot +// while the other is streaming it. A well-behaved client never does this, since +// it picks a fresh id per invocation and retries in sequence, but the server +// should not be relying on that. +// +// Locks are reference counted so the map does not grow by one entry per backup +// forever. Dropping an entry while a caller still held a pointer to it would be +// worse than the leak: the next caller would mint a second lock for the same id +// and the two would run concurrently anyway. +type keyedLocks struct { + mu sync.Mutex + locks map[string]*keyedLock +} + +type keyedLock struct { + mu sync.Mutex + refs int +} + +func newKeyedLocks() *keyedLocks { + return &keyedLocks{locks: map[string]*keyedLock{}} +} + +// acquire blocks until this key is free and returns the release function. +func (t *keyedLocks) acquire(id string) func() { + t.mu.Lock() + l, ok := t.locks[id] + if !ok { + l = &keyedLock{} + t.locks[id] = l + } + l.refs++ + t.mu.Unlock() + + l.mu.Lock() + + return func() { + l.mu.Unlock() + + t.mu.Lock() + l.refs-- + if l.refs == 0 { + delete(t.locks, id) + } + t.mu.Unlock() + } +} + +// transferPath is where a transfer's bytes accumulate. +// +// The id comes from the client, so it is checked rather than trusted: it names +// a file under a directory this server owns, and a caller must not be able to +// aim that at something else. +func (s *Server) transferPath(id string) (string, error) { + if id == "" { + return "", refuse("a transfer id is required to resume an interrupted transfer") + } + if len(id) > 128 { + return "", refuse("transfer id is too long") + } + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_': + default: + return "", refuse("transfer id may only contain letters, digits, dashes and underscores") + } + } + return filepath.Join(s.diskDataPath(), transfersDir, id+".part"), nil +} + +// TransferOffset reports how many bytes of a transfer the server already holds. +// +// An id it has never seen reports zero rather than failing, so a client's first +// attempt and its retries take the same path. +func (s *Server) TransferOffset(ctx context.Context, state *disk_v1alpha.DiskBackupTransferOffset) error { + path, err := s.transferPath(state.Args().TransferId()) + if err != nil { + return err + } + + info, err := os.Stat(path) + switch { + case os.IsNotExist(err): + state.Results().SetReceivedBytes(0) + return nil + case err != nil: + return fmt.Errorf("checking transfer progress: %w", err) + } + + state.Results().SetReceivedBytes(info.Size()) + return nil +} + +// openTransfer opens a transfer's file for appending, creating it if this is +// the first attempt, and checks the caller is resuming from where the server +// actually is. +// +// Refusing a mismatched offset rather than seeking to it is deliberate. The two +// ends disagreeing about how much arrived is exactly the situation where +// writing anyway produces a file that is the right length and the wrong bytes, +// and nothing would notice until a restore months later. +func (s *Server) openTransfer(id string, offset int64) (*os.File, error) { + path, err := s.transferPath(id) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, fmt.Errorf("creating transfer directory: %w", err) + } + + var have int64 + if info, serr := os.Stat(path); serr == nil { + have = info.Size() + } else if !os.IsNotExist(serr) { + return nil, fmt.Errorf("checking transfer progress: %w", serr) + } + + if offset != have { + return nil, refuse( + "this transfer is at %d bytes but the client offered to continue from %d — ask transferOffset where to resume, or use a new transfer id to start over", + have, offset, + ) + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return nil, fmt.Errorf("opening transfer file: %w", err) + } + return f, nil +} + +// stagedSnapshot describes a compressed snapshot waiting to be collected. +// +// It exists because the checksum and the image's size are known only while the +// snapshot is being compressed, and a resumed transfer skips that step. Without +// it, a client that reconnected for the last megabyte would be told the backup +// had no checksum. +type stagedSnapshot struct { + Disk string `json:"disk"` + ImageSize int64 `json:"image_size"` + CompressedSize int64 `json:"compressed_size"` + Checksum string `json:"checksum"` +} + +func (s *Server) stagingMetaPath(id string) (string, error) { + path, err := s.transferPath(id) + if err != nil { + return "", err + } + return strings.TrimSuffix(path, ".part") + ".meta", nil +} + +// loadStaging reports the finished snapshot waiting under this transfer id, or +// nil when there is none. +// +// The metadata file doubles as the completion marker: it is written only after +// compression finishes, so bytes left behind by a server that died partway +// through have no metadata and are correctly treated as nothing at all. +func (s *Server) loadStaging(id string) (*stagedSnapshot, error) { + metaPath, err := s.stagingMetaPath(id) + if err != nil { + return nil, err + } + + data, err := os.ReadFile(metaPath) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading staged snapshot metadata: %w", err) + } + + var meta stagedSnapshot + if err := json.Unmarshal(data, &meta); err != nil { + // Unreadable metadata means we cannot say what the staged bytes are, so + // treat the transfer as absent rather than guess. + s.log.Warn("discarding a staged snapshot with unreadable metadata", "transfer_id", id, "error", err) + s.discardStaging(id) + return nil, nil + } + + path, err := s.transferPath(id) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if os.IsNotExist(err) { + s.discardStaging(id) + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("checking staged snapshot: %w", err) + } + if info.Size() != meta.CompressedSize { + s.log.Warn("discarding a staged snapshot that does not match its metadata", + "transfer_id", id, "on_disk", info.Size(), "recorded", meta.CompressedSize) + s.discardStaging(id) + return nil, nil + } + + return &meta, nil +} + +func (s *Server) saveStaging(id string, meta *stagedSnapshot) error { + metaPath, err := s.stagingMetaPath(id) + if err != nil { + return err + } + data, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("encoding staged snapshot metadata: %w", err) + } + if err := os.WriteFile(metaPath, data, 0600); err != nil { + return fmt.Errorf("writing staged snapshot metadata: %w", err) + } + return nil +} + +// discardStaging removes a staged snapshot and its metadata. +func (s *Server) discardStaging(id string) { + if metaPath, err := s.stagingMetaPath(id); err == nil { + if rerr := os.Remove(metaPath); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove staged snapshot metadata", "transfer_id", id, "error", rerr) + } + } + s.discardTransfer(id) +} + +// discardTransfer removes a transfer's bytes once they are no longer needed. +func (s *Server) discardTransfer(id string) { + path, err := s.transferPath(id) + if err != nil { + return + } + if rerr := os.Remove(path); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove finished transfer", "transfer_id", id, "error", rerr) + } +} + +// sweepTransfers deletes transfers nobody came back for. +// +// This runs when a transfer starts rather than on a timer: it is the moment a +// transfer directory is known to be in use, and it keeps the cleanup on the +// path that creates the mess rather than in a goroutine that has to be owned +// and shut down. +func (s *Server) sweepTransfers() { + dir := filepath.Join(s.diskDataPath(), transfersDir) + entries, err := os.ReadDir(dir) + if err != nil { + if !os.IsNotExist(err) { + s.log.Warn("could not sweep abandoned transfers", "error", err) + } + return + } + + cutoff := time.Now().Add(-transferTTL) + for _, e := range entries { + if e.IsDir() { + continue + } + // Both halves of a staged backup are swept. Taking only the .part would + // leave its .meta behind forever, since nothing else ever looks at a + // metadata file whose bytes are gone. + if !strings.HasSuffix(e.Name(), ".part") && !strings.HasSuffix(e.Name(), ".meta") { + continue + } + info, ierr := e.Info() + if ierr != nil || info.ModTime().After(cutoff) { + continue + } + path := filepath.Join(dir, e.Name()) + if rerr := os.Remove(path); rerr != nil { + s.log.Warn("failed to remove abandoned transfer", "path", path, "error", rerr) + continue + } + s.log.Info("removed abandoned transfer", + "path", path, "size", info.Size(), "idle_for", time.Since(info.ModTime()).Truncate(time.Minute)) + } +} diff --git a/servers/disk/transfer_test.go b/servers/disk/transfer_test.go new file mode 100644 index 000000000..e32a48b2f --- /dev/null +++ b/servers/disk/transfer_test.go @@ -0,0 +1,266 @@ +package disk + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A transfer id names a file under a directory the server owns, and it comes +// from the client, so it must not be usable to point somewhere else. +func TestTransferPathRejectsIdsThatEscapeTheDirectory(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + for _, id := range []string{ + "../../etc/passwd", + "a/b", + "..", + ".", + `a\b`, + "a b", + "a.part", + "", + } { + _, err := s.transferPath(id) + require.Error(t, err, "id %q should be rejected", id) + } +} + +func TestTransferPathAcceptsOrdinaryIds(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("0a1b2c3d-4e5f_6789") + require.NoError(t, err) + assert.Equal(t, + filepath.Join(s.diskDataPath(), transfersDir, "0a1b2c3d-4e5f_6789.part"), + path) +} + +func TestTransferOffsetIsZeroForAnUnknownId(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + // Zero rather than an error is what lets a client's first attempt and its + // retries be the same code path. + f, err := s.openTransfer("never-seen", 0) + require.NoError(t, err) + require.NoError(t, f.Close()) +} + +// The two ends disagreeing about how much arrived is exactly when writing +// anyway produces a file of the right length and the wrong bytes. +func TestOpenTransferRefusesAMismatchedOffset(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + f, err := s.openTransfer("t1", 0) + require.NoError(t, err) + _, err = f.Write([]byte("0123456789")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // Resuming from where the server actually is works. + f, err = s.openTransfer("t1", 10) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // Claiming to be further along does not. + _, err = s.openTransfer("t1", 25) + require.Error(t, err) + assert.Contains(t, err.Error(), "at 10 bytes") + + // Neither does claiming to be behind, which would duplicate bytes. + _, err = s.openTransfer("t1", 4) + require.Error(t, err) +} + +// Appending is what makes a resumed upload continue rather than overwrite. +func TestOpenTransferAppends(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + f, err := s.openTransfer("t1", 0) + require.NoError(t, err) + _, err = f.Write([]byte("abc")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = s.openTransfer("t1", 3) + require.NoError(t, err) + _, err = f.Write([]byte("def")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + path, err := s.transferPath("t1") + require.NoError(t, err) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "abcdef", string(got)) +} + +func TestStagingRoundTrips(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("compressed"), 0600)) + + want := &stagedSnapshot{Disk: "data", ImageSize: 4096, CompressedSize: 10, Checksum: "abc123"} + require.NoError(t, s.saveStaging("t1", want)) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, *want, *got) +} + +// Bytes left behind by a server that died mid-compression have no metadata, and +// must not be mistaken for a snapshot that is ready to send. +func TestStagingWithoutMetadataIsNotUsable(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("half a snapshot"), 0600)) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + assert.Nil(t, got) +} + +// A file that does not match what the metadata says it should be cannot be +// resumed from: the offsets the client is working with would be meaningless. +func TestStagingThatDoesNotMatchItsMetadataIsDiscarded(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("short"), 0600)) + require.NoError(t, s.saveStaging("t1", &stagedSnapshot{CompressedSize: 999})) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + assert.Nil(t, got) + + // And it is gone, rather than left to be reconsidered next time. + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSweepRemovesAbandonedTransfersAndKeepsFreshOnes(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + dir := filepath.Join(s.diskDataPath(), transfersDir) + require.NoError(t, os.MkdirAll(dir, 0700)) + + stale := filepath.Join(dir, "stale.part") + fresh := filepath.Join(dir, "fresh.part") + require.NoError(t, os.WriteFile(stale, []byte("old"), 0600)) + require.NoError(t, os.WriteFile(fresh, []byte("new"), 0600)) + + old := time.Now().Add(-transferTTL - time.Hour) + require.NoError(t, os.Chtimes(stale, old, old)) + + s.sweepTransfers() + + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err), "a transfer nobody came back for should be reclaimed") + assert.FileExists(t, fresh, "a transfer still within its window must survive") +} + +// The lock is what stops two calls carrying the same id from interleaving their +// appends or truncating each other's staging. +func TestTransferLockSerializesTheSameId(t *testing.T) { + locks := newKeyedLocks() + + release := locks.acquire("t1") + + entered := make(chan struct{}) + go func() { + r := locks.acquire("t1") + close(entered) + r() + }() + + select { + case <-entered: + t.Fatal("a second holder of the same id must wait") + case <-time.After(50 * time.Millisecond): + } + + release() + + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("releasing the id should have let the waiter through") + } +} + +// Different transfers must not queue behind each other; two operators backing +// up different disks are not related. +func TestTransferLockDoesNotSerializeDifferentIds(t *testing.T) { + locks := newKeyedLocks() + + release := locks.acquire("t1") + defer release() + + done := make(chan struct{}) + go func() { + r := locks.acquire("t2") + r() + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a different transfer id should not have blocked") + } +} + +// Reference counting keeps the map from growing by one entry per backup, and +// must not drop an entry another caller is still holding. +func TestTransferLockReleasesItsBookkeeping(t *testing.T) { + locks := newKeyedLocks() + + for range 50 { + locks.acquire("t1")() + } + + locks.mu.Lock() + n := len(locks.locks) + locks.mu.Unlock() + assert.Zero(t, n, "a fully released id should leave no entry behind") +} + +// The sweep has to take both halves of a staged backup. Taking only the bytes +// would leave the metadata behind forever, since nothing looks at a metadata +// file whose .part is gone. +func TestSweepRemovesStagingMetadataToo(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + dir := filepath.Join(s.diskDataPath(), transfersDir) + require.NoError(t, os.MkdirAll(dir, 0700)) + + part := filepath.Join(dir, "stale.part") + meta := filepath.Join(dir, "stale.meta") + require.NoError(t, os.WriteFile(part, []byte("bytes"), 0600)) + require.NoError(t, os.WriteFile(meta, []byte("{}"), 0600)) + + old := time.Now().Add(-transferTTL - time.Hour) + for _, p := range []string{part, meta} { + require.NoError(t, os.Chtimes(p, old, old)) + } + + s.sweepTransfers() + + for _, p := range []string{part, meta} { + _, err := os.Stat(p) + assert.True(t, os.IsNotExist(err), "%s should have been reclaimed", p) + } +} diff --git a/servers/disk/undelete.go b/servers/disk/undelete.go new file mode 100644 index 000000000..00ac5a562 --- /dev/null +++ b/servers/disk/undelete.go @@ -0,0 +1,320 @@ +package disk + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/snapshot" +) + +// diskDataPath is where volume directories and the soft-delete holding area +// live under the server's data directory. +func (s *Server) diskDataPath() string { + return filepath.Join(s.dataPath, "disk-data") +} + +// ListDeleted reports the disks still recoverable from the soft-delete holding +// area. +func (s *Server) ListDeleted(ctx context.Context, state *disk_v1alpha.DiskBackupListDeleted) error { + disks, retention, err := s.listDeleted() + if err != nil { + return err + } + state.Results().SetDisks(disks) + state.Results().SetRetentionDays(int32(retention)) + return nil +} + +func (s *Server) listDeleted() ([]*disk_v1alpha.DeletedDisk, int, error) { + entries, err := diskio.ListDeletedVolumes(s.diskDataPath()) + if err != nil { + return nil, 0, fmt.Errorf("listing deleted volumes: %w", err) + } + + // Newest deletion first: the disk someone wants back is almost always the + // one they just lost. + sort.Slice(entries, func(i, j int) bool { + return entries[i].Metadata.DeletedAt.After(entries[j].Metadata.DeletedAt) + }) + + retention := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + + disks := make([]*disk_v1alpha.DeletedDisk, 0, len(entries)) + for _, e := range entries { + meta := e.Metadata + + d := new(disk_v1alpha.DeletedDisk) + d.SetDiskName(meta.DiskName) + d.SetVolumeId(meta.VolumeID) + d.SetSizeGb(meta.SizeGb) + d.SetFilesystem(meta.Filesystem) + d.SetVolumeMode(meta.VolumeMode) + if ts := timestamp(meta.DeletedAt); ts != nil { + d.SetDeletedAt(ts) + } + expiry := meta.DeletedAt.Add(time.Duration(retention) * 24 * time.Hour) + if ts := timestamp(expiry); ts != nil { + d.SetExpiresAt(ts) + } + disks = append(disks, d) + } + + return disks, retention, nil +} + +// Undelete moves a deleted volume's data back into place and recreates the +// entities that point at it. +func (s *Server) Undelete(ctx context.Context, state *disk_v1alpha.DiskBackupUndelete) error { + args := state.Args() + + out, err := s.undelete(ctx, args.Disk(), args.VolumeId()) + if err != nil { + return err + } + + res := new(disk_v1alpha.UndeleteResult) + res.SetDisk(out.name) + res.SetDiskId(string(out.diskID)) + res.SetVolumeId(out.volumeID) + res.SetImagePath(out.imagePath) + state.Results().SetResult(&res) + return nil +} + +// undeleteResult is what a recovery produced, for the caller to report. +type undeleteResult struct { + name string + diskID entity.Id + volumeID string + imagePath string +} + +func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undeleteResult, retErr error) { + if name == "" { + return nil, refuse("disk name is required") + } + + // The name check below and the create that follows it are two steps, so two + // recoveries of the same name could both find it free and both take it, + // leaving two disks answering to one name and every lookup by that name + // ambiguous from then on. + // + // Process-local is the right granularity here rather than a shortcut: disk + // volumes are pinned to the coordinator, so one process serves every + // recovery on a cluster. Making this safe across processes would mean a + // conditional unique-name create in the entity store, which is worth doing + // if disks ever schedule anywhere else. + releaseName := s.names.acquire(name) + defer releaseName() + + entry, err := s.findDeleted(name, volumeID) + if err != nil { + return nil, err + } + meta := entry.Metadata + + // A live disk already owning this name would end up with two entities + // answering to it, and every lookup by name is ambiguous from then on. + // + // Only an actual absence clears the way. A lookup that merely failed says + // nothing about whether the name is taken, and treating it as free is how + // a moment of entity-store trouble becomes the duplicate this check exists + // to prevent. + switch _, err := s.disks.FindDisk(ctx, name); { + case err == nil: + return nil, refuse("a disk named %q already exists — rename or delete it before recovering this one", name) + case !errors.As(err, &snapshot.DiskNotFoundError{}): + return nil, fmt.Errorf("checking whether %q is already taken: %w", name, err) + } + + filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + + diskEntityId, err := s.createRestoringDisk(ctx, meta, filesystem) + if err != nil { + return nil, err + } + + // From here on there are entities to unwind, and unwinding has to survive + // the thing that made it necessary. A client that disconnects mid-recovery + // cancels ctx, which is exactly when the cleanup below must still run. + cleanupCtx := context.WithoutCancel(ctx) + + volID := meta.VolumeID + destPath := filepath.Join(s.diskDataPath(), "volumes", volID) + + // The runner creates this on startup, but a recovery can be the first thing + // that happens on a rebuilt host, and rename will not create it. + if err := os.MkdirAll(filepath.Dir(destPath), 0700); err != nil { + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") + return nil, fmt.Errorf("creating volumes directory: %w", err) + } + + if err := os.Rename(entry.Path, destPath); err != nil { + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") + return nil, fmt.Errorf("moving volume back to %s: %w", destPath, err) + } + + // Until the last entity write lands, put everything back: the data returns + // to the holding area with its metadata intact, so a failed recovery can be + // retried rather than leaving a volume directory nothing references. + committed := false + defer func() { + if committed { + return + } + if rerr := os.Rename(destPath, entry.Path); rerr != nil { + s.log.Warn("failed to move volume back to deleted-volumes", "volume_id", volID, "error", rerr) + } + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") + }() + + imagePath := filepath.Join(destPath, "disk.img") + if _, err := os.Stat(imagePath); err != nil { + return nil, refuse("disk image not found at %s — the deleted volume may be corrupted", imagePath) + } + + nodeId, err := s.disks.FindNodeId(ctx) + if err != nil { + s.log.Warn("could not find node, using the one recorded at deletion", + "disk", name, "node_id", meta.NodeID, "error", err) + nodeId = meta.NodeID.Id() + } + + volEntityId := entity.Id("disk_volume/" + volID) + vol := &storage_v1alpha.DiskVolume{ + Name: meta.DiskName, + DiskId: diskEntityId, + VolumeId: volID, + SizeGb: meta.SizeGb, + Filesystem: filesystem, + VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY. The runner's DiskVolumeController drives + // the recovered volume through the same mount-then-READY ordering as a + // fresh create (skipping the reimage, since disk.img is already in + // place). Advertising DV_READY here would let a lease bind against a + // volume that isn't registered in the runner's in-memory state yet, + // which lands the lease in a terminal FAILED (MIR-1469). + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, + } + + if _, err := s.eac.Create(ctx, entity.New(entity.DBId, volEntityId, vol.Encode).Attrs()); err != nil { + return nil, fmt.Errorf("creating disk_volume entity: %w", err) + } + + // PROVISIONING, not PROVISIONED. The DiskController promotes it only once + // the disk_volume actually reaches DV_READY — after the volume is + // registered and mounted on the node — so the disk never reports itself + // leasable ahead of a real mount. + _, err = s.eac.Patch(ctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volID), + }, 0) + if err != nil { + s.deleteEntity(cleanupCtx, string(volEntityId), "disk_volume") + return nil, fmt.Errorf("updating disk to provisioning: %w", err) + } + + // Committed. The metadata file only exists to describe a volume while it + // sits in the holding area, so it goes now that the volume is live again. + if rerr := os.Remove(filepath.Join(destPath, "metadata.json")); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove deleted-volume metadata", "volume_id", volID, "error", rerr) + } + committed = true + + s.log.Info("recovered deleted disk", + "disk", name, "disk_id", diskEntityId, "volume_id", volID) + + return &undeleteResult{ + name: name, + diskID: diskEntityId, + volumeID: volID, + imagePath: imagePath, + }, nil +} + +// findDeleted picks the one deleted volume the caller meant. +func (s *Server) findDeleted(name, volumeID string) (*diskio.DeletedVolumeEntry, error) { + entries, err := diskio.ListDeletedVolumes(s.diskDataPath()) + if err != nil { + return nil, fmt.Errorf("listing deleted volumes: %w", err) + } + + var matches []diskio.DeletedVolumeEntry + for _, e := range entries { + if e.Metadata.DiskName != name { + continue + } + if volumeID != "" && e.Metadata.VolumeID != volumeID { + continue + } + matches = append(matches, e) + } + + switch len(matches) { + case 0: + if volumeID != "" { + return nil, refuse("no deleted disk named %q with volume %s", name, volumeID) + } + return nil, refuse("no deleted disk found named %q", name) + case 1: + return &matches[0], nil + } + + ids := make([]string, 0, len(matches)) + for _, m := range matches { + ids = append(ids, m.Metadata.VolumeID) + } + return nil, refuse( + "several deleted disks are named %q — pass one of these volume ids to say which: %s", + name, strings.Join(ids, ", "), + ) +} + +// createRestoringDisk makes the disk entity in RESTORING so the disk controller +// leaves it alone while the volume is moved back into place. +func (s *Server) createRestoringDisk( + ctx context.Context, + meta *diskio.DeletedVolumeMetadata, + filesystem string, +) (entity.Id, error) { + disk := &storage_v1alpha.Disk{ + Name: meta.DiskName, + SizeGb: meta.SizeGb, + Filesystem: diskresolve.ParseFilesystem(filesystem), + Status: storage_v1alpha.RESTORING, + } + + // A fresh id, not the one the disk had before it was deleted. Anything that + // still holds a reference to the old id is holding a reference to something + // the operator deleted, and reusing the id would silently reconnect it. + id, err := s.ec.Create(ctx, idgen.GenNS("disk"), disk) + if err != nil { + return "", fmt.Errorf("creating disk entity: %w", err) + } + return id, nil +} + +// deleteEntity is best-effort cleanup on a path that is already failing, so it +// reports rather than returns: the error that got us here is the one to keep. +func (s *Server) deleteEntity(ctx context.Context, id, kind string) { + if _, err := s.eac.Delete(ctx, id); err != nil { + s.log.Warn("failed to clean up entity after a failed recovery", + "kind", kind, "id", id, "error", err) + } +} diff --git a/servers/disk/undelete_test.go b/servers/disk/undelete_test.go new file mode 100644 index 000000000..780afb561 --- /dev/null +++ b/servers/disk/undelete_test.go @@ -0,0 +1,284 @@ +package disk + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + compute "miren.dev/runtime/api/compute/compute_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" +) + +// newUndeleteServer wires a server over a real in-memory entity store, since +// undelete's whole job is entity writes and a fake would not exercise them. +func newUndeleteServer(t *testing.T) (*Server, *testutils.InMemEntityServer, string) { + t.Helper() + + es, cleanup := testutils.NewInMemEntityServer(t) + t.Cleanup(cleanup) + + // One coordinator node, so FindNodeId resolves rather than falling back to + // whatever was recorded at deletion. + _, err := es.EAC.Create(context.Background(), entity.New( + entity.DBId, entity.Id("node/n1"), + entity.Ref(entity.EntityKind, compute.KindNode), + ).Attrs()) + require.NoError(t, err) + + dataPath := t.TempDir() + ec := entityserver.NewClient(testutils.TestLogger(t), es.EAC) + + s := &Server{ + log: slog.Default(), + disks: diskresolve.New(es.EAC, ec), + dataPath: dataPath, + eac: es.EAC, + ec: ec, + mntOps: fakeMountOps{}, + + transfers: newKeyedLocks(), + names: newKeyedLocks(), + } + return s, es, dataPath +} + +// seedDeleted puts a volume into the soft-delete holding area the way +// deleteVolume does: the directory, its image, and the metadata describing it. +func seedDeleted(t *testing.T, dataPath, diskName, volumeID string, deletedAt time.Time) string { + t.Helper() + + diskData := filepath.Join(dataPath, "disk-data") + dir := filepath.Join(diskio.DeletedVolumesPath(diskData), volumeID) + require.NoError(t, os.MkdirAll(dir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "disk.img"), []byte("disk contents"), 0644)) + + require.NoError(t, diskio.SaveDeletedVolumeMetadata(dir, &diskio.DeletedVolumeMetadata{ + DiskID: "disk/old-" + volumeID, + DiskName: diskName, + SizeGb: 1, + Filesystem: "ext4", + VolumeID: volumeID, + VolumeMode: string(storage_v1alpha.VM_UNIVERSAL), + NodeID: compute.NodeId("node/n1"), + DeletedAt: deletedAt, + })) + return dir +} + +func TestListDeletedIsEmptyWhenNothingWasDeleted(t *testing.T) { + s, _, _ := newUndeleteServer(t) + + disks, retention, err := s.listDeleted() + require.NoError(t, err) + assert.Empty(t, disks) + assert.Positive(t, retention, "the client shows an expiry, so retention has to come back") +} + +func TestListDeletedReportsNewestFirstWithAnExpiry(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + + older := time.Now().Add(-48 * time.Hour).Truncate(time.Second) + newer := time.Now().Add(-1 * time.Hour).Truncate(time.Second) + seedDeleted(t, dataPath, "old-disk", "vol-old", older) + seedDeleted(t, dataPath, "new-disk", "vol-new", newer) + + disks, retention, err := s.listDeleted() + require.NoError(t, err) + require.Len(t, disks, 2) + + // The disk someone wants back is almost always the one they just lost. + assert.Equal(t, "new-disk", disks[0].DiskName()) + assert.Equal(t, "old-disk", disks[1].DiskName()) + + assert.Equal(t, "vol-new", disks[0].VolumeId()) + assert.Equal(t, int64(1), disks[0].SizeGb()) + assert.Equal(t, "ext4", disks[0].Filesystem()) + + require.True(t, disks[0].HasDeletedAt()) + assert.Equal(t, newer.Unix(), disks[0].DeletedAt().Seconds()) + + // Expiry is what tells an operator how long they have to act on this. + require.True(t, disks[0].HasExpiresAt()) + wantExpiry := newer.Add(time.Duration(retention) * 24 * time.Hour) + assert.Equal(t, wantExpiry.Unix(), disks[0].ExpiresAt().Seconds()) +} + +func TestUndeleteMovesTheVolumeBackAndRecreatesEntities(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + holding := seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now().Add(-time.Hour)) + + res, err := s.undelete(context.Background(), "mydisk", "") + require.NoError(t, err) + + // The data is live again, and gone from the holding area. + imagePath := filepath.Join(dataPath, "disk-data", "volumes", "vol-1", "disk.img") + assert.FileExists(t, imagePath) + assert.NoDirExists(t, holding) + assert.Equal(t, imagePath, res.imagePath) + + // The metadata only described the volume while it sat in the holding area. + _, statErr := os.Stat(filepath.Join(filepath.Dir(imagePath), "metadata.json")) + assert.True(t, os.IsNotExist(statErr), "metadata.json should be gone once the volume is live") + + disks := listDisks(t, es) + require.Len(t, disks, 1) + // PROVISIONING, not PROVISIONED: the DiskController promotes it only once + // the volume is actually mounted, so it never reports itself leasable early. + assert.Equal(t, storage_v1alpha.PROVISIONING, disks[0].Status) + assert.Equal(t, "vol-1", disks[0].VolumeId) + + vols := listVolumes(t, es) + require.Len(t, vols, 1) + assert.Equal(t, storage_v1alpha.DV_PENDING, vols[0].ActualState) + assert.Equal(t, storage_v1alpha.DV_PRESENT, vols[0].DesiredState) + assert.Equal(t, imagePath, vols[0].ImagePath) + assert.Equal(t, entity.Id("node/n1"), vols[0].NodeId) +} + +// The recovered disk must not reuse the id of the disk that was deleted, or +// anything still pointing at that id gets silently reconnected. +func TestUndeleteGivesTheDiskAFreshId(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.NoError(t, err) + + disks := listDisks(t, es) + require.Len(t, disks, 1) + assert.NotEqual(t, entity.Id("disk/old-vol-1"), disks[0].ID) +} + +func TestUndeleteReportsAnUnknownName(t *testing.T) { + s, _, _ := newUndeleteServer(t) + + _, err := s.undelete(context.Background(), "nope", "") + require.Error(t, err) + assert.Contains(t, err.Error(), `no deleted disk found named "nope"`) +} + +// Two deletions of the same name are ordinary — delete, recreate, delete again +// — so the ambiguity has to be reported with the ids needed to resolve it. +func TestUndeleteNamesTheChoicesWhenSeveralShareAName(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now().Add(-2*time.Hour)) + seedDeleted(t, dataPath, "mydisk", "vol-2", time.Now().Add(-time.Hour)) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "vol-1") + assert.Contains(t, err.Error(), "vol-2") + + // Naming one resolves it. + res, err := s.undelete(context.Background(), "mydisk", "vol-2") + require.NoError(t, err) + assert.Equal(t, "vol-2", res.volumeID) +} + +func TestUndeleteRefusesWhenALiveDiskHoldsTheName(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + // A live disk already answering to the name. + _, err := s.ec.Create(context.Background(), "disk/live", &storage_v1alpha.Disk{ + Name: "mydisk", + SizeGb: 1, + Filesystem: storage_v1alpha.EXT4, + Status: storage_v1alpha.PROVISIONED, + }) + require.NoError(t, err) + + _, err = s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") +} + +// A holding-area directory with no image is corrupt. Recovery must roll back +// rather than leave entities pointing at data that is not there. +func TestUndeleteRollsBackWhenTheImageIsMissing(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + holding := seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + require.NoError(t, os.Remove(filepath.Join(holding, "disk.img"))) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "may be corrupted") + + // The data went back to the holding area, so the operator can try again. + assert.DirExists(t, holding) + assert.NoDirExists(t, filepath.Join(dataPath, "disk-data", "volumes", "vol-1")) + + // And no disk entity was left behind. + assert.Empty(t, listDisks(t, es)) +} + +func listDisks(t *testing.T, es *testutils.InMemEntityServer) []storage_v1alpha.Disk { + t.Helper() + resp, err := es.EAC.List(context.Background(), entity.Ref(entity.EntityKind, storage_v1alpha.KindDisk)) + require.NoError(t, err) + + var out []storage_v1alpha.Disk + for _, v := range resp.Values() { + var d storage_v1alpha.Disk + d.Decode(v.Entity()) + out = append(out, d) + } + return out +} + +func listVolumes(t *testing.T, es *testutils.InMemEntityServer) []storage_v1alpha.DiskVolume { + t.Helper() + resp, err := es.EAC.List(context.Background(), entity.Ref(entity.EntityKind, storage_v1alpha.KindDiskVolume)) + require.NoError(t, err) + + var out []storage_v1alpha.DiskVolume + for _, v := range resp.Values() { + var vol storage_v1alpha.DiskVolume + vol.Decode(v.Entity()) + out = append(out, vol) + } + return out +} + +// The name check and the create that follows it are two steps, so two +// recoveries of the same name must not both find it free. Two disks answering +// to one name make every lookup by that name ambiguous from then on. +func TestUndeleteSerializesRecoveriesOfTheSameName(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + var wg sync.WaitGroup + errs := make([]error, 2) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[i] = s.undelete(context.Background(), "mydisk", "") + }() + } + wg.Wait() + + // One recovers it; the other finds nothing left in the holding area, or + // finds the name taken. Either way it must not create a second disk. + succeeded := 0 + for _, err := range errs { + if err == nil { + succeeded++ + } + } + assert.Equal(t, 1, succeeded, "exactly one recovery should have succeeded") + + disks := listDisks(t, es) + assert.Len(t, disks, 1, "a name must never end up on two disks") +}