From 8a23135dcdfb70ba9e9bb360faa79f76d91c26ab Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:11:33 -0700 Subject: [PATCH 01/19] Move the entity disk resolver into pkg/diskresolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resolver that turns a disk name into a volume and an image path lived in cli/commands, so only the CLI could use it. RFD 108 moves backup and restore onto the server, which needs the same lookups. Nothing about the resolver was CLI-specific — its imports are all api/ and pkg/ — so this is a straight move with the identifiers exported. FindNodeId is exported too, since disk undelete calls it directly. No behavior change. --- cli/commands/disk_backup.go | 3 +- cli/commands/disk_restore.go | 3 +- cli/commands/disk_undelete.go | 5 ++-- .../diskresolve/resolver.go | 28 +++++++++---------- .../diskresolve/resolver_test.go | 12 ++++---- 5 files changed, 27 insertions(+), 24 deletions(-) rename cli/commands/disk_resolver.go => pkg/diskresolve/resolver.go (89%) rename cli/commands/disk_resolver_test.go => pkg/diskresolve/resolver_test.go (97%) diff --git a/cli/commands/disk_backup.go b/cli/commands/disk_backup.go index 86901c8c4..646063d2f 100644 --- a/cli/commands/disk_backup.go +++ b/cli/commands/disk_backup.go @@ -10,6 +10,7 @@ import ( "miren.dev/runtime/components/coordinate" "miren.dev/runtime/components/diskio" "miren.dev/runtime/pkg/cloudauth" + "miren.dev/runtime/pkg/diskresolve" "miren.dev/runtime/pkg/registration" "miren.dev/runtime/pkg/snapshot" ) @@ -33,7 +34,7 @@ func DiskBackup(ctx *Context, opts struct { } eac := entityserver_v1alpha.NewEntityAccessClient(client) - resolver := newEntityDiskResolver(eac, nil) + resolver := diskresolve.New(eac, nil) target, err := snapshot.PrepareBackup(ctx, resolver, opts.Name, opts.DataPath) if err != nil { diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index 6c9f90ae4..89dd9d15f 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -8,6 +8,7 @@ import ( "miren.dev/runtime/api/entityserver" "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/pkg/diskresolve" "miren.dev/runtime/pkg/snapshot" ) @@ -54,7 +55,7 @@ func DiskRestore(ctx *Context, opts struct { eac := entityserver_v1alpha.NewEntityAccessClient(client) ec := entityserver.NewClient(ctx.Log, eac) - resolver := newEntityDiskResolver(eac, ec) + resolver := diskresolve.New(eac, ec) target, err := snapshot.PrepareRestore(ctx, resolver, diskName, opts.DataPath, snapshot.WithCreator(resolver, meta.SizeBytes, meta.Filesystem), diff --git a/cli/commands/disk_undelete.go b/cli/commands/disk_undelete.go index e40674848..47f853c2e 100644 --- a/cli/commands/disk_undelete.go +++ b/cli/commands/disk_undelete.go @@ -12,6 +12,7 @@ import ( "miren.dev/runtime/api/entityserver/entityserver_v1alpha" "miren.dev/runtime/api/storage/storage_v1alpha" "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" ) @@ -73,7 +74,7 @@ func DiskUndelete(ctx *Context, opts struct { eac := entityserver_v1alpha.NewEntityAccessClient(client) ec := entityserver.NewClient(ctx.Log, eac) - resolver := newEntityDiskResolver(eac, ec) + resolver := diskresolve.New(eac, ec) // Check if a disk with this name already exists if _, err := resolver.FindDisk(context.Background(), meta.DiskName); err == nil { @@ -140,7 +141,7 @@ func DiskUndelete(ctx *Context, opts struct { }() // Find the node ID - nodeId, err := resolver.findNodeId(context.Background()) + nodeId, err := resolver.FindNodeId(context.Background()) if err != nil { ctx.Warn("Failed to find node ID, using stored value: %v", err) nodeId = meta.NodeID.Id() diff --git a/cli/commands/disk_resolver.go b/pkg/diskresolve/resolver.go similarity index 89% rename from cli/commands/disk_resolver.go rename to pkg/diskresolve/resolver.go index 63533b40a..85da552c1 100644 --- a/cli/commands/disk_resolver.go +++ b/pkg/diskresolve/resolver.go @@ -1,4 +1,4 @@ -package commands +package diskresolve import ( "context" @@ -17,18 +17,18 @@ import ( "miren.dev/runtime/pkg/snapshot" ) -// entityDiskResolver implements snapshot.DiskResolver using the entity +// Resolver implements snapshot.DiskResolver using the entity // access RPC client. -type entityDiskResolver struct { +type Resolver struct { eac *entityserver_v1alpha.EntityAccessClient ec *entityserver.Client } -func newEntityDiskResolver(eac *entityserver_v1alpha.EntityAccessClient, ec *entityserver.Client) *entityDiskResolver { - return &entityDiskResolver{eac: eac, ec: ec} +func New(eac *entityserver_v1alpha.EntityAccessClient, ec *entityserver.Client) *Resolver { + return &Resolver{eac: eac, ec: ec} } -func (r *entityDiskResolver) FindDisk(ctx context.Context, name string) (*snapshot.DiskState, error) { +func (r *Resolver) FindDisk(ctx context.Context, name string) (*snapshot.DiskState, error) { ref := entity.Ref(entity.EntityKind, storage_v1alpha.KindDisk) results, err := r.eac.List(ctx, ref) if err != nil { @@ -59,7 +59,7 @@ func (r *entityDiskResolver) FindDisk(ctx context.Context, name string) (*snapsh } } -func (r *entityDiskResolver) FindVolume(ctx context.Context, diskID string) (*snapshot.VolumeState, error) { +func (r *Resolver) FindVolume(ctx context.Context, diskID string) (*snapshot.VolumeState, error) { resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskVolumeDiskIdId, entity.Id(diskID))) if err != nil { return nil, fmt.Errorf("listing disk volumes: %w", err) @@ -88,7 +88,7 @@ func (r *entityDiskResolver) FindVolume(ctx context.Context, diskID string) (*sn // controller ignores it while restore writes the image. The returned // RestoreTarget includes a Finalize callback that creates the disk_volume // entity and transitions the disk to PROVISIONED. -func (r *entityDiskResolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { +func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { sizeGb := sizeBytes / (1 << 30) if sizeGb == 0 { sizeGb = 1 @@ -125,7 +125,7 @@ func (r *entityDiskResolver) CreateDiskAndVolume(ctx context.Context, name strin return nil, fmt.Errorf("creating disk entity: %w", err) } - nodeId, err := r.findNodeId(ctx) + nodeId, err := r.FindNodeId(ctx) if err != nil { return nil, fmt.Errorf("finding node: %w", err) } @@ -193,7 +193,7 @@ func (r *entityDiskResolver) CreateDiskAndVolume(ctx context.Context, name strin VolumeId: volId, SizeGb: sizeGb, Filesystem: filesystem, - VolumeMode: detectVolumeMode(), + VolumeMode: DetectVolumeMode(), DesiredState: storage_v1alpha.DV_PRESENT, ActualState: storage_v1alpha.DV_READY, ImagePath: imagePath, @@ -231,10 +231,10 @@ func (r *entityDiskResolver) CreateDiskAndVolume(ctx context.Context, name strin }, nil } -// findNodeId finds the coordinator node. Stateful sandboxes (those with +// FindNodeId finds the coordinator node. Stateful sandboxes (those with // disk volumes) run on the coordinator, so disk_volume entities must // reference it. -func (r *entityDiskResolver) findNodeId(ctx context.Context) (entity.Id, error) { +func (r *Resolver) FindNodeId(ctx context.Context) (entity.Id, error) { resp, err := r.eac.List(ctx, entity.Ref(entity.EntityKind, compute.KindNode)) if err != nil { return "", fmt.Errorf("listing nodes: %w", err) @@ -262,7 +262,7 @@ func (r *entityDiskResolver) findNodeId(ctx context.Context) (entity.Id, error) return "", fmt.Errorf("multiple nodes found but none has role=coordinator") } -func (r *entityDiskResolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.LeaseState, error) { +func (r *Resolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.LeaseState, error) { resp, err := r.eac.List(ctx, entity.Ref(storage_v1alpha.DiskLeaseDiskIdId, entity.Id(diskID))) if err != nil { return nil, fmt.Errorf("listing disk leases: %w", err) @@ -281,7 +281,7 @@ func (r *entityDiskResolver) FindLeases(ctx context.Context, diskID string) ([]s return leases, nil } -func detectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { +func DetectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { return storage_v1alpha.VM_ACCELERATOR } diff --git a/cli/commands/disk_resolver_test.go b/pkg/diskresolve/resolver_test.go similarity index 97% rename from cli/commands/disk_resolver_test.go rename to pkg/diskresolve/resolver_test.go index 88639411c..cf7f48a9b 100644 --- a/cli/commands/disk_resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -1,4 +1,4 @@ -package commands +package diskresolve import ( "context" @@ -73,17 +73,17 @@ func (c cancelAwareRPC) Call(ctx context.Context, method string, args, result an } // setupResolver builds an in-memory entity server, seeds a coordinator node so -// entityDiskResolver.findNodeId succeeds, and returns a resolver wired through +// Resolver.FindNodeId succeeds, and returns a resolver wired through // fault (or a plain eac when fault is nil). Reads in tests should use es.EAC, // which is not fault-injected. -func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, *entityDiskResolver) { +func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, *Resolver) { t.Helper() ctx := t.Context() es, cleanup := testutils.NewInMemEntityServer(t) t.Cleanup(cleanup) - // Seed a single coordinator node so entityDiskResolver.findNodeId returns one. + // Seed a single coordinator node so Resolver.FindNodeId returns one. _, err := es.Client.Create(ctx, "coordinator", &compute.Node{ApiAddress: ":8444"}) require.NoError(t, err) @@ -95,7 +95,7 @@ func setupResolver(t *testing.T, fault *faultRPC) (*testutils.InMemEntityServer, } eac := entityserver_v1alpha.NewEntityAccessClient(cli) ec := entityserver.NewClient(testutils.TestLogger(t), eac) - return es, newEntityDiskResolver(eac, ec) + return es, New(eac, ec) } func listTestDisks(t *testing.T, ctx context.Context, eac *entityserver_v1alpha.EntityAccessClient) []storage_v1alpha.Disk { @@ -355,7 +355,7 @@ func TestCreateDiskAndVolume_CleanupRunsAfterCancellation(t *testing.T) { // The resolver talks through a client that honours cancellation, so a // cleanup that inherited the dead context would fail its Patch. eac := entityserver_v1alpha.NewEntityAccessClient(cancelAwareRPC{Client: es.EAC.Client}) - resolver := newEntityDiskResolver(eac, entityserver.NewClient(testutils.TestLogger(t), eac)) + resolver := New(eac, entityserver.NewClient(testutils.TestLogger(t), eac)) target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) From 3d1f08c1721cbc93613b16eab1e351d6d6c5e8d3 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:12:23 -0700 Subject: [PATCH 02/19] Add the DiskBackup RPC schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The interface a remote client drives for RFD-108: backup, listBackups, restore. Backup takes a to_cloud flag because the two destinations behave differently in a way the caller has to know about — a cloud backup uploads with the cluster's key and sends the client no image bytes, while a cluster with no cloud streams the snapshot down to a file on the client. Progress is a union modelled on build's Status, so a multi-gigabyte transfer shows movement rather than looking like a hang, and so the mounted-disk warning has somewhere to go. RestorePoint deliberately hides the universal/accelerator split from RFD-64: a caller picks a moment in time, and reaching it by fetching a full image or by replaying log segments is the server's business. --- api/disk/disk.go | 4 + api/disk/disk_v1alpha/rpc.gen.go | 1053 ++++++++++++++++++++++++++++++ api/disk/rpc.yml | 211 ++++++ 3 files changed, 1268 insertions(+) create mode 100644 api/disk/disk.go create mode 100644 api/disk/disk_v1alpha/rpc.gen.go create mode 100644 api/disk/rpc.yml diff --git a/api/disk/disk.go b/api/disk/disk.go new file mode 100644 index 000000000..dbca1b572 --- /dev/null +++ b/api/disk/disk.go @@ -0,0 +1,4 @@ +package disk + +//go:generate mkdir -p disk_v1alpha +//go:generate go run ../../pkg/rpc/cmd/rpcgen -pkg disk_v1alpha -input rpc.yml -output disk_v1alpha/rpc.gen.go diff --git a/api/disk/disk_v1alpha/rpc.gen.go b/api/disk/disk_v1alpha/rpc.gen.go new file mode 100644 index 000000000..b1f939218 --- /dev/null +++ b/api/disk/disk_v1alpha/rpc.gen.go @@ -0,0 +1,1053 @@ +package disk_v1alpha + +import ( + "context" + "encoding/json" + "slices" + + "github.com/fxamacker/cbor/v2" + rpc "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/pkg/rpc/standard" + "miren.dev/runtime/pkg/rpc/stream" +) + +type transferData struct { + Done *int64 `cbor:"0,keyasint,omitempty" json:"done,omitempty"` + Total *int64 `cbor:"1,keyasint,omitempty" json:"total,omitempty"` + BytesPerSecond *int64 `cbor:"2,keyasint,omitempty" json:"bytes_per_second,omitempty"` + EtaSeconds *int64 `cbor:"3,keyasint,omitempty" json:"eta_seconds,omitempty"` +} + +type Transfer struct { + data transferData +} + +func (v *Transfer) HasDone() bool { + return v.data.Done != nil +} + +func (v *Transfer) Done() int64 { + if v.data.Done == nil { + return 0 + } + return *v.data.Done +} + +func (v *Transfer) SetDone(done int64) { + v.data.Done = &done +} + +func (v *Transfer) HasTotal() bool { + return v.data.Total != nil +} + +func (v *Transfer) Total() int64 { + if v.data.Total == nil { + return 0 + } + return *v.data.Total +} + +func (v *Transfer) SetTotal(total int64) { + v.data.Total = &total +} + +func (v *Transfer) HasBytesPerSecond() bool { + return v.data.BytesPerSecond != nil +} + +func (v *Transfer) BytesPerSecond() int64 { + if v.data.BytesPerSecond == nil { + return 0 + } + return *v.data.BytesPerSecond +} + +func (v *Transfer) SetBytesPerSecond(bytes_per_second int64) { + v.data.BytesPerSecond = &bytes_per_second +} + +func (v *Transfer) HasEtaSeconds() bool { + return v.data.EtaSeconds != nil +} + +func (v *Transfer) EtaSeconds() int64 { + if v.data.EtaSeconds == nil { + return 0 + } + return *v.data.EtaSeconds +} + +func (v *Transfer) SetEtaSeconds(eta_seconds int64) { + v.data.EtaSeconds = &eta_seconds +} + +func (v *Transfer) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *Transfer) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *Transfer) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *Transfer) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type ProgressUpdate interface { + Which() string + Message() string + SetMessage(string) + Transfer() *Transfer + SetTransfer(*Transfer) + Warning() string + SetWarning(string) + Error() string + SetError(string) +} + +type progressUpdate struct { + U_Message *string `cbor:"1,keyasint,omitempty" json:"message,omitempty"` + U_Transfer **Transfer `cbor:"2,keyasint,omitempty" json:"transfer,omitempty"` + U_Warning *string `cbor:"3,keyasint,omitempty" json:"warning,omitempty"` + U_Error *string `cbor:"4,keyasint,omitempty" json:"error,omitempty"` +} + +func (v *progressUpdate) Which() string { + if v.U_Message != nil { + return "message" + } + if v.U_Transfer != nil { + return "transfer" + } + if v.U_Warning != nil { + return "warning" + } + if v.U_Error != nil { + return "error" + } + return "" +} + +func (v *progressUpdate) Message() string { + if v.U_Message == nil { + return "" + } + return *v.U_Message +} + +func (v *progressUpdate) SetMessage(val string) { + v.U_Transfer = nil + v.U_Warning = nil + v.U_Error = nil + v.U_Message = &val +} + +func (v *progressUpdate) Transfer() *Transfer { + if v.U_Transfer == nil { + return nil + } + return *v.U_Transfer +} + +func (v *progressUpdate) SetTransfer(val *Transfer) { + v.U_Message = nil + v.U_Warning = nil + v.U_Error = nil + v.U_Transfer = &val +} + +func (v *progressUpdate) Warning() string { + if v.U_Warning == nil { + return "" + } + return *v.U_Warning +} + +func (v *progressUpdate) SetWarning(val string) { + v.U_Message = nil + v.U_Transfer = nil + v.U_Error = nil + v.U_Warning = &val +} + +func (v *progressUpdate) Error() string { + if v.U_Error == nil { + return "" + } + return *v.U_Error +} + +func (v *progressUpdate) SetError(val string) { + v.U_Message = nil + v.U_Transfer = nil + v.U_Warning = nil + v.U_Error = &val +} + +type progressData struct { + progressUpdate +} + +type Progress struct { + data progressData +} + +func (v *Progress) Update() ProgressUpdate { + return &v.data.progressUpdate +} + +func (v *Progress) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *Progress) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *Progress) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *Progress) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type restorePointData struct { + Id *string `cbor:"0,keyasint,omitempty" json:"id,omitempty"` + CreatedAt *standard.Timestamp `cbor:"1,keyasint,omitempty" json:"created_at,omitempty"` + SizeBytes *int64 `cbor:"2,keyasint,omitempty" json:"size_bytes,omitempty"` + ImageSizeBytes *int64 `cbor:"3,keyasint,omitempty" json:"image_size_bytes,omitempty"` + Name *string `cbor:"4,keyasint,omitempty" json:"name,omitempty"` + Mode *string `cbor:"5,keyasint,omitempty" json:"mode,omitempty"` +} + +type RestorePoint struct { + data restorePointData +} + +func (v *RestorePoint) HasId() bool { + return v.data.Id != nil +} + +func (v *RestorePoint) Id() string { + if v.data.Id == nil { + return "" + } + return *v.data.Id +} + +func (v *RestorePoint) SetId(id string) { + v.data.Id = &id +} + +func (v *RestorePoint) HasCreatedAt() bool { + return v.data.CreatedAt != nil +} + +func (v *RestorePoint) CreatedAt() *standard.Timestamp { + return v.data.CreatedAt +} + +func (v *RestorePoint) SetCreatedAt(created_at *standard.Timestamp) { + v.data.CreatedAt = created_at +} + +func (v *RestorePoint) HasSizeBytes() bool { + return v.data.SizeBytes != nil +} + +func (v *RestorePoint) SizeBytes() int64 { + if v.data.SizeBytes == nil { + return 0 + } + return *v.data.SizeBytes +} + +func (v *RestorePoint) SetSizeBytes(size_bytes int64) { + v.data.SizeBytes = &size_bytes +} + +func (v *RestorePoint) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *RestorePoint) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *RestorePoint) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *RestorePoint) HasName() bool { + return v.data.Name != nil +} + +func (v *RestorePoint) Name() string { + if v.data.Name == nil { + return "" + } + return *v.data.Name +} + +func (v *RestorePoint) SetName(name string) { + v.data.Name = &name +} + +func (v *RestorePoint) HasMode() bool { + return v.data.Mode != nil +} + +func (v *RestorePoint) Mode() string { + if v.data.Mode == nil { + return "" + } + return *v.data.Mode +} + +func (v *RestorePoint) SetMode(mode string) { + v.data.Mode = &mode +} + +func (v *RestorePoint) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RestorePoint) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RestorePoint) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RestorePoint) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type backupResultData struct { + ImageSizeBytes *int64 `cbor:"0,keyasint,omitempty" json:"image_size_bytes,omitempty"` + CompressedSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"compressed_size_bytes,omitempty"` + Checksum *string `cbor:"2,keyasint,omitempty" json:"checksum,omitempty"` + RestorePointId *string `cbor:"3,keyasint,omitempty" json:"restore_point_id,omitempty"` +} + +type BackupResult struct { + data backupResultData +} + +func (v *BackupResult) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *BackupResult) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *BackupResult) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *BackupResult) HasCompressedSizeBytes() bool { + return v.data.CompressedSizeBytes != nil +} + +func (v *BackupResult) CompressedSizeBytes() int64 { + if v.data.CompressedSizeBytes == nil { + return 0 + } + return *v.data.CompressedSizeBytes +} + +func (v *BackupResult) SetCompressedSizeBytes(compressed_size_bytes int64) { + v.data.CompressedSizeBytes = &compressed_size_bytes +} + +func (v *BackupResult) HasChecksum() bool { + return v.data.Checksum != nil +} + +func (v *BackupResult) Checksum() string { + if v.data.Checksum == nil { + return "" + } + return *v.data.Checksum +} + +func (v *BackupResult) SetChecksum(checksum string) { + v.data.Checksum = &checksum +} + +func (v *BackupResult) HasRestorePointId() bool { + return v.data.RestorePointId != nil +} + +func (v *BackupResult) RestorePointId() string { + if v.data.RestorePointId == nil { + return "" + } + return *v.data.RestorePointId +} + +func (v *BackupResult) SetRestorePointId(restore_point_id string) { + v.data.RestorePointId = &restore_point_id +} + +func (v *BackupResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *BackupResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *BackupResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *BackupResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type restoreResultData struct { + DiskId *string `cbor:"0,keyasint,omitempty" json:"disk_id,omitempty"` + ImageSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"image_size_bytes,omitempty"` + Created *bool `cbor:"2,keyasint,omitempty" json:"created,omitempty"` +} + +type RestoreResult struct { + data restoreResultData +} + +func (v *RestoreResult) HasDiskId() bool { + return v.data.DiskId != nil +} + +func (v *RestoreResult) DiskId() string { + if v.data.DiskId == nil { + return "" + } + return *v.data.DiskId +} + +func (v *RestoreResult) SetDiskId(disk_id string) { + v.data.DiskId = &disk_id +} + +func (v *RestoreResult) HasImageSizeBytes() bool { + return v.data.ImageSizeBytes != nil +} + +func (v *RestoreResult) ImageSizeBytes() int64 { + if v.data.ImageSizeBytes == nil { + return 0 + } + return *v.data.ImageSizeBytes +} + +func (v *RestoreResult) SetImageSizeBytes(image_size_bytes int64) { + v.data.ImageSizeBytes = &image_size_bytes +} + +func (v *RestoreResult) HasCreated() bool { + return v.data.Created != nil +} + +func (v *RestoreResult) Created() bool { + if v.data.Created == nil { + return false + } + return *v.data.Created +} + +func (v *RestoreResult) SetCreated(created bool) { + v.data.Created = &created +} + +func (v *RestoreResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RestoreResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RestoreResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RestoreResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupBackupArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + ToCloud *bool `cbor:"1,keyasint,omitempty" json:"to_cloud,omitempty"` + Pin *string `cbor:"2,keyasint,omitempty" json:"pin,omitempty"` + Data *rpc.Capability `cbor:"3,keyasint,omitempty" json:"data,omitempty"` + Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` +} + +type DiskBackupBackupArgs struct { + call rpc.Call + data diskBackupBackupArgsData +} + +func (v *DiskBackupBackupArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupBackupArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupBackupArgs) HasToCloud() bool { + return v.data.ToCloud != nil +} + +func (v *DiskBackupBackupArgs) ToCloud() bool { + if v.data.ToCloud == nil { + return false + } + return *v.data.ToCloud +} + +func (v *DiskBackupBackupArgs) HasPin() bool { + return v.data.Pin != nil +} + +func (v *DiskBackupBackupArgs) Pin() string { + if v.data.Pin == nil { + return "" + } + return *v.data.Pin +} + +func (v *DiskBackupBackupArgs) HasData() bool { + return v.data.Data != nil +} + +func (v *DiskBackupBackupArgs) Data() *stream.SendStreamClient[[]byte] { + if v.data.Data == nil { + return nil + } + return &stream.SendStreamClient[[]byte]{Client: v.call.NewClient(v.data.Data)} +} + +func (v *DiskBackupBackupArgs) HasProgress() bool { + return v.data.Progress != nil +} + +func (v *DiskBackupBackupArgs) Progress() *stream.SendStreamClient[*Progress] { + if v.data.Progress == nil { + return nil + } + return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} +} + +func (v *DiskBackupBackupArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupBackupArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupBackupArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupBackupArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupBackupResultsData struct { + Result **BackupResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupBackupResults struct { + call rpc.Call + data diskBackupBackupResultsData +} + +func (v *DiskBackupBackupResults) SetResult(result **BackupResult) { + v.data.Result = result +} + +func (v *DiskBackupBackupResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupBackupResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupBackupResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupBackupResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListBackupsArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` +} + +type DiskBackupListBackupsArgs struct { + call rpc.Call + data diskBackupListBackupsArgsData +} + +func (v *DiskBackupListBackupsArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupListBackupsArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupListBackupsArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListBackupsArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListBackupsArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListBackupsArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListBackupsResultsData struct { + Points *[]*RestorePoint `cbor:"0,keyasint,omitempty" json:"points,omitempty"` +} + +type DiskBackupListBackupsResults struct { + call rpc.Call + data diskBackupListBackupsResultsData +} + +func (v *DiskBackupListBackupsResults) SetPoints(points []*RestorePoint) { + x := slices.Clone(points) + v.data.Points = &x +} + +func (v *DiskBackupListBackupsResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListBackupsResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListBackupsResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListBackupsResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupRestoreArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + RestorePoint *string `cbor:"1,keyasint,omitempty" json:"restore_point,omitempty"` + Data *rpc.Capability `cbor:"2,keyasint,omitempty" json:"data,omitempty"` + Force *bool `cbor:"3,keyasint,omitempty" json:"force,omitempty"` + Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` +} + +type DiskBackupRestoreArgs struct { + call rpc.Call + data diskBackupRestoreArgsData +} + +func (v *DiskBackupRestoreArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupRestoreArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupRestoreArgs) HasRestorePoint() bool { + return v.data.RestorePoint != nil +} + +func (v *DiskBackupRestoreArgs) RestorePoint() string { + if v.data.RestorePoint == nil { + return "" + } + return *v.data.RestorePoint +} + +func (v *DiskBackupRestoreArgs) HasData() bool { + return v.data.Data != nil +} + +func (v *DiskBackupRestoreArgs) Data() *stream.RecvStreamClient[[]byte] { + if v.data.Data == nil { + return nil + } + return &stream.RecvStreamClient[[]byte]{Client: v.call.NewClient(v.data.Data)} +} + +func (v *DiskBackupRestoreArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *DiskBackupRestoreArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *DiskBackupRestoreArgs) HasProgress() bool { + return v.data.Progress != nil +} + +func (v *DiskBackupRestoreArgs) Progress() *stream.SendStreamClient[*Progress] { + if v.data.Progress == nil { + return nil + } + return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} +} + +func (v *DiskBackupRestoreArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupRestoreArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupRestoreArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupRestoreArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupRestoreResultsData struct { + Result **RestoreResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupRestoreResults struct { + call rpc.Call + data diskBackupRestoreResultsData +} + +func (v *DiskBackupRestoreResults) SetResult(result **RestoreResult) { + v.data.Result = result +} + +func (v *DiskBackupRestoreResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupRestoreResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupRestoreResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupRestoreResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type DiskBackupBackup struct { + rpc.Call + args DiskBackupBackupArgs + results DiskBackupBackupResults +} + +func (t *DiskBackupBackup) Args() *DiskBackupBackupArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupBackup) Results() *DiskBackupBackupResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupListBackups struct { + rpc.Call + args DiskBackupListBackupsArgs + results DiskBackupListBackupsResults +} + +func (t *DiskBackupListBackups) Args() *DiskBackupListBackupsArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupListBackups) Results() *DiskBackupListBackupsResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupRestore struct { + rpc.Call + args DiskBackupRestoreArgs + results DiskBackupRestoreResults +} + +func (t *DiskBackupRestore) Args() *DiskBackupRestoreArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupRestore) Results() *DiskBackupRestoreResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackup interface { + Backup(ctx context.Context, state *DiskBackupBackup) error + ListBackups(ctx context.Context, state *DiskBackupListBackups) error + Restore(ctx context.Context, state *DiskBackupRestore) error +} + +type reexportDiskBackup struct { + client rpc.Client +} + +func (reexportDiskBackup) Backup(ctx context.Context, state *DiskBackupBackup) error { + panic("not implemented") +} + +func (reexportDiskBackup) ListBackups(ctx context.Context, state *DiskBackupListBackups) error { + panic("not implemented") +} + +func (reexportDiskBackup) Restore(ctx context.Context, state *DiskBackupRestore) error { + panic("not implemented") +} + +func (t reexportDiskBackup) CapabilityClient() rpc.Client { + return t.client +} + +func AdaptDiskBackup(t DiskBackup) *rpc.Interface { + methods := []rpc.Method{ + { + Name: "backup", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "to_cloud", "pin", "data", "progress"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Backup(ctx, &DiskBackupBackup{Call: call}) + }, + }, + { + Name: "listBackups", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.ListBackups(ctx, &DiskBackupListBackups{Call: call}) + }, + }, + { + Name: "restore", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "restore_point", "data", "force", "progress"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Restore(ctx, &DiskBackupRestore{Call: call}) + }, + }, + } + + return rpc.NewInterface(methods, t) +} + +type DiskBackupClient struct { + rpc.Client +} + +func NewDiskBackupClient(client rpc.Client) *DiskBackupClient { + return &DiskBackupClient{Client: client} +} + +func (c DiskBackupClient) Export() DiskBackup { + return reexportDiskBackup{client: c.Client} +} + +type DiskBackupClientBackupResults struct { + client rpc.Client + data diskBackupBackupResultsData +} + +func (v *DiskBackupClientBackupResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientBackupResults) Result() *BackupResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Backup(ctx context.Context, disk string, to_cloud bool, pin string, data stream.SendStream[[]byte], progress stream.SendStream[*Progress]) (*DiskBackupClientBackupResults, error) { + args := DiskBackupBackupArgs{} + caps := map[rpc.OID]*rpc.InlineCapability{} + args.data.Disk = &disk + args.data.ToCloud = &to_cloud + args.data.Pin = &pin + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[[]byte](data), data) + args.data.Data = c + caps[oid] = ic + } + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[*Progress](progress), progress) + args.data.Progress = c + caps[oid] = ic + } + + var ret diskBackupBackupResultsData + + err := v.CallWithCaps(ctx, "backup", &args, &ret, caps) + if err != nil { + return nil, err + } + + return &DiskBackupClientBackupResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientListBackupsResults struct { + client rpc.Client + data diskBackupListBackupsResultsData +} + +func (v *DiskBackupClientListBackupsResults) HasPoints() bool { + return v.data.Points != nil +} + +func (v *DiskBackupClientListBackupsResults) Points() []*RestorePoint { + if v.data.Points == nil { + return nil + } + return *v.data.Points +} + +func (v DiskBackupClient) ListBackups(ctx context.Context, disk string) (*DiskBackupClientListBackupsResults, error) { + args := DiskBackupListBackupsArgs{} + args.data.Disk = &disk + + var ret diskBackupListBackupsResultsData + + err := v.Call(ctx, "listBackups", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientListBackupsResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientRestoreResults struct { + client rpc.Client + data diskBackupRestoreResultsData +} + +func (v *DiskBackupClientRestoreResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientRestoreResults) Result() *RestoreResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_point string, data stream.RecvStream[[]byte], force bool, progress stream.SendStream[*Progress]) (*DiskBackupClientRestoreResults, error) { + args := DiskBackupRestoreArgs{} + caps := map[rpc.OID]*rpc.InlineCapability{} + args.data.Disk = &disk + args.data.RestorePoint = &restore_point + { + ic, oid, c := v.NewInlineCapability(stream.AdaptRecvStream[[]byte](data), data) + args.data.Data = c + caps[oid] = ic + } + args.data.Force = &force + { + ic, oid, c := v.NewInlineCapability(stream.AdaptSendStream[*Progress](progress), progress) + args.data.Progress = c + caps[oid] = ic + } + + var ret diskBackupRestoreResultsData + + err := v.CallWithCaps(ctx, "restore", &args, &ret, caps) + if err != nil { + return nil, err + } + + return &DiskBackupClientRestoreResults{client: v.Client, data: ret}, nil +} diff --git a/api/disk/rpc.yml b/api/disk/rpc.yml new file mode 100644 index 000000000..db20d8268 --- /dev/null +++ b/api/disk/rpc.yml @@ -0,0 +1,211 @@ +apiVersion: miren.dev/rpc/v1 +kind: IDL + +imports: + stream: + path: ../../pkg/rpc/stream/stream.yml + import: miren.dev/runtime/pkg/rpc/stream + standard: + path: ../../pkg/rpc/standard/standard.yml + import: miren.dev/runtime/pkg/rpc/standard + +types: + - type: Transfer + doc: > + How far a byte transfer has got. Backup and restore move whole disk + images, so a client that showed nothing until completion would look + indistinguishable from a hang. + fields: + - name: done + type: int64 + index: 0 + doc: Bytes moved so far + - name: total + type: int64 + index: 1 + doc: Bytes expected in total, or 0 when the server cannot know yet + - name: bytes_per_second + type: int64 + index: 2 + doc: Recent throughput + - name: eta_seconds + type: int64 + index: 3 + doc: Estimated seconds remaining, or 0 when total is unknown + + - type: Progress + doc: > + One event in the progress stream a client passes to backup or restore. + Modelled on build's Status: a discriminated union, so new event kinds do + not break existing clients. + fields: + - name: update + type: union + union: + - name: message + type: string + index: 1 + doc: A step the server has started, for display + - name: transfer + type: Transfer + index: 2 + doc: Byte-movement progress + - name: warning + type: string + index: 3 + doc: > + Something the operator should know but which does not stop the + operation, such as backing up a disk that is currently mounted. + - name: error + type: string + index: 4 + doc: > + A failure being reported before the call returns. The call's own + error is still authoritative. + + - type: RestorePoint + doc: > + A moment in time a disk can be restored to. Universal-mode volumes reach + one by fetching a full image and accelerator-mode volumes by replaying log + segments (RFD-64), but that split is the server's business — a caller + picks a point, not a backup type. + fields: + - name: id + type: string + index: 0 + doc: Opaque identifier to pass back to restore + - name: created_at + type: standard.Timestamp + index: 1 + doc: When the restore point was taken + - name: size_bytes + type: int64 + index: 2 + doc: Compressed size held in the cloud + - name: image_size_bytes + type: int64 + index: 3 + doc: Size of the disk image the point restores to, 0 when not recorded + - name: name + type: string + index: 4 + doc: Operator-supplied name, set when the point was pinned against cleanup + - name: mode + type: string + index: 5 + doc: Volume mode the point was taken from (universal or accelerator) + + - type: BackupResult + fields: + - name: image_size_bytes + type: int64 + index: 0 + doc: Size of the image that was read + - name: compressed_size_bytes + type: int64 + index: 1 + doc: Size of the resulting snapshot + - name: checksum + type: string + index: 2 + doc: SHA-256 of the uncompressed image, hex encoded + - name: restore_point_id + type: string + index: 3 + doc: Identifier of the uploaded restore point, empty for a local backup + + - type: RestoreResult + fields: + - name: disk_id + type: string + index: 0 + doc: Entity id of the restored disk + - name: image_size_bytes + type: int64 + index: 1 + doc: Size of the image that was written + - name: created + type: bool + index: 2 + doc: True when the disk did not exist and was created by this restore + +interfaces: + - name: DiskBackup + doc: > + Backup and restore of disk images, driven by a remote client (RFD-108). + + Two steps genuinely have to happen on the server: moving the image bytes, + and talking to miren.cloud with the cluster's key. Both stay here. The + client orchestrates over this interface whether it runs on the server host + or on a laptop, so there is no local-versus-remote mode to detect and no + chance of mistaking "the data directory happens to exist here" for "this + client is colocated with the cluster it targets". + + Every method is admin-level, matching the other disk mutations. + methods: + - name: backup + doc: > + Snapshot a disk. With to_cloud set, the server compresses the image + and uploads it to miren.cloud as a restore point using the cluster's + own key, and no image bytes reach the client. Otherwise the compressed + snapshot is streamed down data, which is how a cluster with no cloud + still backs up to a file on the client. + parameters: + - name: disk + type: string + doc: Disk name to back up + - name: to_cloud + type: bool + doc: Upload to miren.cloud instead of streaming to the client + - name: pin + type: string + doc: Name the uploaded restore point, pinning it against cleanup + - name: data + type: stream.SendStream[[]byte] + doc: Stream the client receives the compressed snapshot on + - name: progress + type: stream.SendStream[*Progress] + doc: Stream the client receives progress events on + results: + - name: result + type: '*BackupResult' + + - name: listBackups + doc: > + List the restore points available for a disk, newest first, so a + client can show them and let the operator pick one. + parameters: + - name: disk + type: string + doc: Disk name to list restore points for + results: + - name: points + type: list + element: RestorePoint + + - name: restore + doc: > + Rebuild a disk's image. With restore_point set, the server downloads + that point from miren.cloud. Otherwise it reads a snapshot the client + streams up data. Restoring a disk the cluster has never seen creates + it, which is what makes this usable for disaster recovery onto a fresh + host. + parameters: + - name: disk + type: string + doc: Disk name to restore to + - name: restore_point + type: string + doc: Restore point to fetch from miren.cloud, empty to read from data + - name: data + type: stream.RecvStream[[]byte] + doc: Stream the server reads a client-supplied snapshot from + - name: force + type: bool + doc: Overwrite an existing disk image + - name: progress + type: stream.SendStream[*Progress] + doc: Stream the client receives progress events on + results: + - name: result + type: '*RestoreResult' From fac72fad39114281f8a483087b880409db211f6e Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:20:11 -0700 Subject: [PATCH 03/19] Serve disk backup and restore over RPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server half of RFD-108. servers/disk implements backup, listBackups and restore, registered on the coordinator, where disk images already live — cli/commands/disk_resolver.go pins disk volumes to the coordinator node, and `miren server` runs a runner in the same process. Backup to the cloud reuses diskio.ImageSnapshotter, which was already written and tested but had no caller in the server. Backup without a cloud stages a compressed snapshot and streams it to the client, so an air-gapped cluster still gets a backup. Restore reads either a cloud restore point or a snapshot the client streams up, and creates the disk if the cluster has never seen it, which is the disaster-recovery case. Restore now refuses when the image is loop-attached. This looks like a new restriction and is not: LoopAttach hands the kernel an open file descriptor, so a loop device holds the image's inode rather than its path, and renaming a restored image over an attached one leaves the mounted filesystem reading the old one. That restore has always reported success and changed nothing. Deliberately no --force for it — --force means "overwrite an existing image", and letting it also mean "write over a live loop device" would let an operator ask for a silent no-op. The same check replaces the mounted-disk warning on backup. The old warning keyed off disk.status == ATTACHED, which nothing in the runtime ever sets, so it had never fired once. ImageSnapshotter.Snapshot now returns sizes and a checksum alongside the update id, since the caller reports them to an operator. --- api/disk/disk_v1alpha/rpc.gen.go | 16 +- api/disk/rpc.yml | 4 +- components/coordinate/coordinate.go | 21 ++ components/diskio/image_snapshot.go | 39 ++- components/diskio/image_snapshot_test.go | 7 +- hack/e2e_disk/main.go | 3 +- pkg/workloadroles/roles.go | 3 + servers/disk/backup.go | 240 +++++++++++++++++ servers/disk/restore.go | 237 +++++++++++++++++ servers/disk/server.go | 219 ++++++++++++++++ servers/disk/server_test.go | 311 +++++++++++++++++++++++ 11 files changed, 1077 insertions(+), 23 deletions(-) create mode 100644 servers/disk/backup.go create mode 100644 servers/disk/restore.go create mode 100644 servers/disk/server.go create mode 100644 servers/disk/server_test.go diff --git a/api/disk/disk_v1alpha/rpc.gen.go b/api/disk/disk_v1alpha/rpc.gen.go index b1f939218..9be63abcb 100644 --- a/api/disk/disk_v1alpha/rpc.gen.go +++ b/api/disk/disk_v1alpha/rpc.gen.go @@ -421,7 +421,7 @@ func (v *BackupResult) UnmarshalJSON(data []byte) error { } type restoreResultData struct { - DiskId *string `cbor:"0,keyasint,omitempty" json:"disk_id,omitempty"` + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` ImageSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"image_size_bytes,omitempty"` Created *bool `cbor:"2,keyasint,omitempty" json:"created,omitempty"` } @@ -430,19 +430,19 @@ type RestoreResult struct { data restoreResultData } -func (v *RestoreResult) HasDiskId() bool { - return v.data.DiskId != nil +func (v *RestoreResult) HasDisk() bool { + return v.data.Disk != nil } -func (v *RestoreResult) DiskId() string { - if v.data.DiskId == nil { +func (v *RestoreResult) Disk() string { + if v.data.Disk == nil { return "" } - return *v.data.DiskId + return *v.data.Disk } -func (v *RestoreResult) SetDiskId(disk_id string) { - v.data.DiskId = &disk_id +func (v *RestoreResult) SetDisk(disk string) { + v.data.Disk = &disk } func (v *RestoreResult) HasImageSizeBytes() bool { diff --git a/api/disk/rpc.yml b/api/disk/rpc.yml index db20d8268..1fac2f09f 100644 --- a/api/disk/rpc.yml +++ b/api/disk/rpc.yml @@ -116,10 +116,10 @@ types: - type: RestoreResult fields: - - name: disk_id + - name: disk type: string index: 0 - doc: Entity id of the restored disk + doc: Name of the restored disk - name: image_size_bytes type: int64 index: 1 diff --git a/components/coordinate/coordinate.go b/components/coordinate/coordinate.go index 2b9694bfc..45eca13bb 100644 --- a/components/coordinate/coordinate.go +++ b/components/coordinate/coordinate.go @@ -28,6 +28,7 @@ import ( "miren.dev/runtime/api/core/core_v1alpha" "miren.dev/runtime/api/debug/debug_v1alpha" deployment_v1alpha "miren.dev/runtime/api/deployment/deployment_v1alpha" + "miren.dev/runtime/api/disk/disk_v1alpha" aes "miren.dev/runtime/api/entityserver" esv1 "miren.dev/runtime/api/entityserver/entityserver_v1alpha" "miren.dev/runtime/api/exec/exec_v1alpha" @@ -43,6 +44,7 @@ import ( "miren.dev/runtime/components/activator" "miren.dev/runtime/components/autotls" "miren.dev/runtime/components/buildkit" + "miren.dev/runtime/components/diskio" "miren.dev/runtime/components/netresolve" addonctrl "miren.dev/runtime/controllers/addon" artifactctrl "miren.dev/runtime/controllers/artifact" @@ -93,6 +95,7 @@ import ( "miren.dev/runtime/servers/build" debugsrv "miren.dev/runtime/servers/debug" "miren.dev/runtime/servers/deployment" + disksrv "miren.dev/runtime/servers/disk" "miren.dev/runtime/servers/entityserver" execproxy "miren.dev/runtime/servers/exec_proxy" "miren.dev/runtime/servers/httpingress" @@ -1659,6 +1662,24 @@ func (c *Coordinator) Start(ctx context.Context) (retErr error) { } server.ExposeValue(rpc.ServiceSqliteBackup, sqlitebackup_v1alpha.AdaptSqliteBackup(sqliteBackupServer)) + // Disk backup and restore, driven by a client that need not be on this host + // (RFD-108). The image bytes and the cloud key stay here; everything else is + // the client's orchestration. + // + // The updates client is nil when the cluster has no cloud registration, + // which leaves the local-file backup path working and refuses the cloud + // ones with an explanation. + var diskUpdates diskio.CloudUpdatesClient + if c.authClient != nil { + cloudURL := c.CloudAuth.CloudURL + if cloudURL == "" { + cloudURL = DefaultCloudURL + } + diskUpdates = diskio.NewCloudUpdatesClient(c.Log, cloudURL, c.authClient) + } + diskBackupServer := disksrv.NewServer(c.Log, eac, ec, c.DataPath, diskUpdates) + server.ExposeValue("dev.miren.runtime/disk-backup", disk_v1alpha.AdaptDiskBackup(diskBackupServer)) + // Create httpingress server for internal HTTP requests ingressConfig := httpingress.IngressConfig{ RequestTimeout: c.HTTPRequestTimeout, diff --git a/components/diskio/image_snapshot.go b/components/diskio/image_snapshot.go index 3b058e1d6..732f4a463 100644 --- a/components/diskio/image_snapshot.go +++ b/components/diskio/image_snapshot.go @@ -60,33 +60,47 @@ type SnapshotRequest struct { StagingDir string } -// Snapshot compresses the image and uploads it, returning the cloud's update ID. -func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (string, error) { +// SnapshotResult describes the restore point a Snapshot produced. The sizes and +// checksum are what a caller reports back to an operator, so they come from the +// snapshot that was actually uploaded rather than being recomputed. +type SnapshotResult struct { + // UpdateID is the cloud's identifier for the uploaded restore point. + UpdateID string + // ImageSize is the uncompressed size of the image that was read. + ImageSize int64 + // CompressedSize is the size of the uploaded snapshot. + CompressedSize int64 + // Checksum is the SHA-256 of the uncompressed image, hex encoded. + Checksum string +} + +// Snapshot compresses the image and uploads it as a restore point. +func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (*SnapshotResult, error) { if s.updates == nil { - return "", fmt.Errorf("no cloud updates client configured") + return nil, fmt.Errorf("no cloud updates client configured") } if req.VolumeID == "" { - return "", fmt.Errorf("volume ID is required") + return nil, fmt.Errorf("volume ID is required") } info, err := os.Stat(req.ImagePath) if err != nil { - return "", fmt.Errorf("stat image: %w", err) + return nil, fmt.Errorf("stat image: %w", err) } staged, checksum, err := s.stage(req, info) if err != nil { - return "", err + return nil, err } defer os.Remove(staged.Name()) defer staged.Close() stagedInfo, err := staged.Stat() if err != nil { - return "", fmt.Errorf("stat staged snapshot: %w", err) + return nil, fmt.Errorf("stat staged snapshot: %w", err) } if _, err := staged.Seek(0, io.SeekStart); err != nil { - return "", fmt.Errorf("rewind staged snapshot: %w", err) + return nil, fmt.Errorf("rewind staged snapshot: %w", err) } // The ordering key must sort against its siblings and match the cloud's @@ -109,7 +123,7 @@ func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (s LeaseNonce: req.LeaseNonce, }, staged, stagedInfo.Size()) if err != nil { - return "", fmt.Errorf("upload image snapshot: %w", err) + return nil, fmt.Errorf("upload image snapshot: %w", err) } s.log.Info("uploaded volume image snapshot", @@ -119,7 +133,12 @@ func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (s "image_size", info.Size(), "compressed_size", stagedInfo.Size(), ) - return updateID, nil + return &SnapshotResult{ + UpdateID: updateID, + ImageSize: info.Size(), + CompressedSize: stagedInfo.Size(), + Checksum: checksum, + }, nil } // stage writes a compressed snapshot to a temp file. diff --git a/components/diskio/image_snapshot_test.go b/components/diskio/image_snapshot_test.go index 4393429d0..47c019424 100644 --- a/components/diskio/image_snapshot_test.go +++ b/components/diskio/image_snapshot_test.go @@ -29,7 +29,7 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { content := bytes.Repeat([]byte("disk contents "), 512) snapshotter, fake, imagePath := newSnapshotFixture(t, content) - updateID, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ + res, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ VolumeID: "vol-1", ImagePath: imagePath, Name: "data", @@ -37,7 +37,10 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { SnapshotName: "pre-migration", }) require.NoError(t, err) - assert.Equal(t, "volup-fake", updateID) + assert.Equal(t, "volup-fake", res.UpdateID) + assert.Equal(t, int64(len(content)), res.ImageSize) + assert.NotZero(t, res.CompressedSize) + assert.NotEmpty(t, res.Checksum) require.Len(t, fake.uploads, 1) up := fake.uploads[0] diff --git a/hack/e2e_disk/main.go b/hack/e2e_disk/main.go index 6659f9d0c..676468dd8 100644 --- a/hack/e2e_disk/main.go +++ b/hack/e2e_disk/main.go @@ -67,13 +67,14 @@ func main() { check(os.WriteFile(filepath.Join(diskPath, "disk.img"), imageData, 0644), "write image") snapshotter := diskio.NewImageSnapshotter(log, updates) - imageUpdateID, err := snapshotter.Snapshot(ctx, diskio.SnapshotRequest{ + imageSnap, err := snapshotter.Snapshot(ctx, diskio.SnapshotRequest{ VolumeID: volumeID, ImagePath: filepath.Join(diskPath, "disk.img"), Name: "e2e-disk", Filesystem: "ext4", }) check(err, "snapshot image") + imageUpdateID := imageSnap.UpdateID fmt.Printf("✓ uploaded loop_image snapshot %s\n", imageUpdateID) // --- list both kinds back --- diff --git a/pkg/workloadroles/roles.go b/pkg/workloadroles/roles.go index 304bcc3ee..29edfe659 100644 --- a/pkg/workloadroles/roles.go +++ b/pkg/workloadroles/roles.go @@ -155,6 +155,9 @@ func clusterAdminPerms() perms { "internalhttp": set("dorequest"), "disks": set("new", "delete"), "addons": set("createinstance", "deleteinstance"), + // Backup and restore read and rewrite a disk's contents wholesale, + // so they sit with the other disk mutations rather than with reads. + "diskbackup": set("backup", "restore", "listbackups"), }, ) } diff --git a/servers/disk/backup.go b/servers/disk/backup.go new file mode 100644 index 000000000..a3b75dcce --- /dev/null +++ b/servers/disk/backup.go @@ -0,0 +1,240 @@ +package disk + +import ( + "context" + "fmt" + "io" + "os" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// Backup snapshots a disk, either to miren.cloud or down to the client. +func (s *Server) Backup(ctx context.Context, state *disk_v1alpha.DiskBackupBackup) error { + args := state.Args() + prog := s.newProgress(ctx, args.Progress()) + + target, err := s.prepareBackup(ctx, args.Disk()) + if err != nil { + return err + } + + // Backing up an image while its loop device is still writing gives a read + // whose head and tail come from different moments — weaker than the + // power-loss state fsck and Postgres's WAL recovery are built for. Say so + // and continue: the operator is the one deciding this is safe. + dev, err := s.liveImageDevice(target.ImagePath) + if err != nil { + return err + } + if dev != "" { + prog.Warn("Disk %q is in use (%s) and may be written during the backup.", target.Name, dev) + prog.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") + prog.Warn("Detach the disk first for a backup you can rely on.") + } + + if args.ToCloud() { + return s.backupToCloud(ctx, state, prog, target) + } + return s.backupToClient(ctx, state, prog, target) +} + +func (s *Server) backupToCloud( + ctx context.Context, + state *disk_v1alpha.DiskBackupBackup, + prog progressSink, + target *snapshot.BackupTarget, +) error { + if s.updates == nil { + return errNoCloud("backing up to miren.cloud") + } + if target.CloudVolumeID == "" { + return fmt.Errorf( + "disk %q is not registered with miren.cloud yet, so there is nowhere to upload to — it registers on its own shortly after the disk is created", + target.Name, + ) + } + + prog.Message("Compressing and uploading %s to miren.cloud", target.Name) + + snapper := diskio.NewImageSnapshotter(s.log, s.updates) + res, err := snapper.Snapshot(ctx, diskio.SnapshotRequest{ + VolumeID: target.CloudVolumeID, + ImagePath: target.ImagePath, + Name: target.Name, + Filesystem: target.Filesystem, + SnapshotName: state.Args().Pin(), + StagingDir: s.stagingDir(target.ImagePath), + }) + if err != nil { + return err + } + + s.log.Info("backed up disk to cloud", + "disk", target.Name, + "restore_point", res.UpdateID, + "image_size", res.ImageSize, + "compressed_size", res.CompressedSize, + ) + + out := new(disk_v1alpha.BackupResult) + out.SetImageSizeBytes(res.ImageSize) + out.SetCompressedSizeBytes(res.CompressedSize) + out.SetChecksum(res.Checksum) + out.SetRestorePointId(res.UpdateID) + state.Results().SetResult(&out) + return nil +} + +// backupToClient compresses the image and streams it down to the caller, which +// is how a cluster with no miren.cloud still produces a backup. +// +// The snapshot is staged to a temp file rather than compressed straight into +// the stream: snapshot.Backup rewrites the header with the image's checksum +// once it has read the whole image, so it needs somewhere it can seek back to. +func (s *Server) backupToClient( + ctx context.Context, + state *disk_v1alpha.DiskBackupBackup, + prog progressSink, + target *snapshot.BackupTarget, +) error { + out := state.Args().Data() + if out == nil { + return fmt.Errorf("backup needs either --cloud or somewhere to write the snapshot") + } + + img, err := os.Open(target.ImagePath) + if err != nil { + return fmt.Errorf("opening disk image: %w", err) + } + defer img.Close() + + info, err := img.Stat() + if err != nil { + return fmt.Errorf("stat disk image: %w", err) + } + + prog.Message("Compressing %s (%d bytes)", target.Name, info.Size()) + + staged, err := os.CreateTemp(s.stagingDir(target.ImagePath), ".disk-backup-*") + if err != nil { + return fmt.Errorf("creating staging file: %w", err) + } + defer os.Remove(staged.Name()) + defer staged.Close() + + checksum, err := snapshot.Backup(staged, img, target.Name, info.Size(), target.Filesystem) + if err != nil { + return err + } + + stagedInfo, err := staged.Stat() + if err != nil { + return fmt.Errorf("stat staging file: %w", err) + } + if _, err := staged.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewind staging file: %w", err) + } + + prog.Message("Sending %d bytes", stagedInfo.Size()) + + w := stream.ToWriter(ctx, out) + sent, err := io.Copy(w, s.trackReads(staged, prog, stagedInfo.Size())) + if err != nil { + return fmt.Errorf("sending snapshot: %w", err) + } + // The stream belongs to the caller, so closing our writer flushes what we + // wrote without tearing their client down. + if err := w.Close(); err != nil { + return fmt.Errorf("finishing snapshot stream: %w", err) + } + if sent != stagedInfo.Size() { + return fmt.Errorf("sent %d bytes of a %d byte snapshot", sent, stagedInfo.Size()) + } + + s.log.Info("backed up disk to client", + "disk", target.Name, + "image_size", info.Size(), + "compressed_size", stagedInfo.Size(), + ) + + res := new(disk_v1alpha.BackupResult) + res.SetImageSizeBytes(info.Size()) + res.SetCompressedSizeBytes(stagedInfo.Size()) + res.SetChecksum(checksum) + state.Results().SetResult(&res) + return nil +} + +// trackReads reports progress as bytes are pulled out of r. +func (s *Server) trackReads(r io.Reader, prog progressSink, total int64) io.Reader { + start := time.Now() + var done int64 + var lastReport time.Time + + return readerFunc(func(p []byte) (int, error) { + n, err := r.Read(p) + done += int64(n) + + // One event per 250ms. The client renders a bar from these, and a + // report per read would be thousands of RPCs for one backup. + if now := time.Now(); now.Sub(lastReport) >= 250*time.Millisecond || err == io.EOF { + lastReport = now + elapsed := now.Sub(start).Seconds() + var perSecond, eta int64 + if elapsed > 0 { + perSecond = int64(float64(done) / elapsed) + } + if perSecond > 0 && total > done { + eta = (total - done) / perSecond + } + prog.Transfer(done, total, perSecond, eta) + } + return n, err + }) +} + +type readerFunc func(p []byte) (int, error) + +func (f readerFunc) Read(p []byte) (int, error) { return f(p) } + +// ListBackups returns the restore points a disk can be restored to, newest +// first. +func (s *Server) ListBackups(ctx context.Context, state *disk_v1alpha.DiskBackupListBackups) error { + args := state.Args() + + target, err := s.prepareBackup(ctx, args.Disk()) + if err != nil { + return err + } + + if s.updates == nil { + return errNoCloud("listing restore points") + } + if target.CloudVolumeID == "" { + // Not an error: a disk that has never been registered simply has no + // restore points, which is a different thing from a broken lookup. + state.Results().SetPoints(nil) + return nil + } + + updates, err := s.updates.List(ctx, target.CloudVolumeID, diskio.ListOptions{ + Kind: diskio.KindLoopImage, + Descending: true, + }) + if err != nil { + return fmt.Errorf("listing restore points for %s: %w", target.Name, err) + } + + points := make([]*disk_v1alpha.RestorePoint, 0, len(updates)) + for _, u := range updates { + points = append(points, restorePointFromUpdate(u)) + } + + state.Results().SetPoints(points) + return nil +} diff --git a/servers/disk/restore.go b/servers/disk/restore.go new file mode 100644 index 000000000..f44bee9c7 --- /dev/null +++ b/servers/disk/restore.go @@ -0,0 +1,237 @@ +package disk + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// Restore rebuilds a disk's image, either from a restore point in miren.cloud +// or from a snapshot the client streams up. +// +// Restoring a disk this cluster has never seen creates it, which is the case +// that matters for disaster recovery: the host is gone and the whole task is to +// rebuild onto a new one. +func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRestore) (retErr error) { + args := state.Args() + prog := s.newProgress(ctx, args.Progress()) + + name := args.Disk() + if name == "" { + return fmt.Errorf("disk name is required") + } + + src, compressedSize, closeSrc, err := s.restoreSource(ctx, name, args, prog) + if err != nil { + return err + } + defer closeSrc() + + meta, err := snapshot.ReadHeader(src) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + + target, err := snapshot.PrepareRestore(ctx, s.disks, name, s.dataPath, + snapshot.WithCreator(s.disks, meta.SizeBytes, meta.Filesystem)) + if err != nil { + return err + } + + // Roll the entities back if anything below fails, but only when this + // restore is what created them. + defer func() { + if retErr != nil && target.Created && target.Cleanup != nil { + if cerr := target.Cleanup(ctx); cerr != nil { + s.log.Warn("failed to clean up after restore", "disk", name, "error", cerr) + } + } + }() + + if err := s.refuseLiveImage(target, name); err != nil { + return err + } + + if !target.Created { + if _, err := os.Stat(target.ImagePath); err == nil && !args.Force() { + return fmt.Errorf( + "disk %q already has an image at %s — pass --force to overwrite it", + name, target.ImagePath, + ) + } + } + + prog.Message("Restoring %s (%d bytes)", name, meta.SizeBytes) + + if err := s.writeImage(target.ImagePath, src, compressedSize, meta, prog); err != nil { + return err + } + + if target.Finalize != nil { + if err := target.Finalize(ctx); err != nil { + return err + } + } + + s.log.Info("restored disk", + "disk", name, + "image_size", meta.SizeBytes, + "created", target.Created, + ) + + res := new(disk_v1alpha.RestoreResult) + res.SetDisk(name) + res.SetImageSizeBytes(meta.SizeBytes) + res.SetCreated(target.Created) + state.Results().SetResult(&res) + return nil +} + +// restoreSource opens the snapshot to restore from, whichever end it came from. +// +// The second return is the compressed size when it is known, and 0 when it is +// not. Progress is measured in compressed bytes because that is what actually +// moves; a client streaming a snapshot up has not told us how big it is, so +// there is nothing honest to show a percentage against. +func (s *Server) restoreSource( + ctx context.Context, + name string, + args *disk_v1alpha.DiskBackupRestoreArgs, + prog progressSink, +) (io.Reader, int64, func(), error) { + point := args.RestorePoint() + if point == "" { + in := args.Data() + if in == nil { + return nil, 0, nil, fmt.Errorf("restore needs either a restore point or a snapshot to read") + } + prog.Message("Reading snapshot from client") + r := stream.ToReader(ctx, in) + // The stream belongs to the caller; closing our reader would tear + // their client down, so leave it to them. + return r, 0, func() {}, nil + } + + if s.updates == nil { + return nil, 0, nil, errNoCloud("restoring from a restore point") + } + + // The restore point lives against the disk's cloud volume, so the disk has + // to already exist for this path. Creating a disk from a cloud restore + // point it has no record of is not something this can resolve. + target, err := s.prepareBackup(ctx, name) + if err != nil { + return nil, 0, nil, err + } + if target.CloudVolumeID == "" { + return nil, 0, nil, fmt.Errorf("disk %q is not registered with miren.cloud, so it has no restore points", name) + } + + size := s.restorePointSize(ctx, target.CloudVolumeID, point) + + prog.Message("Downloading restore point %s", point) + + body, err := s.updates.Download(ctx, target.CloudVolumeID, point) + if err != nil { + return nil, 0, nil, fmt.Errorf("downloading restore point %s: %w", point, err) + } + return body, size, func() { body.Close() }, nil +} + +// restorePointSize looks up how big a restore point is, so the download can +// show a percentage. Best effort: a failure here costs a progress bar, not a +// restore, so it is logged rather than returned. +func (s *Server) restorePointSize(ctx context.Context, cloudVolumeID, point string) int64 { + updates, err := s.updates.List(ctx, cloudVolumeID, diskio.ListOptions{ + Kind: diskio.KindLoopImage, + Descending: true, + }) + if err != nil { + s.log.Debug("could not size restore point", "restore_point", point, "error", err) + return 0 + } + for _, u := range updates { + if u.UpdateID == point { + return u.Size + } + } + return 0 +} + +// refuseLiveImage stops a restore that would silently do nothing. +// +// See liveImageDevice: a loop device holds the image's inode, not its path, so +// writing a new image and renaming it into place leaves the running system on +// the old one. There is deliberately no --force for this. --force means +// "overwrite an existing image", and letting it also mean "write over a live +// loop device" would let an operator ask for a no-op and be told it worked. +func (s *Server) refuseLiveImage(target *snapshot.RestoreTarget, name string) error { + dev, err := s.liveImageDevice(target.ImagePath) + if err != nil { + return err + } + if dev == "" { + return nil + } + return fmt.Errorf( + "disk %q is in use (%s is backing %s), and restoring it now would write an image nothing reads — "+ + "stop everything using the disk first, or restore into a new disk instead", + name, dev, target.ImagePath, + ) +} + +// writeImage decompresses a snapshot into the image path. +// +// It writes to a temp file and renames, so an interrupted restore cannot leave +// a half-written image that looks complete. +func (s *Server) writeImage(imagePath string, src io.Reader, compressedSize int64, meta *snapshot.Meta, prog progressSink) error { + if err := os.MkdirAll(filepath.Dir(imagePath), 0700); err != nil { + return fmt.Errorf("creating volume directory: %w", err) + } + + tmpPath := imagePath + ".restore.tmp" + out, err := os.Create(tmpPath) + if err != nil { + return fmt.Errorf("creating image file: %w", err) + } + + closed := false + cleanup := true + defer func() { + if !closed { + out.Close() + } + if cleanup { + os.Remove(tmpPath) + } + }() + + // Preallocate sparsely so the sparse-aware writer can seek over zero runs. + if err := out.Truncate(meta.SizeBytes); err != nil { + return fmt.Errorf("preallocating image: %w", err) + } + + if err := snapshot.RestoreImage(out, s.trackReads(src, prog, compressedSize), meta); err != nil { + return err + } + + if err := out.Sync(); err != nil { + return fmt.Errorf("flushing image: %w", err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("closing image: %w", err) + } + closed = true + if err := os.Rename(tmpPath, imagePath); err != nil { + return fmt.Errorf("moving image into place: %w", err) + } + cleanup = false + return nil +} diff --git a/servers/disk/server.go b/servers/disk/server.go new file mode 100644 index 000000000..8d5567c30 --- /dev/null +++ b/servers/disk/server.go @@ -0,0 +1,219 @@ +// Package disk serves the DiskBackup RPC interface: backup and restore of disk +// images, driven by a client that need not be running on the server host. +// +// Two steps genuinely have to happen here rather than on the client. Moving the +// image bytes, because the image is a file in the server's data directory. And +// talking to miren.cloud, because that authenticates as the cluster using a key +// that lives only on the server and should never reach a laptop. Everything +// else — resolving which disk the operator means, deciding which restore point +// to use, writing a snapshot to a file — is orchestration the client does. +// +// See RFD-108. +package disk + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/rpc/standard" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" +) + +// diskEntities is what the handlers need from the entity store: which disk the +// operator means, and how to conjure one that does not exist yet. +type diskEntities interface { + snapshot.DiskResolver + snapshot.DiskCreator +} + +// Server implements disk_v1alpha.DiskBackup. +type Server struct { + log *slog.Logger + disks diskEntities + dataPath string + + // updates is nil on a cluster with no miren.cloud registration. The + // streaming paths still work in that case; only the cloud ones refuse. + updates diskio.CloudUpdatesClient + + // mntOps answers "is this image currently in use". It reads kernel state + // directly, so it needs no handle on the runner's controllers. + mntOps diskio.DiskMountOps +} + +// NewServer builds the disk backup service. updates may be nil, which means the +// cluster has no cloud to back up to. +func NewServer( + log *slog.Logger, + eac *entityserver_v1alpha.EntityAccessClient, + ec *entityserver.Client, + dataPath string, + updates diskio.CloudUpdatesClient, +) *Server { + log = log.With("module", "disk-backup") + return &Server{ + log: log, + disks: diskresolve.New(eac, ec), + dataPath: dataPath, + updates: updates, + mntOps: diskio.NewRealDiskMountOps(log), + } +} + +// errNoCloud is what every cloud-dependent path reports. It names the command +// that would fix it, because "no cloud configured" on its own leaves an +// operator guessing whether that is a bug or a setup step they skipped. +func errNoCloud(what string) error { + return fmt.Errorf( + "%s needs a miren.cloud registration, and this cluster has none — run `miren register` first, or back up to a local file instead", + what, + ) +} + +// liveImageDevice reports the loop device currently backing an image, or "" if +// there is none. +// +// This is the check that matters before rewriting an image, and it is not the +// same question as "does a lease exist". LoopAttach hands the kernel an open +// file descriptor, so a loop device is bound to the image's inode and not to +// its path. Renaming a fresh image over an attached one therefore changes +// nothing an operator can see: the old inode survives unlinked, the mounted +// filesystem keeps reading and writing it, and the restore reports success +// having accomplished nothing. +// +// A lease check does not cover this. Universal-mode volumes are mounted by the +// volume controller whether or not anything has leased them, and releasing the +// last lease deliberately leaves the mount up. +func (s *Server) liveImageDevice(imagePath string) (string, error) { + dev, err := s.mntOps.FindLoopByBacking(imagePath) + if err != nil { + // Fail closed. Not knowing whether the image is in use is not the same + // as knowing it is idle, and guessing wrong here loses data silently. + return "", fmt.Errorf("checking whether %s is in use: %w", imagePath, err) + } + return dev, nil +} + +// progressSink adapts the client's progress stream to something the handlers +// can call without checking for nil or caring about send failures. A client +// that has stopped listening should not fail an in-flight backup. +type progressSink struct { + log *slog.Logger + send func(*disk_v1alpha.Progress) +} + +func (p progressSink) Message(format string, args ...any) { + up := new(disk_v1alpha.Progress) + up.Update().SetMessage(fmt.Sprintf(format, args...)) + p.send(up) +} + +func (p progressSink) Warn(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + p.log.Warn(msg) + up := new(disk_v1alpha.Progress) + up.Update().SetWarning(msg) + p.send(up) +} + +func (p progressSink) Transfer(done, total, perSecond, etaSeconds int64) { + t := new(disk_v1alpha.Transfer) + t.SetDone(done) + t.SetTotal(total) + t.SetBytesPerSecond(perSecond) + t.SetEtaSeconds(etaSeconds) + + up := new(disk_v1alpha.Progress) + up.Update().SetTransfer(t) + p.send(up) +} + +// newProgress wraps a client's progress stream, tolerating a caller that did +// not supply one. +func (s *Server) newProgress(ctx context.Context, out *stream.SendStreamClient[*disk_v1alpha.Progress]) progressSink { + return progressSink{ + log: s.log, + send: func(up *disk_v1alpha.Progress) { + if out == nil { + return + } + if _, err := out.Send(ctx, up); err != nil { + s.log.Debug("dropping progress event", "error", err) + } + }, + } +} + +// prepareBackup resolves a disk to an image on this host. +func (s *Server) prepareBackup(ctx context.Context, name string) (*snapshot.BackupTarget, error) { + if name == "" { + return nil, fmt.Errorf("disk name is required") + } + return snapshot.PrepareBackup(ctx, s.disks, name, s.dataPath) +} + +// restorePointFromUpdate converts a cloud update into the restore point a +// client picks from. +// +// The cloud's UpdateInfo carries no timestamp, so it is recovered from the +// ordering key, whose format is fixed per update kind: loop_image keys are Unix +// nanoseconds in 16 hex digits. +func restorePointFromUpdate(u diskio.UpdateInfo) *disk_v1alpha.RestorePoint { + rp := new(disk_v1alpha.RestorePoint) + rp.SetId(u.UpdateID) + rp.SetSizeBytes(u.Size) + rp.SetName(u.SnapshotName) + rp.SetMode(modeForKind(u.Kind)) + if ts := timestamp(orderingKeyTime(u)); ts != nil { + rp.SetCreatedAt(ts) + } + return rp +} + +func modeForKind(kind string) string { + switch diskio.UpdateKind(kind) { + case diskio.KindLoopImage: + return "universal" + case diskio.KindLBDLog: + return "accelerator" + default: + return kind + } +} + +func orderingKeyTime(u diskio.UpdateInfo) time.Time { + if diskio.UpdateKind(u.Kind) != diskio.KindLoopImage { + return time.Time{} + } + var nanos int64 + if _, err := fmt.Sscanf(u.OrderingKey, "%016x", &nanos); err != nil { + return time.Time{} + } + return time.Unix(0, nanos) +} + +func timestamp(t time.Time) *standard.Timestamp { + if t.IsZero() { + return nil + } + ts := new(standard.Timestamp) + ts.SetSeconds(t.Unix()) + ts.SetNanoseconds(int32(t.Nanosecond())) + return ts +} + +// stagingDir is where a snapshot is compressed before it is uploaded or +// streamed. It sits beside the image so the write stays on the same filesystem +// as the data it came from, which is the one sized for it. +func (s *Server) stagingDir(imagePath string) string { + return filepath.Dir(imagePath) +} diff --git a/servers/disk/server_test.go b/servers/disk/server_test.go new file mode 100644 index 000000000..ff873c0e9 --- /dev/null +++ b/servers/disk/server_test.go @@ -0,0 +1,311 @@ +package disk + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/snapshot" +) + +// fakeDisks stands in for the entity store. +type fakeDisks struct { + disk *snapshot.DiskState + volume *snapshot.VolumeState + leases []snapshot.LeaseState + created *snapshot.RestoreTarget +} + +func (f *fakeDisks) FindDisk(_ context.Context, name string) (*snapshot.DiskState, error) { + if f.disk == nil { + return nil, fmt.Errorf("disk %q not found", name) + } + return f.disk, nil +} + +func (f *fakeDisks) FindVolume(context.Context, string) (*snapshot.VolumeState, error) { + if f.volume == nil { + return nil, errors.New("no volume") + } + return f.volume, nil +} + +func (f *fakeDisks) FindLeases(context.Context, string) ([]snapshot.LeaseState, error) { + return f.leases, nil +} + +func (f *fakeDisks) CreateDiskAndVolume(context.Context, string, int64, string, string) (*snapshot.RestoreTarget, error) { + if f.created == nil { + return nil, errors.New("creation not configured") + } + return f.created, nil +} + +// fakeUpdates stands in for miren.cloud. +type fakeUpdates struct { + uploaded []diskio.UploadRequest + bodies [][]byte + list []diskio.UpdateInfo + download map[string][]byte + + listErr error + downloadErr error +} + +func (f *fakeUpdates) Upload(_ context.Context, _ string, req diskio.UploadRequest, body io.Reader, size int64) (string, error) { + data, err := io.ReadAll(body) + if err != nil { + return "", err + } + if int64(len(data)) != size { + return "", fmt.Errorf("declared %d bytes but body is %d", size, len(data)) + } + f.uploaded = append(f.uploaded, req) + f.bodies = append(f.bodies, data) + return "volup-test", nil +} + +func (f *fakeUpdates) List(context.Context, string, diskio.ListOptions) ([]diskio.UpdateInfo, error) { + return f.list, f.listErr +} + +func (f *fakeUpdates) Download(_ context.Context, _, updateID string) (io.ReadCloser, error) { + if f.downloadErr != nil { + return nil, f.downloadErr + } + data, ok := f.download[updateID] + if !ok { + return nil, fmt.Errorf("no such update %q", updateID) + } + return io.NopCloser(newReader(data)), nil +} + +// fakeMountOps reports whether an image is loop-attached. +type fakeMountOps struct { + diskio.DiskMountOps + device string + err error +} + +func (f fakeMountOps) FindLoopByBacking(string) (string, error) { return f.device, f.err } + +// newTestServer builds a server over a temp data directory holding one disk +// image, and returns the server, its fakes, and the image path. +func newTestServer(t *testing.T, content []byte) (*Server, *fakeDisks, *fakeUpdates, string) { + t.Helper() + + dataPath := t.TempDir() + volDir := filepath.Join(dataPath, "disk-data", "volumes", "vol-1") + require.NoError(t, os.MkdirAll(volDir, 0700)) + imagePath := filepath.Join(volDir, "disk.img") + require.NoError(t, os.WriteFile(imagePath, content, 0644)) + + disks := &fakeDisks{ + disk: &snapshot.DiskState{ + ID: "disk/1", Name: "data", Status: "PROVISIONED", Filesystem: "ext4", + }, + volume: &snapshot.VolumeState{ + VolumeID: "vol-1", CloudVolumeID: "cloud-vol-1", ImagePath: imagePath, + }, + } + updates := &fakeUpdates{download: map[string][]byte{}} + + s := &Server{ + log: slog.Default(), + disks: disks, + dataPath: dataPath, + updates: updates, + mntOps: fakeMountOps{}, + } + return s, disks, updates, imagePath +} + +func TestPrepareBackupRequiresAName(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + _, err := s.prepareBackup(context.Background(), "") + require.Error(t, err) + assert.Contains(t, err.Error(), "disk name is required") +} + +func TestPrepareBackupResolvesTheImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + assert.Equal(t, imagePath, target.ImagePath) + assert.Equal(t, "cloud-vol-1", target.CloudVolumeID) +} + +// A cluster with no cloud must say so rather than failing obscurely, and it +// must point at the thing that would fix it. +func TestNoCloudErrorNamesTheRemedy(t *testing.T) { + err := errNoCloud("backing up to miren.cloud") + assert.Contains(t, err.Error(), "backing up to miren.cloud") + assert.Contains(t, err.Error(), "miren register") + assert.Contains(t, err.Error(), "local file") +} + +// Backup of an unregistered disk is what makes an air-gapped cluster work, so +// resolution must succeed and simply report no cloud volume. +func TestPrepareBackupToleratesAnUnregisteredDisk(t *testing.T) { + s, disks, _, imagePath := newTestServer(t, []byte("hello")) + disks.volume.CloudVolumeID = "" + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + assert.Equal(t, imagePath, target.ImagePath) + assert.Empty(t, target.CloudVolumeID) +} + +func TestRestorePointCarriesATimestampDecodedFromTheOrderingKey(t *testing.T) { + when := time.Unix(0, 1_700_000_000_123_456_789) + + rp := restorePointFromUpdate(diskio.UpdateInfo{ + UpdateID: "u-1", + Kind: string(diskio.KindLoopImage), + OrderingKey: fmt.Sprintf("%016x", when.UnixNano()), + Size: 4096, + SnapshotName: "pre-migration", + }) + + assert.Equal(t, "u-1", rp.Id()) + assert.Equal(t, int64(4096), rp.SizeBytes()) + assert.Equal(t, "pre-migration", rp.Name()) + assert.Equal(t, "universal", rp.Mode()) + require.True(t, rp.HasCreatedAt()) + assert.Equal(t, when.Unix(), rp.CreatedAt().Seconds()) + assert.Equal(t, int32(when.Nanosecond()), rp.CreatedAt().Nanoseconds()) +} + +// An accelerator-mode segment has a TAI64N key, which is not Unix nanoseconds. +// Reporting a garbage date would be worse than reporting none. +func TestRestorePointOmitsATimestampItCannotDecode(t *testing.T) { + rp := restorePointFromUpdate(diskio.UpdateInfo{ + UpdateID: "u-2", + Kind: string(diskio.KindLBDLog), + OrderingKey: "400000005f5e1000", + }) + + assert.Equal(t, "accelerator", rp.Mode()) + assert.False(t, rp.HasCreatedAt()) +} + +func TestLiveImageDeviceReportsAnAttachedLoop(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{device: "/dev/loop3"} + + dev, err := s.liveImageDevice(imagePath) + require.NoError(t, err) + assert.Equal(t, "/dev/loop3", dev) +} + +// Not knowing whether an image is in use is not the same as knowing it is idle. +func TestLiveImageDeviceFailsClosed(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{err: errors.New("sysfs unreadable")} + + _, err := s.liveImageDevice(imagePath) + require.Error(t, err) + assert.Contains(t, err.Error(), "sysfs unreadable") +} + +// The refusal is the whole point of the check: renaming an image over a live +// loop device is silently a no-op, so a restore that "succeeded" would have +// changed nothing. +func TestRestoreRefusesALiveImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + s.mntOps = fakeMountOps{device: "/dev/loop7"} + + err := s.refuseLiveImage(&snapshot.RestoreTarget{ImagePath: imagePath}, "data") + require.Error(t, err) + assert.Contains(t, err.Error(), "/dev/loop7") + assert.Contains(t, err.Error(), "nothing reads") +} + +func TestRestoreAllowsAnIdleImage(t *testing.T) { + s, _, _, imagePath := newTestServer(t, []byte("hello")) + + require.NoError(t, s.refuseLiveImage(&snapshot.RestoreTarget{ImagePath: imagePath}, "data")) +} + +// A backup written by this server must be restorable by it, byte for byte. +func TestWriteImageRoundTripsASnapshot(t *testing.T) { + content := make([]byte, 128*1024) + for i := range content { + content[i] = byte(i % 251) + } + + s, _, _, imagePath := newTestServer(t, content) + + staged := filepath.Join(t.TempDir(), "snap.miren.zst") + out, err := os.Create(staged) + require.NoError(t, err) + + img, err := os.Open(imagePath) + require.NoError(t, err) + defer img.Close() + + _, err = snapshot.Backup(out, img, "data", int64(len(content)), "ext4") + require.NoError(t, err) + require.NoError(t, out.Close()) + + src, err := os.Open(staged) + require.NoError(t, err) + defer src.Close() + + meta, err := snapshot.ReadHeader(src) + require.NoError(t, err) + + restored := filepath.Join(t.TempDir(), "restored.img") + prog := s.newProgress(context.Background(), nil) + require.NoError(t, s.writeImage(restored, src, 0, meta, prog)) + + got, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, got) +} + +// An interrupted restore must not leave something that looks like a finished +// image. +func TestWriteImageLeavesNothingBehindOnACorruptSnapshot(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + restored := filepath.Join(t.TempDir(), "restored.img") + prog := s.newProgress(context.Background(), nil) + + meta := &snapshot.Meta{Name: "data", SizeBytes: 4096, Checksum: "deadbeef"} + err := s.writeImage(restored, newReader([]byte("not a zstd stream")), 0, meta, prog) + require.Error(t, err) + + _, statErr := os.Stat(restored) + assert.True(t, os.IsNotExist(statErr), "a failed restore must not leave an image behind") + + _, tmpErr := os.Stat(restored + ".restore.tmp") + assert.True(t, os.IsNotExist(tmpErr), "a failed restore must not leave its temp file behind") +} + +func newReader(b []byte) io.Reader { return &sliceReader{b: b} } + +type sliceReader struct { + b []byte + i int +} + +func (r *sliceReader) Read(p []byte) (int, error) { + if r.i >= len(r.b) { + return 0, io.EOF + } + n := copy(p, r.b[r.i:]) + r.i += n + return n, nil +} From ebf9684b6c1df4f96aba4a96ee12edeae94dca90 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:23:17 -0700 Subject: [PATCH 04/19] Drive disk backup and restore over RPC from the CLI `miren disk backup` and `miren disk restore` now ask the server to do the work instead of opening the disk image themselves, so they work from a laptop. There is one path whether you run them on the server host or not, which is why the old "data path /var/lib/miren not found" refusal is gone rather than being made conditional: with no local-versus-remote detection there is no detection to guess wrong. --data-path goes with it. The client never reads the image, and the server knows where its own data lives. restore gains --from-cloud and --restore-point. With neither a snapshot file nor a specific point, it lists what miren.cloud holds and offers a picker; off a terminal it takes the newest, which is what a recovery almost always wants. The old image-touching implementations move to `miren debug disk backup` and `miren debug disk restore`, keeping --data-path. They are break-glass for a host whose RPC listener is down, not a second supported way to back up. --- cli/commands/commands.go | 5 + cli/commands/debug_disk_backup.go | 224 +++++++++++++++++++++++++++++ cli/commands/debug_disk_restore.go | 152 ++++++++++++++++++++ cli/commands/disk_backup.go | 218 +++++++--------------------- cli/commands/disk_progress.go | 87 +++++++++++ cli/commands/disk_restore.go | 209 ++++++++++++++------------- 6 files changed, 626 insertions(+), 269 deletions(-) create mode 100644 cli/commands/debug_disk_backup.go create mode 100644 cli/commands/debug_disk_restore.go create mode 100644 cli/commands/disk_progress.go diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 3cb80895c..ed1c7400f 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1271,6 +1271,11 @@ Warning: These commands are intended for advanced users and developers. They may d.Dispatch("debug disk lease-delete", Infer("debug disk lease-delete", "Delete a disk lease entity", DebugDiskLeaseDelete)) d.Dispatch("debug disk lease-status", Infer("debug disk lease-status", "Show detailed status of a disk lease", DebugDiskLeaseStatus)) d.Dispatch("debug disk mounts", Infer("debug disk mounts", "List all mounted disks from /proc/mounts", DebugDiskMounts)) + // Break-glass: `miren disk backup`/`restore` drive the server and are the + // supported commands. These touch the image directly, for a host whose RPC + // listener is down, and so must run on the server. + d.Dispatch("debug disk backup", Infer("debug disk backup", "Back up a disk by reading its image directly (break-glass)", DebugDiskBackup)) + d.Dispatch("debug disk restore", Infer("debug disk restore", "Restore a disk by writing its image directly (break-glass)", DebugDiskRestore)) // Debug saga commands d.Dispatch("debug saga", Section("debug saga", "Saga execution debug commands", "", WithSectionDescription(sagaSectionDescription))) diff --git a/cli/commands/debug_disk_backup.go b/cli/commands/debug_disk_backup.go new file mode 100644 index 000000000..6b67bd8ee --- /dev/null +++ b/cli/commands/debug_disk_backup.go @@ -0,0 +1,224 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/components/coordinate" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cloudauth" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/registration" + "miren.dev/runtime/pkg/snapshot" +) + +// DebugDiskBackup backs up a disk by reading its image directly, without going +// through the server. +// +// This is break-glass, not a second supported way to back up. `miren disk +// backup` drives the server over RPC and is the command to use. This one exists +// for the case that one cannot cover: the server's RPC listener being down on a +// host you can still get a shell on. It therefore has to run on the server and +// keeps --data-path. +func DebugDiskBackup(ctx *Context, opts struct { + ConfigCentric + Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` + Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + Cloud bool `long:"cloud" description:"Also upload the snapshot to miren.cloud as a restore point"` + Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` +}) (retErr error) { + if _, err := os.Stat(opts.DataPath); err != nil { + return fmt.Errorf("data path %s not found — disk backup must be run on the server", opts.DataPath) + } + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + resolver := diskresolve.New(eac, nil) + + target, err := snapshot.PrepareBackup(ctx, resolver, opts.Name, opts.DataPath) + if err != nil { + return err + } + + imgInfo, err := os.Stat(target.ImagePath) + if err != nil { + return fmt.Errorf("disk image not found at %s: %w", target.ImagePath, err) + } + + if target.IsAttached { + // Nothing here freezes the filesystem or takes a copy-on-write clone, so + // this is a sequential read of a file the loop device is still writing. + // The head and tail of the image come from different moments, which is + // weaker than the power-loss state fsck and Postgres recovery are built + // for. Say so plainly: the operator is the one deciding this is safe. + ctx.Warn("Disk is attached and may be written during the backup.") + ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") + ctx.Warn("Detach the disk first for a backup you can rely on.") + } + + outputPath := opts.Output + if outputPath == "" { + outputPath = fmt.Sprintf("%s-%s.miren.zst", opts.Name, time.Now().Format("20060102-150405")) + } + + ctx.Info("Backing up disk %q (%s)", opts.Name, formatBytes(imgInfo.Size())) + ctx.Info("Image: %s", target.ImagePath) + ctx.Info("Output: %s", outputPath) + + start := time.Now() + + imgFile, err := os.Open(target.ImagePath) + if err != nil { + return fmt.Errorf("opening image file: %w", err) + } + defer imgFile.Close() + + outFile, err := os.Create(outputPath) + if err != nil { + return fmt.Errorf("creating output file: %w", err) + } + // Set once the snapshot itself is written. A later step failing (the cloud + // upload) must not take a finished local backup down with it. + snapshotComplete := false + + defer func() { + closeErr := outFile.Close() + if closeErr != nil && retErr == nil { + retErr = fmt.Errorf("closing output file: %w", closeErr) + } + // Only discard a snapshot that never finished, or one whose close + // failed and may therefore be truncated. + if retErr != nil && (!snapshotComplete || closeErr != nil) { + os.Remove(outputPath) + } + }() + + ctx.Info("Computing checksum and compressing...") + + checksum, err := snapshot.Backup(outFile, imgFile, opts.Name, imgInfo.Size(), target.Filesystem) + if err != nil { + return err + } + + outInfo, err := outFile.Stat() + if err != nil { + return fmt.Errorf("stat output file: %w", err) + } + + snapshotComplete = true + + duration := time.Since(start) + ratio := float64(outInfo.Size()) / float64(imgInfo.Size()) * 100 + + ctx.Info("Backup complete") + ctx.Info(" Original size: %s", formatBytes(imgInfo.Size())) + ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(outInfo.Size()), ratio) + ctx.Info(" Checksum: %s", checksum) + ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) + ctx.Info(" Snapshot: %s", outputPath) + + if opts.Cloud { + updateID, err := uploadSnapshotToCloud(ctx, opts.DataPath, target, outputPath, cloudSnapshotDetails{ + Pin: opts.Pin, + ImageSize: imgInfo.Size(), + ImageChecksum: checksum, + CompressedSize: outInfo.Size(), + }) + if err != nil { + // The local snapshot is finished and valid; say where it is rather + // than leaving the operator thinking they got nothing. + ctx.Warn("Upload failed, but the local snapshot is intact at %s", outputPath) + return fmt.Errorf("uploading snapshot to miren.cloud: %w", err) + } + ctx.Info(" Uploaded: %s", updateID) + } + + return nil +} + +// cloudSnapshotDetails carries what the sidecar records about the image the +// snapshot was taken from. It mirrors what diskio.ImageSnapshotter writes, so a +// restore sees the same fields whichever path produced the update. +type cloudSnapshotDetails struct { + Pin string + ImageSize int64 + ImageChecksum string + CompressedSize int64 +} + +// uploadSnapshotToCloud sends a finished snapshot file to miren.cloud as a +// loop_image update for the disk's volume. +// +// The cluster's service account key lives in the registration this server wrote +// at enrolment, which is why this has to run on the server alongside the data +// directory. +func uploadSnapshotToCloud(ctx *Context, dataPath string, target *snapshot.BackupTarget, snapshotPath string, details cloudSnapshotDetails) (string, error) { + if target.CloudVolumeID == "" { + return "", fmt.Errorf("disk %q is not registered with miren.cloud yet; "+ + "registration happens on the next disk reconcile, so try again shortly", target.Name) + } + + reg, err := registration.LoadRegistration(filepath.Join(dataPath, "server")) + if err != nil { + return "", fmt.Errorf("loading cluster registration: %w", err) + } + if reg == nil || reg.Status != "approved" || reg.PrivateKey == "" { + return "", fmt.Errorf("this cluster is not registered with miren.cloud") + } + + keyPair, err := cloudauth.LoadKeyPairFromPEM(reg.PrivateKey) + if err != nil { + return "", fmt.Errorf("parsing cluster key: %w", err) + } + + cloudURL := reg.CloudURL + if cloudURL == "" { + cloudURL = coordinate.DefaultCloudURL + } + + authClient, err := cloudauth.NewAuthClient(cloudURL, keyPair) + if err != nil { + return "", fmt.Errorf("creating cloud auth client: %w", err) + } + + file, err := os.Open(snapshotPath) + if err != nil { + return "", fmt.Errorf("opening snapshot: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return "", fmt.Errorf("stat snapshot: %w", err) + } + + ctx.Info("Uploading to %s...", cloudURL) + + updates := diskio.NewCloudUpdatesClient(ctx.Log, cloudURL, authClient) + // ctx is the command context, so Ctrl-C interrupts the upload rather than + // leaving it running to the end of a multi-gigabyte image. + return updates.Upload(ctx, target.CloudVolumeID, diskio.UploadRequest{ + Kind: diskio.KindLoopImage, + OrderingKey: fmt.Sprintf("%016x", time.Now().UnixNano()), + SnapshotName: details.Pin, + Metadata: map[string]any{ + "compression": "zstd", + "format": "miren-snapshot", + "filesystem": target.Filesystem, + "image_size": details.ImageSize, + "image_sha256": details.ImageChecksum, + "compressed_size": details.CompressedSize, + // Records that this was taken from a live disk, so the image is a + // smear across the read rather than a point-in-time copy. + "was_attached": target.IsAttached, + }, + }, file, info.Size()) +} diff --git a/cli/commands/debug_disk_restore.go b/cli/commands/debug_disk_restore.go new file mode 100644 index 000000000..e5917de3d --- /dev/null +++ b/cli/commands/debug_disk_restore.go @@ -0,0 +1,152 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/snapshot" +) + +// DebugDiskRestore restores a disk by writing its image directly, without going +// through the server. +// +// Break-glass, for when the server's RPC listener is down; `miren disk restore` +// is the supported command. Note that this path cannot see whether the image is +// currently loop-attached, so it can rename a restored image over a live one and +// report success having changed nothing. `miren disk restore` refuses that case. +func DebugDiskRestore(ctx *Context, opts struct { + ConfigCentric + Snapshot string `short:"s" long:"snapshot" description:"Path to snapshot file" required:"true"` + Name string `short:"n" long:"name" description:"Disk name to restore to (default: original name from snapshot)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + Force bool `short:"f" long:"force" description:"Overwrite existing disk image without confirmation"` +}) (retErr error) { + if _, err := os.Stat(opts.DataPath); err != nil { + return fmt.Errorf("data path %s not found — disk restore must be run on the server", opts.DataPath) + } + + snapFile, err := os.Open(opts.Snapshot) + if err != nil { + return fmt.Errorf("opening snapshot file: %w", err) + } + defer snapFile.Close() + + meta, err := snapshot.ReadHeader(snapFile) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + + diskName := opts.Name + if diskName == "" { + diskName = meta.Name + } + + ctx.Info("Restoring from snapshot: %s", opts.Snapshot) + ctx.Info(" Original disk: %s", meta.Name) + ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Created: %s", meta.Timestamp.Format(time.RFC3339)) + ctx.Info(" Checksum: %s", meta.Checksum) + ctx.Info(" Target disk: %s", diskName) + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + ec := entityserver.NewClient(ctx.Log, eac) + resolver := diskresolve.New(eac, ec) + + target, err := snapshot.PrepareRestore(ctx, resolver, diskName, opts.DataPath, + snapshot.WithCreator(resolver, meta.SizeBytes, meta.Filesystem), + ) + if err != nil { + return err + } + + // If the disk was freshly created and anything fails, clean up the + // RESTORING disk entity so it doesn't become a zombie. + if target.Created && target.Cleanup != nil { + defer func() { + if retErr != nil { + ctx.Warn("Cleaning up disk entity after failed restore") + if cerr := target.Cleanup(ctx); cerr != nil { + ctx.Warn("Failed to clean up disk entity: %v", cerr) + } + } + }() + } + + if !target.Created { + if _, err := os.Stat(target.ImagePath); err == nil { + if !opts.Force { + return fmt.Errorf("disk image already exists at %s — use --force to overwrite", target.ImagePath) + } + ctx.Warn("Overwriting existing disk image at %s", target.ImagePath) + } else if !os.IsNotExist(err) { + return fmt.Errorf("checking existing image at %s: %w", target.ImagePath, err) + } + } + + ctx.Info("Restoring to: %s", target.ImagePath) + + start := time.Now() + + if err := os.MkdirAll(filepath.Dir(target.ImagePath), 0o755); err != nil { + return fmt.Errorf("creating image directory: %w", err) + } + + tmpPath := target.ImagePath + ".restore.tmp" + outFile, err := os.Create(tmpPath) + if err != nil { + return fmt.Errorf("creating temp image file: %w", err) + } + defer func() { + outFile.Close() + if retErr != nil { + os.Remove(tmpPath) + } + }() + + if err := outFile.Truncate(meta.SizeBytes); err != nil { + return fmt.Errorf("truncating image file: %w", err) + } + + ctx.Info("Decompressing...") + + if err := snapshot.RestoreImage(outFile, snapFile, meta); err != nil { + return err + } + + if err := outFile.Close(); err != nil { + return fmt.Errorf("closing restored image: %w", err) + } + + if err := os.Rename(tmpPath, target.ImagePath); err != nil { + return fmt.Errorf("moving restored image into place: %w", err) + } + + if target.Finalize != nil { + ctx.Info("Finalizing disk entities...") + if err := target.Finalize(ctx); err != nil { + return fmt.Errorf("finalizing restore: %w", err) + } + } + + duration := time.Since(start) + + ctx.Info("Restore complete") + ctx.Info(" Disk: %s", diskName) + ctx.Info(" Image: %s", target.ImagePath) + ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) + ctx.Info(" Checksum: verified") + ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) + + return nil +} diff --git a/cli/commands/disk_backup.go b/cli/commands/disk_backup.go index 646063d2f..6a0f6bc66 100644 --- a/cli/commands/disk_backup.go +++ b/cli/commands/disk_backup.go @@ -3,58 +3,48 @@ package commands import ( "fmt" "os" - "path/filepath" "time" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/components/coordinate" - "miren.dev/runtime/components/diskio" - "miren.dev/runtime/pkg/cloudauth" - "miren.dev/runtime/pkg/diskresolve" - "miren.dev/runtime/pkg/registration" - "miren.dev/runtime/pkg/snapshot" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/rpc/stream" ) -// DiskBackup backs up a disk to a compressed snapshot file. +// DiskBackup backs up a disk, driving the server over RPC. +// +// There is one path here whether you run this on the server host or on your +// laptop, so there is no local-versus-remote detection and no failure mode where +// that detection guesses wrong. The image bytes and the cluster's miren.cloud +// key both stay on the server; see RFD-108. func DiskBackup(ctx *Context, opts struct { ConfigCentric - Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` - Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` - Cloud bool `long:"cloud" description:"Also upload the snapshot to miren.cloud as a restore point"` - Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` + Name string `short:"n" long:"name" description:"Disk name to backup" required:"true"` + Output string `short:"o" long:"output" description:"Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst)"` + Cloud bool `long:"cloud" description:"Upload the snapshot to miren.cloud as a restore point instead of writing a local file"` + Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` }) (retErr error) { - if _, err := os.Stat(opts.DataPath); err != nil { - return fmt.Errorf("data path %s not found — disk backup must be run on the server", opts.DataPath) - } - - client, err := ctx.RPCClient("entities") + client, err := ctx.RPCClient(diskBackupService) if err != nil { return err } + dc := disk_v1alpha.NewDiskBackupClient(client) - eac := entityserver_v1alpha.NewEntityAccessClient(client) - resolver := diskresolve.New(eac, nil) - - target, err := snapshot.PrepareBackup(ctx, resolver, opts.Name, opts.DataPath) - if err != nil { - return err + if opts.Pin != "" && !opts.Cloud { + return fmt.Errorf("--pin names a restore point in miren.cloud, so it only applies with --cloud") } - imgInfo, err := os.Stat(target.ImagePath) - if err != nil { - return fmt.Errorf("disk image not found at %s: %w", target.ImagePath, err) - } + start := time.Now() + progress := diskProgress(ctx) + + if opts.Cloud { + ctx.Info("Backing up disk %q to miren.cloud", opts.Name) - if target.IsAttached { - // Nothing here freezes the filesystem or takes a copy-on-write clone, so - // this is a sequential read of a file the loop device is still writing. - // The head and tail of the image come from different moments, which is - // weaker than the power-loss state fsck and Postgres recovery are built - // for. Say so plainly: the operator is the one deciding this is safe. - ctx.Warn("Disk is attached and may be written during the backup.") - ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") - ctx.Warn("Detach the disk first for a backup you can rely on.") + res, err := dc.Backup(ctx, opts.Name, true, opts.Pin, nil, progress) + if err != nil { + return err + } + reportBackup(ctx, res.Result(), time.Since(start)) + ctx.Info(" Restore point: %s", res.Result().RestorePointId()) + return nil } outputPath := opts.Output @@ -62,156 +52,50 @@ func DiskBackup(ctx *Context, opts struct { outputPath = fmt.Sprintf("%s-%s.miren.zst", opts.Name, time.Now().Format("20060102-150405")) } - ctx.Info("Backing up disk %q (%s)", opts.Name, formatBytes(imgInfo.Size())) - ctx.Info("Image: %s", target.ImagePath) - ctx.Info("Output: %s", outputPath) - - start := time.Now() - - imgFile, err := os.Open(target.ImagePath) - if err != nil { - return fmt.Errorf("opening image file: %w", err) - } - defer imgFile.Close() - outFile, err := os.Create(outputPath) if err != nil { return fmt.Errorf("creating output file: %w", err) } - // Set once the snapshot itself is written. A later step failing (the cloud - // upload) must not take a finished local backup down with it. - snapshotComplete := false + // Discard a snapshot that never finished. A truncated .miren.zst that looks + // like a backup is worse than no file at all. + complete := false defer func() { closeErr := outFile.Close() if closeErr != nil && retErr == nil { retErr = fmt.Errorf("closing output file: %w", closeErr) } - // Only discard a snapshot that never finished, or one whose close - // failed and may therefore be truncated. - if retErr != nil && (!snapshotComplete || closeErr != nil) { + if retErr != nil || !complete { os.Remove(outputPath) } }() - ctx.Info("Computing checksum and compressing...") + ctx.Info("Backing up disk %q", opts.Name) + ctx.Info("Output: %s", outputPath) - checksum, err := snapshot.Backup(outFile, imgFile, opts.Name, imgInfo.Size(), target.Filesystem) + res, err := dc.Backup(ctx, opts.Name, false, "", stream.ServeWriter(ctx, outFile), progress) if err != nil { return err } + complete = true - outInfo, err := outFile.Stat() - if err != nil { - return fmt.Errorf("stat output file: %w", err) - } - - snapshotComplete = true - - duration := time.Since(start) - ratio := float64(outInfo.Size()) / float64(imgInfo.Size()) * 100 - - ctx.Info("Backup complete") - ctx.Info(" Original size: %s", formatBytes(imgInfo.Size())) - ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(outInfo.Size()), ratio) - ctx.Info(" Checksum: %s", checksum) - ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) + reportBackup(ctx, res.Result(), time.Since(start)) ctx.Info(" Snapshot: %s", outputPath) - - if opts.Cloud { - updateID, err := uploadSnapshotToCloud(ctx, opts.DataPath, target, outputPath, cloudSnapshotDetails{ - Pin: opts.Pin, - ImageSize: imgInfo.Size(), - ImageChecksum: checksum, - CompressedSize: outInfo.Size(), - }) - if err != nil { - // The local snapshot is finished and valid; say where it is rather - // than leaving the operator thinking they got nothing. - ctx.Warn("Upload failed, but the local snapshot is intact at %s", outputPath) - return fmt.Errorf("uploading snapshot to miren.cloud: %w", err) - } - ctx.Info(" Uploaded: %s", updateID) - } - return nil } -// cloudSnapshotDetails carries what the sidecar records about the image the -// snapshot was taken from. It mirrors what diskio.ImageSnapshotter writes, so a -// restore sees the same fields whichever path produced the update. -type cloudSnapshotDetails struct { - Pin string - ImageSize int64 - ImageChecksum string - CompressedSize int64 -} - -// uploadSnapshotToCloud sends a finished snapshot file to miren.cloud as a -// loop_image update for the disk's volume. -// -// The cluster's service account key lives in the registration this server wrote -// at enrolment, which is why this has to run on the server alongside the data -// directory. -func uploadSnapshotToCloud(ctx *Context, dataPath string, target *snapshot.BackupTarget, snapshotPath string, details cloudSnapshotDetails) (string, error) { - if target.CloudVolumeID == "" { - return "", fmt.Errorf("disk %q is not registered with miren.cloud yet; "+ - "registration happens on the next disk reconcile, so try again shortly", target.Name) - } - - reg, err := registration.LoadRegistration(filepath.Join(dataPath, "server")) - if err != nil { - return "", fmt.Errorf("loading cluster registration: %w", err) - } - if reg == nil || reg.Status != "approved" || reg.PrivateKey == "" { - return "", fmt.Errorf("this cluster is not registered with miren.cloud") - } - - keyPair, err := cloudauth.LoadKeyPairFromPEM(reg.PrivateKey) - if err != nil { - return "", fmt.Errorf("parsing cluster key: %w", err) - } - - cloudURL := reg.CloudURL - if cloudURL == "" { - cloudURL = coordinate.DefaultCloudURL - } - - authClient, err := cloudauth.NewAuthClient(cloudURL, keyPair) - if err != nil { - return "", fmt.Errorf("creating cloud auth client: %w", err) - } - - file, err := os.Open(snapshotPath) - if err != nil { - return "", fmt.Errorf("opening snapshot: %w", err) - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - return "", fmt.Errorf("stat snapshot: %w", err) - } - - ctx.Info("Uploading to %s...", cloudURL) - - updates := diskio.NewCloudUpdatesClient(ctx.Log, cloudURL, authClient) - // ctx is the command context, so Ctrl-C interrupts the upload rather than - // leaving it running to the end of a multi-gigabyte image. - return updates.Upload(ctx, target.CloudVolumeID, diskio.UploadRequest{ - Kind: diskio.KindLoopImage, - OrderingKey: fmt.Sprintf("%016x", time.Now().UnixNano()), - SnapshotName: details.Pin, - Metadata: map[string]any{ - "compression": "zstd", - "format": "miren-snapshot", - "filesystem": target.Filesystem, - "image_size": details.ImageSize, - "image_sha256": details.ImageChecksum, - "compressed_size": details.CompressedSize, - // Records that this was taken from a live disk, so the image is a - // smear across the read rather than a point-in-time copy. - "was_attached": target.IsAttached, - }, - }, file, info.Size()) +func reportBackup(ctx *Context, res *disk_v1alpha.BackupResult, took time.Duration) { + ctx.Info("Backup complete") + if res == nil { + return + } + ctx.Info(" Original size: %s", formatBytes(res.ImageSizeBytes())) + if res.ImageSizeBytes() > 0 { + ratio := float64(res.CompressedSizeBytes()) / float64(res.ImageSizeBytes()) * 100 + ctx.Info(" Compressed size: %s (%.1f%%)", formatBytes(res.CompressedSizeBytes()), ratio) + } else { + ctx.Info(" Compressed size: %s", formatBytes(res.CompressedSizeBytes())) + } + ctx.Info(" Checksum: %s", res.Checksum()) + ctx.Info(" Duration: %s", took.Truncate(time.Millisecond)) } diff --git a/cli/commands/disk_progress.go b/cli/commands/disk_progress.go new file mode 100644 index 000000000..ea4267a24 --- /dev/null +++ b/cli/commands/disk_progress.go @@ -0,0 +1,87 @@ +package commands + +import ( + "fmt" + "os" + "time" + + "golang.org/x/term" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/progress/upload" + "miren.dev/runtime/pkg/rpc/stream" +) + +const diskBackupService = "dev.miren.runtime/disk-backup" + +// diskProgress renders the server's progress events for backup and restore. +// +// Disk images are large enough that a silent wait is indistinguishable from a +// hang, so transfer events get a line that updates in place on a terminal. Off a +// terminal the same events are printed one per line and throttled, so a log does +// not fill with thousands of near-identical rows. +func diskProgress(ctx *Context) stream.SendStream[*disk_v1alpha.Progress] { + tty := term.IsTerminal(int(os.Stdout.Fd())) + var lastLine time.Time + inPlace := false + + // Clear a partially written in-place line before printing anything else, so + // a message never lands on top of a progress bar. + clear := func() { + if inPlace { + fmt.Fprint(os.Stdout, "\r\033[K") + inPlace = false + } + } + + return stream.Callback(func(p *disk_v1alpha.Progress) error { + switch p.Update().Which() { + case "message": + clear() + ctx.Info("%s", p.Update().Message()) + + case "warning": + clear() + ctx.Warn("%s", p.Update().Warning()) + + case "error": + clear() + ctx.Warn("%s", p.Update().Error()) + + case "transfer": + t := p.Update().Transfer() + if t == nil { + return nil + } + line := formatTransfer(t) + + if tty { + fmt.Fprintf(os.Stdout, "\r\033[K %s", line) + inPlace = true + return nil + } + // One line per half second is enough to show life in a log. + if now := time.Now(); now.Sub(lastLine) >= 500*time.Millisecond { + lastLine = now + ctx.Info(" %s", line) + } + } + return nil + }) +} + +func formatTransfer(t *disk_v1alpha.Transfer) string { + moved := upload.FormatBytes(t.Done()) + speed := upload.FormatSpeed(float64(t.BytesPerSecond())) + + if t.Total() <= 0 { + return fmt.Sprintf("%s at %s", moved, speed) + } + + pct := float64(t.Done()) / float64(t.Total()) * 100 + out := fmt.Sprintf("%s of %s (%.0f%%) at %s", + moved, upload.FormatBytes(t.Total()), pct, speed) + if eta := t.EtaSeconds(); eta > 0 { + out += fmt.Sprintf(", %s left", upload.FormatDuration(time.Duration(eta)*time.Second)) + } + return out +} diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index 89dd9d15f..ec2be9d62 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -3,144 +3,149 @@ package commands import ( "fmt" "os" - "path/filepath" + "time" - "miren.dev/runtime/api/entityserver" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/pkg/diskresolve" - "miren.dev/runtime/pkg/snapshot" + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/pkg/progress/upload" + "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/ui" ) -// DiskRestore restores a disk from a compressed snapshot file. +// DiskRestore restores a disk, driving the server over RPC. +// +// Either from a local snapshot file, which is streamed up, or from a restore +// point in miren.cloud, which the server downloads itself. Restoring a disk this +// cluster has never seen creates it, so this works on a freshly built host with +// nothing on it — which is the whole point, since disaster recovery means the +// original host is gone. func DiskRestore(ctx *Context, opts struct { ConfigCentric - Snapshot string `short:"s" long:"snapshot" description:"Path to snapshot file" required:"true"` - Name string `short:"n" long:"name" description:"Disk name to restore to (default: original name from snapshot)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` - Force bool `short:"f" long:"force" description:"Overwrite existing disk image without confirmation"` -}) (retErr error) { - if _, err := os.Stat(opts.DataPath); err != nil { - return fmt.Errorf("data path %s not found — disk restore must be run on the server", opts.DataPath) - } - - snapFile, err := os.Open(opts.Snapshot) + Snapshot string `short:"s" long:"snapshot" description:"Path to a snapshot file to restore from"` + Name string `short:"n" long:"name" description:"Disk name to restore to" required:"true"` + FromCloud bool `long:"from-cloud" description:"Restore from a miren.cloud restore point"` + RestorePoint string `long:"restore-point" description:"Restore point to use (implies --from-cloud; default: the newest)"` + Force bool `short:"f" long:"force" description:"Overwrite an existing disk image"` +}) error { + client, err := ctx.RPCClient(diskBackupService) if err != nil { - return fmt.Errorf("opening snapshot file: %w", err) + return err } - defer snapFile.Close() + dc := disk_v1alpha.NewDiskBackupClient(client) - meta, err := snapshot.ReadHeader(snapFile) - if err != nil { - return fmt.Errorf("reading snapshot header: %w", err) + fromCloud := opts.FromCloud || opts.RestorePoint != "" + if opts.Snapshot == "" && !fromCloud { + return fmt.Errorf("restore needs either --snapshot to read a local file or --from-cloud to fetch a restore point") } - - diskName := opts.Name - if diskName == "" { - diskName = meta.Name + if opts.Snapshot != "" && fromCloud { + return fmt.Errorf("pass either --snapshot or --from-cloud, not both") } - ctx.Info("Restoring from snapshot: %s", opts.Snapshot) - ctx.Info(" Original disk: %s", meta.Name) - ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Created: %s", meta.Timestamp.Format(time.RFC3339)) - ctx.Info(" Checksum: %s", meta.Checksum) - ctx.Info(" Target disk: %s", diskName) + start := time.Now() + progress := diskProgress(ctx) + + if fromCloud { + point := opts.RestorePoint + if point == "" { + point, err = pickRestorePoint(ctx, dc, opts.Name) + if err != nil { + return err + } + } - client, err := ctx.RPCClient("entities") - if err != nil { - return err - } + ctx.Info("Restoring disk %q from restore point %s", opts.Name, point) - eac := entityserver_v1alpha.NewEntityAccessClient(client) - ec := entityserver.NewClient(ctx.Log, eac) - resolver := diskresolve.New(eac, ec) + res, err := dc.Restore(ctx, opts.Name, point, nil, opts.Force, progress) + if err != nil { + return err + } + reportRestore(ctx, res.Result(), time.Since(start)) + return nil + } - target, err := snapshot.PrepareRestore(ctx, resolver, diskName, opts.DataPath, - snapshot.WithCreator(resolver, meta.SizeBytes, meta.Filesystem), - ) + snapFile, err := os.Open(opts.Snapshot) if err != nil { - return err + return fmt.Errorf("opening snapshot: %w", err) } + defer snapFile.Close() - // If the disk was freshly created and anything fails, clean up the - // RESTORING disk entity so it doesn't become a zombie. - if target.Created && target.Cleanup != nil { - defer func() { - if retErr != nil { - ctx.Warn("Cleaning up disk entity after failed restore") - if cerr := target.Cleanup(ctx); cerr != nil { - ctx.Warn("Failed to clean up disk entity: %v", cerr) - } - } - }() - } + ctx.Info("Restoring disk %q from %s", opts.Name, opts.Snapshot) - if !target.Created { - if _, err := os.Stat(target.ImagePath); err == nil { - if !opts.Force { - return fmt.Errorf("disk image already exists at %s — use --force to overwrite", target.ImagePath) - } - ctx.Warn("Overwriting existing disk image at %s", target.ImagePath) - } else if !os.IsNotExist(err) { - return fmt.Errorf("checking existing image at %s: %w", target.ImagePath, err) - } + res, err := dc.Restore(ctx, opts.Name, "", stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), opts.Force, progress) + if err != nil { + return err } + reportRestore(ctx, res.Result(), time.Since(start)) + return nil +} - ctx.Info("Restoring to: %s", target.ImagePath) +// restorePointItem adapts a restore point to the shared picker. +type restorePointItem struct { + point *disk_v1alpha.RestorePoint +} - start := time.Now() +func (r restorePointItem) ID() string { return r.point.Id() } - if err := os.MkdirAll(filepath.Dir(target.ImagePath), 0o755); err != nil { - return fmt.Errorf("creating image directory: %w", err) +func (r restorePointItem) Row() []string { + when := "unknown" + if r.point.HasCreatedAt() { + ts := r.point.CreatedAt() + when = time.Unix(ts.Seconds(), int64(ts.Nanoseconds())).Format("2006-01-02 15:04:05") + } + return []string{ + when, + upload.FormatBytes(r.point.SizeBytes()), + r.point.Name(), + r.point.Id(), } +} - tmpPath := target.ImagePath + ".restore.tmp" - outFile, err := os.Create(tmpPath) +// pickRestorePoint lists what is available and asks, so an operator recovering a +// disk does not have to already know a restore point id. +func pickRestorePoint(ctx *Context, dc *disk_v1alpha.DiskBackupClient, name string) (string, error) { + res, err := dc.ListBackups(ctx, name) if err != nil { - return fmt.Errorf("creating temp image file: %w", err) + return "", err } - defer func() { - outFile.Close() - if retErr != nil { - os.Remove(tmpPath) - } - }() - if err := outFile.Truncate(meta.SizeBytes); err != nil { - return fmt.Errorf("truncating image file: %w", err) + points := res.Points() + if len(points) == 0 { + return "", fmt.Errorf("disk %q has no restore points in miren.cloud", name) } - ctx.Info("Decompressing...") - - if err := snapshot.RestoreImage(outFile, snapFile, meta); err != nil { - return err + // The server returns newest first. On a non-interactive run take that one: + // it is what a recovery almost always wants, and there is nobody to ask. + if !ui.IsInteractive() { + ctx.Info("Using the newest restore point (%s of %d)", points[0].Id(), len(points)) + return points[0].Id(), nil } - if err := outFile.Close(); err != nil { - return fmt.Errorf("closing restored image: %w", err) + items := make([]ui.PickerItem, 0, len(points)) + for _, p := range points { + items = append(items, restorePointItem{point: p}) } - if err := os.Rename(tmpPath, target.ImagePath); err != nil { - return fmt.Errorf("moving restored image into place: %w", err) + chosen, err := ui.RunPicker(items, + ui.WithTitle(fmt.Sprintf("Restore point for %q", name)), + ui.WithHeaders([]string{"Taken", "Size", "Pinned as", "ID"}), + ) + if err != nil { + return "", err } - - if target.Finalize != nil { - ctx.Info("Finalizing disk entities...") - if err := target.Finalize(ctx); err != nil { - return fmt.Errorf("finalizing restore: %w", err) - } + if chosen == nil { + return "", fmt.Errorf("no restore point selected") } + return chosen.ID(), nil +} - duration := time.Since(start) - +func reportRestore(ctx *Context, res *disk_v1alpha.RestoreResult, took time.Duration) { ctx.Info("Restore complete") - ctx.Info(" Disk: %s", diskName) - ctx.Info(" Image: %s", target.ImagePath) - ctx.Info(" Size: %s", formatBytes(meta.SizeBytes)) - ctx.Info(" Checksum: verified") - ctx.Info(" Duration: %s", duration.Truncate(time.Millisecond)) - - return nil + if res == nil { + return + } + if res.Created() { + ctx.Info(" Created disk: %s", res.Disk()) + } + ctx.Info(" Restored size: %s", formatBytes(res.ImageSizeBytes())) + ctx.Info(" Duration: %s", took.Truncate(time.Millisecond)) } From 98c7e87534969002b97363348fb06e51811e1b62 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:25:28 -0700 Subject: [PATCH 05/19] Update the docs that said backup was server-only Both pages promised backup must be run over SSH on the server, and disks.md listed remote backup on the roadmap. Neither is true now. Two corrections while here. addons.md claimed a backup taken while the disk is in use is crash-consistent and therefore safe for PostgreSQL. It is not: nothing freezes the filesystem, so the head and tail of the image come from different moments, which is a state the disk never actually held. And restore of an attached disk now refuses rather than silently doing nothing, which the restore section needs to say. Command docs regenerated. --- cli/commands/disk_restore.go | 31 +++++++++++++++++++--- docs/command-sidebar.json | 2 ++ docs/docs/addons.md | 28 +++++++++++++------- docs/docs/command/debug-disk-backup.md | 35 +++++++++++++++++++++++++ docs/docs/command/debug-disk-restore.md | 34 ++++++++++++++++++++++++ docs/docs/command/debug-disk.md | 2 ++ docs/docs/command/disk-backup.md | 3 +-- docs/docs/command/disk-restore.md | 9 ++++--- docs/docs/commands.md | 2 ++ docs/docs/disks.md | 4 +-- 10 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 docs/docs/command/debug-disk-backup.md create mode 100644 docs/docs/command/debug-disk-restore.md diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index ec2be9d62..bddabd79f 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "io" "os" "time" @@ -9,6 +10,7 @@ import ( "miren.dev/runtime/api/disk/disk_v1alpha" "miren.dev/runtime/pkg/progress/upload" "miren.dev/runtime/pkg/rpc/stream" + "miren.dev/runtime/pkg/snapshot" "miren.dev/runtime/pkg/ui" ) @@ -22,7 +24,7 @@ import ( func DiskRestore(ctx *Context, opts struct { ConfigCentric Snapshot string `short:"s" long:"snapshot" description:"Path to a snapshot file to restore from"` - Name string `short:"n" long:"name" description:"Disk name to restore to" required:"true"` + Name string `short:"n" long:"name" description:"Disk name to restore to (default: the name recorded in the snapshot)"` FromCloud bool `long:"from-cloud" description:"Restore from a miren.cloud restore point"` RestorePoint string `long:"restore-point" description:"Restore point to use (implies --from-cloud; default: the newest)"` Force bool `short:"f" long:"force" description:"Overwrite an existing disk image"` @@ -45,6 +47,10 @@ func DiskRestore(ctx *Context, opts struct { progress := diskProgress(ctx) if fromCloud { + if opts.Name == "" { + return fmt.Errorf("--name says which disk's restore points to look at, so it is required with --from-cloud") + } + point := opts.RestorePoint if point == "" { point, err = pickRestorePoint(ctx, dc, opts.Name) @@ -69,9 +75,28 @@ func DiskRestore(ctx *Context, opts struct { } defer snapFile.Close() - ctx.Info("Restoring disk %q from %s", opts.Name, opts.Snapshot) + name := opts.Name + if name == "" { + // The snapshot records the disk it came from, so restoring one back + // where it belongs needs no --name. The server reads the header again + // for the size and filesystem; this read is only to answer "which + // disk", which the client has to know before it can ask. + meta, err := snapshot.ReadHeader(snapFile) + if err != nil { + return fmt.Errorf("reading snapshot header: %w", err) + } + name = meta.Name + if name == "" { + return fmt.Errorf("%s records no disk name, so pass --name", opts.Snapshot) + } + if _, err := snapFile.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("rewinding snapshot: %w", err) + } + } + + ctx.Info("Restoring disk %q from %s", name, opts.Snapshot) - res, err := dc.Restore(ctx, opts.Name, "", stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), opts.Force, progress) + res, err := dc.Restore(ctx, name, "", stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), opts.Force, progress) if err != nil { return err } diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index 293642d95..b09b4c856 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -123,6 +123,7 @@ "command/debug-ctr", "command/debug-ctr-nuke", "command/debug-disk", + "command/debug-disk-backup", "command/debug-disk-create", "command/debug-disk-delete", "command/debug-disk-lease", @@ -132,6 +133,7 @@ "command/debug-disk-lease-status", "command/debug-disk-list", "command/debug-disk-mounts", + "command/debug-disk-restore", "command/debug-disk-status", "command/debug-entity", "command/debug-entity-create", diff --git a/docs/docs/addons.md b/docs/docs/addons.md index 97d360fe1..e0d65beaa 100644 --- a/docs/docs/addons.md +++ b/docs/docs/addons.md @@ -371,7 +371,7 @@ Miren provisions PostgreSQL, injects `DATABASE_URL`, and starts your app once th :::info[Early version] Addon backup and restore uses the general-purpose disk backup system. We plan to add addon-aware backup commands in a future release that will simplify this workflow. For now, the steps below work reliably for PostgreSQL addon data. -The `disk backup` and `disk restore` commands must be run directly on the server (via SSH or `miren ssh`), not from your local machine. Remote backup support is planned. +The `disk backup` and `disk restore` commands run from anywhere you can reach the cluster, including your own machine. The server does the work; the snapshot is streamed to or from you. ::: Each PostgreSQL addon stores its data on a Miren disk. You can back up and restore this disk using the `miren disk backup` and `miren disk restore` commands. @@ -390,19 +390,23 @@ Addon disks are named with a `pg-` prefix. For dedicated (`small`) addons, the n ### Creating a Backup -Back up the disk to a compressed snapshot file. This must be run on the server: +Back up the disk to a compressed snapshot file: - + ```miren miren disk backup -n ``` -This creates a timestamped `.miren.zst` file in the current directory. If the disk is currently in use, the backup will be crash-consistent (safe for PostgreSQL, which uses write-ahead logging). +This creates a timestamped `.miren.zst` file in the current directory. + +:::warning[Detach the disk first] +Nothing freezes the filesystem while the snapshot is read, so backing up a disk that is currently attached reads it as it is being written. The head and tail of the image come from different moments, which is weaker than the power-loss state PostgreSQL's write-ahead log recovery is built for. The command warns you when it detects this. Stop the services using the disk for a backup you can rely on. +::: Example: - + ```miren miren disk backup -n pg-pg-myapp-sCZDabc123-data # Output: pg-pg-myapp-sCZDabc123-data-20260324-120000.miren.zst @@ -411,7 +415,7 @@ miren disk backup -n pg-pg-myapp-sCZDabc123-data You can specify a custom output path with `-o`: - + ```miren miren disk backup -n pg-pg-myapp-sCZDabc123-data -o /backups/myapp-db.miren.zst ``` @@ -419,9 +423,9 @@ miren disk backup -n pg-pg-myapp-sCZDabc123-data -o /backups/myapp-db.miren.zst ### Restoring from a Backup -To restore from a backup, provide the snapshot file. This must also be run on the server: +To restore from a backup, provide the snapshot file: - + ```miren miren disk restore -s ``` @@ -429,7 +433,7 @@ miren disk restore -s The restore procedure recreates the disk with the original name. If the disk already exists, use `--force` to overwrite: - + ```miren miren disk restore -s myapp-db.miren.zst --force ``` @@ -437,12 +441,16 @@ miren disk restore -s myapp-db.miren.zst --force To restore to a different disk name: - + ```miren miren disk restore -s myapp-db.miren.zst -n new-disk-name ``` +:::warning[Stop the app before restoring] +Restore refuses to write over a disk that is still attached. A disk in use is held open by the kernel, so a restore into it would leave the running database on the old data while reporting success — the command stops rather than let that happen. Scale the app to zero, or restore into a new disk name and switch over. +::: + After restoring, restart your app to pick up the restored data: diff --git a/docs/docs/command/debug-disk-backup.md b/docs/docs/command/debug-disk-backup.md new file mode 100644 index 000000000..182745165 --- /dev/null +++ b/docs/docs/command/debug-disk-backup.md @@ -0,0 +1,35 @@ +--- +title: "miren debug disk backup" +sidebar_label: "debug disk backup" +description: "Back up a disk by reading its image directly (break-glass)" +--- + +# miren debug disk backup + +Back up a disk by reading its image directly (break-glass) + +## Usage + +```bash +miren debug disk backup [flags] +``` + +## Flags + +- `--cloud` — Also upload the snapshot to miren.cloud as a restore point +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--name, -n` — Disk name to backup +- `--output, -o` — Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst) +- `--pin` — Name the uploaded restore point, pinning it against cleanup + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](/command/debug-disk) diff --git a/docs/docs/command/debug-disk-restore.md b/docs/docs/command/debug-disk-restore.md new file mode 100644 index 000000000..72230494c --- /dev/null +++ b/docs/docs/command/debug-disk-restore.md @@ -0,0 +1,34 @@ +--- +title: "miren debug disk restore" +sidebar_label: "debug disk restore" +description: "Restore a disk by writing its image directly (break-glass)" +--- + +# miren debug disk restore + +Restore a disk by writing its image directly (break-glass) + +## Usage + +```bash +miren debug disk restore [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--force, -f` — Overwrite existing disk image without confirmation +- `--name, -n` — Disk name to restore to (default: original name from snapshot) +- `--snapshot, -s` — Path to snapshot file + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](/command/debug-disk) diff --git a/docs/docs/command/debug-disk.md b/docs/docs/command/debug-disk.md index b08f299e2..0b06873d4 100644 --- a/docs/docs/command/debug-disk.md +++ b/docs/docs/command/debug-disk.md @@ -55,6 +55,7 @@ miren debug disk [flags] ## Subcommands +- [`miren debug disk backup`](/command/debug-disk-backup) — Back up a disk by reading its image directly (break-glass) - [`miren debug disk create`](/command/debug-disk-create) — Create a disk entity for testing - [`miren debug disk delete`](/command/debug-disk-delete) — Delete a disk entity - [`miren debug disk lease`](/command/debug-disk-lease) — Create a disk lease for testing @@ -64,6 +65,7 @@ miren debug disk [flags] - [`miren debug disk lease-status`](/command/debug-disk-lease-status) — Show detailed status of a disk lease - [`miren debug disk list`](/command/debug-disk-list) — List all disk entities - [`miren debug disk mounts`](/command/debug-disk-mounts) — List all mounted disks from /proc/mounts +- [`miren debug disk restore`](/command/debug-disk-restore) — Restore a disk by writing its image directly (break-glass) - [`miren debug disk status`](/command/debug-disk-status) — Show status of a disk entity ## See also diff --git a/docs/docs/command/disk-backup.md b/docs/docs/command/disk-backup.md index 0b388b897..f1ae161f5 100644 --- a/docs/docs/command/disk-backup.md +++ b/docs/docs/command/disk-backup.md @@ -16,10 +16,9 @@ miren disk backup [flags] ## Flags -- `--cloud` — Also upload the snapshot to miren.cloud as a restore point +- `--cloud` — Upload the snapshot to miren.cloud as a restore point instead of writing a local file - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--name, -n` — Disk name to backup - `--output, -o` — Output snapshot path (default: DISK-YYYYMMDD-HHMMSS.miren.zst) - `--pin` — Name the uploaded restore point, pinning it against cleanup diff --git a/docs/docs/command/disk-restore.md b/docs/docs/command/disk-restore.md index 5a20208d2..3ae1eaa6d 100644 --- a/docs/docs/command/disk-restore.md +++ b/docs/docs/command/disk-restore.md @@ -18,10 +18,11 @@ miren disk restore [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) -- `--force, -f` — Overwrite existing disk image without confirmation -- `--name, -n` — Disk name to restore to (default: original name from snapshot) -- `--snapshot, -s` — Path to snapshot file +- `--force, -f` — Overwrite an existing disk image +- `--from-cloud` — Restore from a miren.cloud restore point +- `--name, -n` — Disk name to restore to (default: the name recorded in the snapshot) +- `--restore-point` — Restore point to use (implies --from-cloud; default: the newest) +- `--snapshot, -s` — Path to a snapshot file to restore from ## Global Options diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 3fa2c070b..f5123aff6 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -312,6 +312,7 @@ These commands are intended for advanced debugging and troubleshooting. They may | [`miren debug ctr`](/command/debug-ctr) | Run ctr with miren defaults | | [`miren debug ctr nuke`](/command/debug-ctr-nuke) | Nuke a containerd namespace | | [`miren debug disk`](/command/debug-disk) | Disk entity debug commands | +| [`miren debug disk backup`](/command/debug-disk-backup) | Back up a disk by reading its image directly (break-glass) | | [`miren debug disk create`](/command/debug-disk-create) | Create a disk entity for testing | | [`miren debug disk delete`](/command/debug-disk-delete) | Delete a disk entity | | [`miren debug disk lease`](/command/debug-disk-lease) | Create a disk lease for testing | @@ -321,6 +322,7 @@ These commands are intended for advanced debugging and troubleshooting. They may | [`miren debug disk lease-status`](/command/debug-disk-lease-status) | Show detailed status of a disk lease | | [`miren debug disk list`](/command/debug-disk-list) | List all disk entities | | [`miren debug disk mounts`](/command/debug-disk-mounts) | List all mounted disks from /proc/mounts | +| [`miren debug disk restore`](/command/debug-disk-restore) | Restore a disk by writing its image directly (break-glass) | | [`miren debug disk status`](/command/debug-disk-status) | Show status of a disk entity | | [`miren debug entity`](/command/debug-entity) | Entity store debug commands | | [`miren debug entity create`](/command/debug-entity-create) | Create a new entity | diff --git a/docs/docs/disks.md b/docs/docs/disks.md index 41a26af1a..53adeda11 100644 --- a/docs/docs/disks.md +++ b/docs/docs/disks.md @@ -98,7 +98,7 @@ If any of your environment variables reference `/miren/data/local`, Miren will a ## Miren Disks :::note[Backups] -Miren Disks live on your server. Back up important data with `miren disk backup` and restore it with `miren disk restore`. Cloud backup is on the [roadmap](#roadmap-cloud-backup--sync). +Miren Disks live on your server. Back up important data with `miren disk backup` and restore it with `miren disk restore`, from your own machine or from the server. Add `--cloud` to store the backup in Miren Cloud instead of a local file. ::: Miren Disks provide managed persistent storage for your applications. Disks are provisioned with a specific size and filesystem, support exclusive leasing for data consistency, and persist across app restarts and redeployments. @@ -309,7 +309,7 @@ Your server must have the mkfs tools to format the disk types. We're building toward cloud-connected storage for Miren Disks. Here's what's planned: -- **Remote backup & restore** (next up): Trigger backups of your disks to Miren Cloud and restore them on any cluster. This extends the existing local backup/restore functionality to work remotely. +- **Remote backup & restore** (shipped): `miren disk backup` and `miren disk restore` run from any machine that can reach the cluster. Add `--cloud` to store a backup in Miren Cloud as a restore point, and `--from-cloud` to restore from one. - **Automatic cloud sync**: Background replication of disk data to Miren Cloud, enabling seamless portability across clusters. We'll update this page and the [changelog](https://miren.md/changelog) as these capabilities land. From c5f5e0030cf77c6d529824ae22b1372fc5a92f36 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:26:25 -0700 Subject: [PATCH 06/19] Add blackbox coverage for backup and restore over RPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was none for either command. The round-trip is what proves the RFD-108 claim, so it runs every command as a plain client invocation: no sudo, no --data-path, where the old commands needed both. It also pins the two behaviors most likely to regress quietly — restoring into a name the cluster has never seen creates the disk, and restoring into a disk that is still mounted is refused rather than silently doing nothing. --- blackbox/disk_backup_test.go | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 blackbox/disk_backup_test.go diff --git a/blackbox/disk_backup_test.go b/blackbox/disk_backup_test.go new file mode 100644 index 000000000..9dcf56f06 --- /dev/null +++ b/blackbox/disk_backup_test.go @@ -0,0 +1,92 @@ +//go:build blackbox + +package blackbox + +import ( + "fmt" + "testing" + "time" + + "miren.dev/runtime/blackbox/harness" +) + +// TestDiskBackupRestoreOverRPC exercises the RFD-108 claim: backup and restore +// work from a client, without a shell on the server. +// +// The proof is in what the commands are NOT given. Every invocation here is a +// plain `m disk ...` with no sudo and no --data-path, where the old commands +// needed both — they opened /var/lib/miren directly and stopped early when it +// was not there. +func TestDiskBackupRestoreOverRPC(t *testing.T) { + c := harness.NewCluster(t) + m := harness.NewMiren(t, c) + + diskName := harness.UniqueAppName(t, "bk-disk") + restoredName := diskName + "-restored" + snapshotPath := fmt.Sprintf("/tmp/%s.miren.zst", diskName) + + t.Log("Creating source disk...") + m.MustRun("debug", "disk", "create", "-n", diskName, "-s", "1") + waitDiskProvisioned(t, m, diskName) + + // No sudo, no --data-path. + t.Log("Backing up over RPC...") + r := m.MustRun("disk", "backup", "-n", diskName, "-o", snapshotPath) + r.RequireContains(t, "Backup complete") + r.RequireContains(t, "Checksum:") + + // The snapshot has to have actually landed on the client side, since that + // is the half the server streamed rather than wrote. + r = m.RunCmd("test", "-s", snapshotPath) + r.RequireSuccess(t) + + // Restoring into a name the cluster has never seen must create the disk. + // This is the disaster-recovery path: on a rebuilt host there is nothing to + // restore into. + t.Log("Restoring into a new disk over RPC...") + r = m.MustRun("disk", "restore", "-s", snapshotPath, "-n", restoredName) + r.RequireContains(t, "Restore complete") + r.RequireContains(t, "Created disk") + + waitDiskProvisioned(t, m, restoredName) + + // A universal-mode disk is mounted by the volume controller whether or not + // anything leased it, and a loop device holds the image's inode rather than + // its path. Restoring over it would leave the mounted filesystem on the old + // image while reporting success, so it must be refused instead. + t.Log("Checking that restore refuses a disk that is in use...") + r = m.Run("disk", "restore", "-s", snapshotPath, "-n", diskName, "--force") + if r.Success() { + t.Errorf("restore into a mounted disk should have been refused, got:\n%s", r.Stdout+r.Stderr) + } + r.RequireContains(t, "in use") +} + +// TestDiskBackupRejectsPinWithoutCloud covers the flag combination an operator +// is most likely to try first, since --pin only means anything to miren.cloud. +func TestDiskBackupRejectsPinWithoutCloud(t *testing.T) { + c := harness.NewCluster(t) + m := harness.NewMiren(t, c) + + r := m.Run("disk", "backup", "-n", "does-not-matter", "--pin", "some-name") + if r.Success() { + t.Fatalf("--pin without --cloud should have been rejected, got:\n%s", r.Stdout+r.Stderr) + } + r.RequireContains(t, "--cloud") +} + +func waitDiskProvisioned(t *testing.T, m *harness.Miren, name string) { + t.Helper() + harness.Poll(t, "disk provisioned: "+name, 60*time.Second, 2*time.Second, + func() (bool, string) { + r := m.Run("debug", "disk", "list") + if !r.Success() { + return false, "debug disk list failed" + } + if r.OutputContains(name) && r.OutputContains("provisioned") { + return true, "" + } + return false, "disk not yet provisioned" + }, + ) +} From e7336db10c90e6d42c3b593c107745f3191a283f Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:27:39 -0700 Subject: [PATCH 07/19] Let backup proceed when it cannot tell if the disk is in use Restore fails closed on that check because it writes: not knowing whether an image is live is not the same as knowing it is idle, and guessing wrong loses data silently. Backup only reads, so the same failure is a reason to say less, not a reason to refuse. Also log the in-use condition once on the server rather than once per line of operator-facing text. --- servers/disk/backup.go | 14 +++++++++----- servers/disk/server.go | 9 ++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/servers/disk/backup.go b/servers/disk/backup.go index a3b75dcce..9c14c6ea2 100644 --- a/servers/disk/backup.go +++ b/servers/disk/backup.go @@ -27,11 +27,15 @@ func (s *Server) Backup(ctx context.Context, state *disk_v1alpha.DiskBackupBacku // whose head and tail come from different moments — weaker than the // power-loss state fsck and Postgres's WAL recovery are built for. Say so // and continue: the operator is the one deciding this is safe. - dev, err := s.liveImageDevice(target.ImagePath) - if err != nil { - return err - } - if dev != "" { + // + // Unlike restore, this check is best effort. Backup only reads, so being + // unable to tell whether the disk is in use is a reason to say less, not a + // reason to refuse. + if dev, err := s.liveImageDevice(target.ImagePath); err != nil { + s.log.Warn("could not tell whether disk image is in use", "disk", target.Name, "error", err) + prog.Warn("Could not tell whether %q is in use, so this backup may not be a point-in-time copy.", target.Name) + } else if dev != "" { + s.log.Info("backing up a disk that is in use", "disk", target.Name, "device", dev) prog.Warn("Disk %q is in use (%s) and may be written during the backup.", target.Name, dev) prog.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") prog.Warn("Detach the disk first for a backup you can rely on.") diff --git a/servers/disk/server.go b/servers/disk/server.go index 8d5567c30..d8f30ca6a 100644 --- a/servers/disk/server.go +++ b/servers/disk/server.go @@ -107,7 +107,6 @@ func (s *Server) liveImageDevice(imagePath string) (string, error) { // can call without checking for nil or caring about send failures. A client // that has stopped listening should not fail an in-flight backup. type progressSink struct { - log *slog.Logger send func(*disk_v1alpha.Progress) } @@ -117,11 +116,12 @@ func (p progressSink) Message(format string, args ...any) { p.send(up) } +// Warn sends a warning to the client. Callers log the underlying condition +// themselves, once, rather than having this emit a server log line per line of +// operator-facing text. func (p progressSink) Warn(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - p.log.Warn(msg) up := new(disk_v1alpha.Progress) - up.Update().SetWarning(msg) + up.Update().SetWarning(fmt.Sprintf(format, args...)) p.send(up) } @@ -141,7 +141,6 @@ func (p progressSink) Transfer(done, total, perSecond, etaSeconds int64) { // not supply one. func (s *Server) newProgress(ctx context.Context, out *stream.SendStreamClient[*disk_v1alpha.Progress]) progressSink { return progressSink{ - log: s.log, send: func(up *disk_v1alpha.Progress) { if out == nil { return From 171757628689887b19b37164d8f982c415b8f22c Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 19:33:18 -0700 Subject: [PATCH 08/19] Send refusals as validation failures so the message survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain error from an RPC handler reaches the operator as "remote error: generic unknown: ", which buries the sentence they need to read under two words that tell them nothing. The refusals here are the ones most likely to be hit — the disk is in use, there is no cloud registered, the image already exists — so they now travel as typed validation failures and print on their own. Genuine failures still surface as failures; this only covers requests the server declines to carry out. --- servers/disk/backup.go | 4 ++-- servers/disk/restore.go | 10 +++++----- servers/disk/server.go | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/servers/disk/backup.go b/servers/disk/backup.go index 9c14c6ea2..83035dd04 100644 --- a/servers/disk/backup.go +++ b/servers/disk/backup.go @@ -57,7 +57,7 @@ func (s *Server) backupToCloud( return errNoCloud("backing up to miren.cloud") } if target.CloudVolumeID == "" { - return fmt.Errorf( + return refuse( "disk %q is not registered with miren.cloud yet, so there is nowhere to upload to — it registers on its own shortly after the disk is created", target.Name, ) @@ -108,7 +108,7 @@ func (s *Server) backupToClient( ) error { out := state.Args().Data() if out == nil { - return fmt.Errorf("backup needs either --cloud or somewhere to write the snapshot") + return refuse("backup needs either --cloud or somewhere to write the snapshot") } img, err := os.Open(target.ImagePath) diff --git a/servers/disk/restore.go b/servers/disk/restore.go index f44bee9c7..0eb5040bb 100644 --- a/servers/disk/restore.go +++ b/servers/disk/restore.go @@ -25,7 +25,7 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest name := args.Disk() if name == "" { - return fmt.Errorf("disk name is required") + return refuse("disk name is required") } src, compressedSize, closeSrc, err := s.restoreSource(ctx, name, args, prog) @@ -61,7 +61,7 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest if !target.Created { if _, err := os.Stat(target.ImagePath); err == nil && !args.Force() { - return fmt.Errorf( + return refuse( "disk %q already has an image at %s — pass --force to overwrite it", name, target.ImagePath, ) @@ -110,7 +110,7 @@ func (s *Server) restoreSource( if point == "" { in := args.Data() if in == nil { - return nil, 0, nil, fmt.Errorf("restore needs either a restore point or a snapshot to read") + return nil, 0, nil, refuse("restore needs either a restore point or a snapshot to read") } prog.Message("Reading snapshot from client") r := stream.ToReader(ctx, in) @@ -131,7 +131,7 @@ func (s *Server) restoreSource( return nil, 0, nil, err } if target.CloudVolumeID == "" { - return nil, 0, nil, fmt.Errorf("disk %q is not registered with miren.cloud, so it has no restore points", name) + return nil, 0, nil, refuse("disk %q is not registered with miren.cloud, so it has no restore points", name) } size := s.restorePointSize(ctx, target.CloudVolumeID, point) @@ -180,7 +180,7 @@ func (s *Server) refuseLiveImage(target *snapshot.RestoreTarget, name string) er if dev == "" { return nil } - return fmt.Errorf( + return refuse( "disk %q is in use (%s is backing %s), and restoring it now would write an image nothing reads — "+ "stop everything using the disk first, or restore into a new disk instead", name, dev, target.ImagePath, diff --git a/servers/disk/server.go b/servers/disk/server.go index d8f30ca6a..31c7af8ba 100644 --- a/servers/disk/server.go +++ b/servers/disk/server.go @@ -22,6 +22,7 @@ import ( "miren.dev/runtime/api/entityserver" "miren.dev/runtime/api/entityserver/entityserver_v1alpha" "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cond" "miren.dev/runtime/pkg/diskresolve" "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/rpc/stream" @@ -69,11 +70,22 @@ func NewServer( } } +// errCategory groups this service's errors in the RPC error envelope. +const errCategory = "disk-backup" + +// refuse reports a request this server will not carry out, as opposed to one +// that failed partway. It travels as a validation failure so the client prints +// the message on its own rather than wrapping it in "remote error: generic +// unknown", which buries the sentence an operator needs to read. +func refuse(format string, args ...any) error { + return cond.ValidationFailure(errCategory, fmt.Sprintf(format, args...)) +} + // errNoCloud is what every cloud-dependent path reports. It names the command // that would fix it, because "no cloud configured" on its own leaves an // operator guessing whether that is a bug or a setup step they skipped. func errNoCloud(what string) error { - return fmt.Errorf( + return refuse( "%s needs a miren.cloud registration, and this cluster has none — run `miren register` first, or back up to a local file instead", what, ) @@ -155,7 +167,7 @@ func (s *Server) newProgress(ctx context.Context, out *stream.SendStreamClient[* // prepareBackup resolves a disk to an image on this host. func (s *Server) prepareBackup(ctx context.Context, name string) (*snapshot.BackupTarget, error) { if name == "" { - return nil, fmt.Errorf("disk name is required") + return nil, refuse("disk name is required") } return snapshot.PrepareBackup(ctx, s.disks, name, s.dataPath) } From 29e89753520ae31ae51ea5a69d24d0de0f2b5dc8 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 20:02:06 -0700 Subject: [PATCH 09/19] Stop the resolver tests writing to /var/lib/miren Four of the seven passed a hardcoded /var/lib/miren as the data path while the other three already used t.TempDir(). That only looked fine because the directory usually does not exist, so the cleanup path's os.Remove returned IsNotExist and was tolerated. Run the same test in a container where a dev server has created those directories as root and it fails on permission denied instead. Surfaced by moving the file; the hardcoding predates it. --- pkg/diskresolve/resolver_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index cf7f48a9b..8eeb03f5b 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -153,7 +153,7 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { ctx := t.Context() es, resolver := setupResolver(t, nil) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) require.NotNil(t, target) assert.True(t, target.Created) @@ -196,7 +196,7 @@ func TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan(t *testing.T) { fault := newFaultRPC(nil, "create", 1, fmt.Errorf("simulated disk_volume create failure")) es, resolver := setupResolver(t, fault) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) @@ -238,7 +238,7 @@ func TestCreateDiskAndVolume_FinalizePatchFailsLeavesNoOrphan(t *testing.T) { fault := newFaultRPC(nil, "patch", 1, fmt.Errorf("simulated disk patch failure")) es, resolver := setupResolver(t, fault) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) @@ -274,7 +274,7 @@ func TestCreateDiskAndVolume_CleanupDoesNotHardDeleteDisk(t *testing.T) { ctx := t.Context() es, resolver := setupResolver(t, nil) - target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", "/var/lib/miren") + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) require.NoError(t, err) disks := listTestDisks(t, ctx, es.EAC) diskID := disks[0].ID From ee2a38d30eb60b473de65be263209c5d09fd7c3f Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 21:04:15 -0700 Subject: [PATCH 10/19] Make disk undelete and list-deleted work over RPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last two disk commands that demanded a shell on the server. Both read the soft-delete holding area under /var/lib/miren directly, and undelete also needed root, so recovering a disk meant SSH plus sudo. They now go through the server like backup and restore, so a deleted disk is recoverable from a laptop. The old implementations move to `miren debug disk undelete` and `miren debug disk list-deleted` as break-glass. list-deleted returns newest deletion first, which the old command did not order at all — the disk someone wants back is almost always the one they just lost. The JSON shape is unchanged. Two fixes found while moving it. The ambiguity error now names the volume ids to choose from rather than just saying to pass one. And recovery creates the volumes directory if it is missing, which it can be when a recovery is the first thing that happens on a rebuilt host — exactly the case this is meant to serve. --- api/disk/disk_v1alpha/rpc.gen.go | 504 +++++++++++++++++++ api/disk/rpc.yml | 73 +++ blackbox/disk_undelete_test.go | 14 +- cli/commands/commands.go | 2 + cli/commands/debug_disk_list_deleted.go | 91 ++++ cli/commands/debug_disk_undelete.go | 219 ++++++++ cli/commands/disk_list_deleted.go | 76 +-- cli/commands/disk_progress.go | 19 + cli/commands/disk_restore.go | 5 +- cli/commands/disk_undelete.go | 208 +------- docs/command-sidebar.json | 2 + docs/docs/command/debug-disk-list-deleted.md | 33 ++ docs/docs/command/debug-disk-undelete.md | 33 ++ docs/docs/command/debug-disk.md | 2 + docs/docs/command/disk-list-deleted.md | 1 - docs/docs/command/disk-undelete.md | 3 +- docs/docs/commands.md | 2 + pkg/diskresolve/resolver.go | 27 +- pkg/workloadroles/roles.go | 8 +- servers/disk/server.go | 14 +- servers/disk/server_test.go | 9 + servers/disk/undelete.go | 295 +++++++++++ servers/disk/undelete_test.go | 248 +++++++++ 23 files changed, 1629 insertions(+), 259 deletions(-) create mode 100644 cli/commands/debug_disk_list_deleted.go create mode 100644 cli/commands/debug_disk_undelete.go create mode 100644 docs/docs/command/debug-disk-list-deleted.md create mode 100644 docs/docs/command/debug-disk-undelete.md create mode 100644 servers/disk/undelete.go create mode 100644 servers/disk/undelete_test.go diff --git a/api/disk/disk_v1alpha/rpc.gen.go b/api/disk/disk_v1alpha/rpc.gen.go index 9be63abcb..baaea6a11 100644 --- a/api/disk/disk_v1alpha/rpc.gen.go +++ b/api/disk/disk_v1alpha/rpc.gen.go @@ -420,6 +420,222 @@ func (v *BackupResult) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &v.data) } +type deletedDiskData struct { + DiskName *string `cbor:"0,keyasint,omitempty" json:"disk_name,omitempty"` + VolumeId *string `cbor:"1,keyasint,omitempty" json:"volume_id,omitempty"` + SizeGb *int64 `cbor:"2,keyasint,omitempty" json:"size_gb,omitempty"` + Filesystem *string `cbor:"3,keyasint,omitempty" json:"filesystem,omitempty"` + DeletedAt *standard.Timestamp `cbor:"4,keyasint,omitempty" json:"deleted_at,omitempty"` + ExpiresAt *standard.Timestamp `cbor:"5,keyasint,omitempty" json:"expires_at,omitempty"` + VolumeMode *string `cbor:"6,keyasint,omitempty" json:"volume_mode,omitempty"` +} + +type DeletedDisk struct { + data deletedDiskData +} + +func (v *DeletedDisk) HasDiskName() bool { + return v.data.DiskName != nil +} + +func (v *DeletedDisk) DiskName() string { + if v.data.DiskName == nil { + return "" + } + return *v.data.DiskName +} + +func (v *DeletedDisk) SetDiskName(disk_name string) { + v.data.DiskName = &disk_name +} + +func (v *DeletedDisk) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *DeletedDisk) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *DeletedDisk) SetVolumeId(volume_id string) { + v.data.VolumeId = &volume_id +} + +func (v *DeletedDisk) HasSizeGb() bool { + return v.data.SizeGb != nil +} + +func (v *DeletedDisk) SizeGb() int64 { + if v.data.SizeGb == nil { + return 0 + } + return *v.data.SizeGb +} + +func (v *DeletedDisk) SetSizeGb(size_gb int64) { + v.data.SizeGb = &size_gb +} + +func (v *DeletedDisk) HasFilesystem() bool { + return v.data.Filesystem != nil +} + +func (v *DeletedDisk) Filesystem() string { + if v.data.Filesystem == nil { + return "" + } + return *v.data.Filesystem +} + +func (v *DeletedDisk) SetFilesystem(filesystem string) { + v.data.Filesystem = &filesystem +} + +func (v *DeletedDisk) HasDeletedAt() bool { + return v.data.DeletedAt != nil +} + +func (v *DeletedDisk) DeletedAt() *standard.Timestamp { + return v.data.DeletedAt +} + +func (v *DeletedDisk) SetDeletedAt(deleted_at *standard.Timestamp) { + v.data.DeletedAt = deleted_at +} + +func (v *DeletedDisk) HasExpiresAt() bool { + return v.data.ExpiresAt != nil +} + +func (v *DeletedDisk) ExpiresAt() *standard.Timestamp { + return v.data.ExpiresAt +} + +func (v *DeletedDisk) SetExpiresAt(expires_at *standard.Timestamp) { + v.data.ExpiresAt = expires_at +} + +func (v *DeletedDisk) HasVolumeMode() bool { + return v.data.VolumeMode != nil +} + +func (v *DeletedDisk) VolumeMode() string { + if v.data.VolumeMode == nil { + return "" + } + return *v.data.VolumeMode +} + +func (v *DeletedDisk) SetVolumeMode(volume_mode string) { + v.data.VolumeMode = &volume_mode +} + +func (v *DeletedDisk) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DeletedDisk) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DeletedDisk) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DeletedDisk) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type undeleteResultData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + DiskId *string `cbor:"1,keyasint,omitempty" json:"disk_id,omitempty"` + VolumeId *string `cbor:"2,keyasint,omitempty" json:"volume_id,omitempty"` + ImagePath *string `cbor:"3,keyasint,omitempty" json:"image_path,omitempty"` +} + +type UndeleteResult struct { + data undeleteResultData +} + +func (v *UndeleteResult) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *UndeleteResult) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *UndeleteResult) SetDisk(disk string) { + v.data.Disk = &disk +} + +func (v *UndeleteResult) HasDiskId() bool { + return v.data.DiskId != nil +} + +func (v *UndeleteResult) DiskId() string { + if v.data.DiskId == nil { + return "" + } + return *v.data.DiskId +} + +func (v *UndeleteResult) SetDiskId(disk_id string) { + v.data.DiskId = &disk_id +} + +func (v *UndeleteResult) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *UndeleteResult) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *UndeleteResult) SetVolumeId(volume_id string) { + v.data.VolumeId = &volume_id +} + +func (v *UndeleteResult) HasImagePath() bool { + return v.data.ImagePath != nil +} + +func (v *UndeleteResult) ImagePath() string { + if v.data.ImagePath == nil { + return "" + } + return *v.data.ImagePath +} + +func (v *UndeleteResult) SetImagePath(image_path string) { + v.data.ImagePath = &image_path +} + +func (v *UndeleteResult) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *UndeleteResult) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *UndeleteResult) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *UndeleteResult) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type restoreResultData struct { Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` ImageSizeBytes *int64 `cbor:"1,keyasint,omitempty" json:"image_size_bytes,omitempty"` @@ -783,6 +999,141 @@ func (v *DiskBackupRestoreResults) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &v.data) } +type diskBackupListDeletedArgsData struct{} + +type DiskBackupListDeletedArgs struct { + call rpc.Call + data diskBackupListDeletedArgsData +} + +func (v *DiskBackupListDeletedArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListDeletedArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListDeletedArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListDeletedArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupListDeletedResultsData struct { + Disks *[]*DeletedDisk `cbor:"0,keyasint,omitempty" json:"disks,omitempty"` + RetentionDays *int32 `cbor:"1,keyasint,omitempty" json:"retention_days,omitempty"` +} + +type DiskBackupListDeletedResults struct { + call rpc.Call + data diskBackupListDeletedResultsData +} + +func (v *DiskBackupListDeletedResults) SetDisks(disks []*DeletedDisk) { + x := slices.Clone(disks) + v.data.Disks = &x +} + +func (v *DiskBackupListDeletedResults) SetRetentionDays(retention_days int32) { + v.data.RetentionDays = &retention_days +} + +func (v *DiskBackupListDeletedResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupListDeletedResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupListDeletedResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupListDeletedResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupUndeleteArgsData struct { + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + VolumeId *string `cbor:"1,keyasint,omitempty" json:"volume_id,omitempty"` +} + +type DiskBackupUndeleteArgs struct { + call rpc.Call + data diskBackupUndeleteArgsData +} + +func (v *DiskBackupUndeleteArgs) HasDisk() bool { + return v.data.Disk != nil +} + +func (v *DiskBackupUndeleteArgs) Disk() string { + if v.data.Disk == nil { + return "" + } + return *v.data.Disk +} + +func (v *DiskBackupUndeleteArgs) HasVolumeId() bool { + return v.data.VolumeId != nil +} + +func (v *DiskBackupUndeleteArgs) VolumeId() string { + if v.data.VolumeId == nil { + return "" + } + return *v.data.VolumeId +} + +func (v *DiskBackupUndeleteArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupUndeleteArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupUndeleteArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupUndeleteArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupUndeleteResultsData struct { + Result **UndeleteResult `cbor:"0,keyasint,omitempty" json:"result,omitempty"` +} + +type DiskBackupUndeleteResults struct { + call rpc.Call + data diskBackupUndeleteResultsData +} + +func (v *DiskBackupUndeleteResults) SetResult(result **UndeleteResult) { + v.data.Result = result +} + +func (v *DiskBackupUndeleteResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupUndeleteResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupUndeleteResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupUndeleteResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type DiskBackupBackup struct { rpc.Call args DiskBackupBackupArgs @@ -861,10 +1212,64 @@ func (t *DiskBackupRestore) Results() *DiskBackupRestoreResults { return results } +type DiskBackupListDeleted struct { + rpc.Call + args DiskBackupListDeletedArgs + results DiskBackupListDeletedResults +} + +func (t *DiskBackupListDeleted) Args() *DiskBackupListDeletedArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupListDeleted) Results() *DiskBackupListDeletedResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type DiskBackupUndelete struct { + rpc.Call + args DiskBackupUndeleteArgs + results DiskBackupUndeleteResults +} + +func (t *DiskBackupUndelete) Args() *DiskBackupUndeleteArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupUndelete) Results() *DiskBackupUndeleteResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type DiskBackup interface { Backup(ctx context.Context, state *DiskBackupBackup) error ListBackups(ctx context.Context, state *DiskBackupListBackups) error Restore(ctx context.Context, state *DiskBackupRestore) error + ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error + Undelete(ctx context.Context, state *DiskBackupUndelete) error } type reexportDiskBackup struct { @@ -883,6 +1288,14 @@ func (reexportDiskBackup) Restore(ctx context.Context, state *DiskBackupRestore) panic("not implemented") } +func (reexportDiskBackup) ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error { + panic("not implemented") +} + +func (reexportDiskBackup) Undelete(ctx context.Context, state *DiskBackupUndelete) error { + panic("not implemented") +} + func (t reexportDiskBackup) CapabilityClient() rpc.Client { return t.client } @@ -919,6 +1332,26 @@ func AdaptDiskBackup(t DiskBackup) *rpc.Interface { return t.Restore(ctx, &DiskBackupRestore{Call: call}) }, }, + { + Name: "listDeleted", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.ListDeleted(ctx, &DiskBackupListDeleted{Call: call}) + }, + }, + { + Name: "undelete", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"disk", "volume_id"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.Undelete(ctx, &DiskBackupUndelete{Call: call}) + }, + }, } return rpc.NewInterface(methods, t) @@ -1051,3 +1484,74 @@ func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_poin return &DiskBackupClientRestoreResults{client: v.Client, data: ret}, nil } + +type DiskBackupClientListDeletedResults struct { + client rpc.Client + data diskBackupListDeletedResultsData +} + +func (v *DiskBackupClientListDeletedResults) HasDisks() bool { + return v.data.Disks != nil +} + +func (v *DiskBackupClientListDeletedResults) Disks() []*DeletedDisk { + if v.data.Disks == nil { + return nil + } + return *v.data.Disks +} + +func (v *DiskBackupClientListDeletedResults) HasRetentionDays() bool { + return v.data.RetentionDays != nil +} + +func (v *DiskBackupClientListDeletedResults) RetentionDays() int32 { + if v.data.RetentionDays == nil { + return 0 + } + return *v.data.RetentionDays +} + +func (v DiskBackupClient) ListDeleted(ctx context.Context) (*DiskBackupClientListDeletedResults, error) { + args := DiskBackupListDeletedArgs{} + + var ret diskBackupListDeletedResultsData + + err := v.Call(ctx, "listDeleted", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientListDeletedResults{client: v.Client, data: ret}, nil +} + +type DiskBackupClientUndeleteResults struct { + client rpc.Client + data diskBackupUndeleteResultsData +} + +func (v *DiskBackupClientUndeleteResults) HasResult() bool { + return v.data.Result != nil +} + +func (v *DiskBackupClientUndeleteResults) Result() *UndeleteResult { + if v.data.Result == nil { + return nil + } + return *v.data.Result +} + +func (v DiskBackupClient) Undelete(ctx context.Context, disk string, volume_id string) (*DiskBackupClientUndeleteResults, error) { + args := DiskBackupUndeleteArgs{} + args.data.Disk = &disk + args.data.VolumeId = &volume_id + + var ret diskBackupUndeleteResultsData + + err := v.Call(ctx, "undelete", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientUndeleteResults{client: v.Client, data: ret}, nil +} diff --git a/api/disk/rpc.yml b/api/disk/rpc.yml index 1fac2f09f..ca058a978 100644 --- a/api/disk/rpc.yml +++ b/api/disk/rpc.yml @@ -114,6 +114,52 @@ types: index: 3 doc: Identifier of the uploaded restore point, empty for a local backup + - type: DeletedDisk + doc: > + A disk that was deleted but whose data is still sitting in the + soft-delete holding area, waiting out its retention period. + fields: + - name: disk_name + type: string + index: 0 + - name: volume_id + type: string + index: 1 + doc: Identifies which one to recover when several share a name + - name: size_gb + type: int64 + index: 2 + - name: filesystem + type: string + index: 3 + - name: deleted_at + type: standard.Timestamp + index: 4 + - name: expires_at + type: standard.Timestamp + index: 5 + doc: When the retention period runs out and the data is reclaimed + - name: volume_mode + type: string + index: 6 + + - type: UndeleteResult + fields: + - name: disk + type: string + index: 0 + doc: Name of the recovered disk + - name: disk_id + type: string + index: 1 + doc: Entity id of the recreated disk + - name: volume_id + type: string + index: 2 + - name: image_path + type: string + index: 3 + - type: RestoreResult fields: - name: disk @@ -209,3 +255,30 @@ interfaces: results: - name: result type: '*RestoreResult' + + - name: listDeleted + doc: > + List disks whose data is still recoverable from the soft-delete + holding area, newest deletion first. + results: + - name: disks + type: list + element: DeletedDisk + - name: retention_days + type: int32 + doc: How long this server keeps deleted disk data before reclaiming it + + - name: undelete + doc: > + Recover a deleted disk: move its data back out of the holding area + and recreate the entities that point at it. + parameters: + - name: disk + type: string + doc: Name of the deleted disk to recover + - name: volume_id + type: string + doc: Which volume to recover, when several deleted disks share a name + results: + - name: result + type: '*UndeleteResult' diff --git a/blackbox/disk_undelete_test.go b/blackbox/disk_undelete_test.go index d96dd022c..ca770f188 100644 --- a/blackbox/disk_undelete_test.go +++ b/blackbox/disk_undelete_test.go @@ -60,17 +60,16 @@ func TestDiskUndelete(t *testing.T) { }, ) - // Step 6: Verify the disk appears in list-deleted - // These commands need sudo because /var/lib/miren/disk-data is owned by root + // Step 6: Verify the disk appears in list-deleted. + // No sudo and no --data-path: these go through the server now, so they work + // from a client that cannot see /var/lib/miren at all. t.Log("Checking list-deleted...") - r = m.RunCmd("sudo", "m", "disk", "list-deleted") - r.RequireSuccess(t) + r = m.MustRun("disk", "list-deleted") r.RequireContains(t, diskName) // Step 7: Undelete the disk t.Log("Undeleting disk...") - r = m.RunCmd("sudo", "m", "disk", "undelete", "-n", diskName) - r.RequireSuccess(t) + r = m.MustRun("disk", "undelete", "-n", diskName) r.RequireContains(t, "Disk restored successfully") // Step 8: Verify the disk is back and provisioned @@ -89,8 +88,7 @@ func TestDiskUndelete(t *testing.T) { ) // Step 9: Verify it's no longer in list-deleted - r = m.RunCmd("sudo", "m", "disk", "list-deleted") - r.RequireSuccess(t) + r = m.MustRun("disk", "list-deleted") if r.OutputContains(diskName) { t.Error("disk should no longer appear in list-deleted after undelete") } diff --git a/cli/commands/commands.go b/cli/commands/commands.go index ed1c7400f..a0effca68 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1276,6 +1276,8 @@ Warning: These commands are intended for advanced users and developers. They may // listener is down, and so must run on the server. d.Dispatch("debug disk backup", Infer("debug disk backup", "Back up a disk by reading its image directly (break-glass)", DebugDiskBackup)) d.Dispatch("debug disk restore", Infer("debug disk restore", "Restore a disk by writing its image directly (break-glass)", DebugDiskRestore)) + d.Dispatch("debug disk undelete", Infer("debug disk undelete", "Recover a deleted disk by moving its data directly (break-glass)", DebugDiskUndelete)) + d.Dispatch("debug disk list-deleted", Infer("debug disk list-deleted", "Read the soft-delete holding area directly (break-glass)", DebugDiskListDeleted)) // Debug saga commands d.Dispatch("debug saga", Section("debug saga", "Saga execution debug commands", "", WithSectionDescription(sagaSectionDescription))) diff --git a/cli/commands/debug_disk_list_deleted.go b/cli/commands/debug_disk_list_deleted.go new file mode 100644 index 000000000..765df229b --- /dev/null +++ b/cli/commands/debug_disk_list_deleted.go @@ -0,0 +1,91 @@ +package commands + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "miren.dev/runtime/components/diskio" +) + +// DebugDiskListDeleted reads the soft-delete holding area directly, without +// going through the server. +// +// Break-glass; `miren disk list-deleted` is the supported command. +func DebugDiskListDeleted(ctx *Context, opts struct { + FormatOptions + ConfigCentric + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` +}) error { + diskDataPath := filepath.Join(opts.DataPath, "disk-data") + if _, err := os.Stat(diskDataPath); err != nil { + return fmt.Errorf("data path %s not found — this command must be run on the server", diskDataPath) + } + + entries, err := diskio.ListDeletedVolumes(diskDataPath) + if err != nil { + return fmt.Errorf("listing deleted volumes: %w", err) + } + + retentionDays := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + + if opts.IsJSON() { + type deletedDiskJSON struct { + DiskName string `json:"disk_name"` + VolumeID string `json:"volume_id"` + SizeGB int64 `json:"size_gb"` + Filesystem string `json:"filesystem"` + DeletedAt string `json:"deleted_at"` + ExpiresAt string `json:"expires_at"` + RetentionDays int `json:"retention_days"` + } + + var items []deletedDiskJSON + for _, e := range entries { + meta := e.Metadata + expiresAt := meta.DeletedAt.Add(time.Duration(retentionDays) * 24 * time.Hour) + items = append(items, deletedDiskJSON{ + DiskName: meta.DiskName, + VolumeID: meta.VolumeID, + SizeGB: meta.SizeGb, + Filesystem: meta.Filesystem, + DeletedAt: meta.DeletedAt.Format(time.RFC3339), + ExpiresAt: expiresAt.Format(time.RFC3339), + RetentionDays: retentionDays, + }) + } + + return PrintJSON(items) + } + + if len(entries) == 0 { + ctx.Info("No deleted disks found") + return nil + } + + ctx.Info("Deleted disks available for recovery:") + ctx.Info("") + + for _, e := range entries { + meta := e.Metadata + age := time.Since(meta.DeletedAt) + remaining := time.Duration(retentionDays)*24*time.Hour - age + + ctx.Info("Name: %s", meta.DiskName) + ctx.Info(" Volume ID: %s", meta.VolumeID) + ctx.Info(" Size: %d GB", meta.SizeGb) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Deleted: %s (%s ago)", meta.DeletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) + if remaining > 0 { + ctx.Info(" Expires in: %s", remaining.Truncate(time.Minute)) + } else { + ctx.Info(" Expires in: imminent (past retention period)") + } + ctx.Info("") + } + + ctx.Info("To restore: miren disk undelete --name ") + + return nil +} diff --git a/cli/commands/debug_disk_undelete.go b/cli/commands/debug_disk_undelete.go new file mode 100644 index 000000000..99477b033 --- /dev/null +++ b/cli/commands/debug_disk_undelete.go @@ -0,0 +1,219 @@ +package commands + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/entityserver/entityserver_v1alpha" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/idgen" +) + +// DebugDiskUndelete recovers a deleted disk by moving its data back and +// recreating its entities directly, without going through the server. +// +// Break-glass, for when the server's RPC listener is down; `miren disk +// undelete` is the supported command. It therefore has to run on the server and +// keeps --data-path. +func DebugDiskUndelete(ctx *Context, opts struct { + ConfigCentric + Name string `short:"n" long:"name" description:"Disk name to undelete" required:"true"` + VolumeID string `short:"V" long:"volume-id" description:"Volume ID to restore (when multiple deleted disks share a name)"` + DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` +}) error { + diskDataPath := filepath.Join(opts.DataPath, "disk-data") + if _, err := os.Stat(diskDataPath); err != nil { + return fmt.Errorf("data path %s not found — disk undelete must be run on the server", diskDataPath) + } + + entries, err := diskio.ListDeletedVolumes(diskDataPath) + if err != nil { + return fmt.Errorf("listing deleted volumes: %w", err) + } + + // Filter by name + var matches []diskio.DeletedVolumeEntry + for _, e := range entries { + if e.Metadata.DiskName == opts.Name { + if opts.VolumeID == "" || e.Metadata.VolumeID == opts.VolumeID { + matches = append(matches, e) + } + } + } + + if len(matches) == 0 { + return fmt.Errorf("no deleted disk found with name %q", opts.Name) + } + + if len(matches) > 1 { + ctx.Info("Multiple deleted disks found with name %q:", opts.Name) + for _, m := range matches { + age := time.Since(m.Metadata.DeletedAt).Truncate(time.Minute) + ctx.Info(" Volume ID: %s (deleted %s ago, %d GB, %s)", + m.Metadata.VolumeID, age, m.Metadata.SizeGb, m.Metadata.Filesystem) + } + return fmt.Errorf("specify --volume-id to select which disk to restore") + } + + entry := matches[0] + meta := entry.Metadata + + ctx.Info("Restoring deleted disk:") + ctx.Info(" Name: %s", meta.DiskName) + ctx.Info(" Size: %d GB", meta.SizeGb) + ctx.Info(" Filesystem: %s", meta.Filesystem) + ctx.Info(" Deleted: %s", meta.DeletedAt.Format(time.RFC3339)) + + client, err := ctx.RPCClient("entities") + if err != nil { + return err + } + + eac := entityserver_v1alpha.NewEntityAccessClient(client) + ec := entityserver.NewClient(ctx.Log, eac) + resolver := diskresolve.New(eac, ec) + + // Check if a disk with this name already exists + if _, err := resolver.FindDisk(context.Background(), meta.DiskName); err == nil { + return fmt.Errorf("a disk named %q already exists — rename or delete it before restoring", meta.DiskName) + } + + // Normalize filesystem + filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + + var fs storage_v1alpha.DiskFilesystem + switch filesystem { + case "ext4": + fs = storage_v1alpha.EXT4 + case "xfs": + fs = storage_v1alpha.XFS + case "btrfs": + fs = storage_v1alpha.BTRFS + default: + fs = storage_v1alpha.EXT4 + } + + // Create the disk entity in RESTORING state + diskId := idgen.GenNS("disk") + disk := &storage_v1alpha.Disk{ + Name: meta.DiskName, + SizeGb: meta.SizeGb, + Filesystem: fs, + Status: storage_v1alpha.RESTORING, + } + + diskEntityId, err := ec.Create(context.Background(), diskId, disk) + if err != nil { + return fmt.Errorf("creating disk entity: %w", err) + } + + ctx.Info("Created disk entity: %s", diskEntityId) + + // Move the volume directory back + volId := meta.VolumeID + destPath := filepath.Join(diskDataPath, "volumes", volId) + + if err := os.Rename(entry.Path, destPath); err != nil { + // Clean up the disk entity we just created + if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk entity: %v", derr) + } + return fmt.Errorf("moving volume back to %s: %w", destPath, err) + } + + // Deferred rollback: if we don't reach the final commit, move the + // volume back to deleted-volumes (with its metadata intact) and clean + // up any entities we created. + committed := false + defer func() { + if committed { + return + } + if rerr := os.Rename(destPath, entry.Path); rerr != nil { + ctx.Warn("Failed to move volume back to deleted-volumes: %v", rerr) + } + if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk entity: %v", derr) + } + }() + + // Find the node ID + nodeId, err := resolver.FindNodeId(context.Background()) + if err != nil { + ctx.Warn("Failed to find node ID, using stored value: %v", err) + nodeId = meta.NodeID.Id() + } + + imagePath := filepath.Join(destPath, "disk.img") + + // Verify the disk image actually exists before creating entities + if _, err := os.Stat(imagePath); err != nil { + return fmt.Errorf("disk image not found at %s — deleted volume may be corrupted", imagePath) + } + + // Create disk_volume entity + volEntityId := entity.Id("disk_volume/" + volId) + vol := &storage_v1alpha.DiskVolume{ + Name: meta.DiskName, + DiskId: diskEntityId, + VolumeId: volId, + SizeGb: meta.SizeGb, + Filesystem: filesystem, + VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY. The runner's DiskVolumeController drives + // the restored volume through the same mount-then-READY ordering as a + // fresh create (skipping the reimage, since disk.img is already in + // place). Advertising DV_READY here would let a lease bind against a + // volume that isn't registered in the runner's in-memory state yet, + // which lands the lease in a terminal FAILED (MIR-1469). + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, + } + + _, err = eac.Create(context.Background(), entity.New( + entity.DBId, volEntityId, + vol.Encode, + ).Attrs()) + if err != nil { + return fmt.Errorf("creating disk_volume entity: %w", err) + } + + // Transition disk to PROVISIONING (not PROVISIONED). The DiskController + // promotes it to PROVISIONED only once the disk_volume actually reaches + // DV_READY — i.e. after the volume is registered and mounted on the node — + // so the disk never reports "provisioned" (and thus leasable) ahead of a + // real mount. + _, err = eac.Patch(context.Background(), []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volId), + }, 0) + if err != nil { + // Also clean up the volume entity + if _, derr := eac.Delete(context.Background(), string(volEntityId)); derr != nil { + ctx.Warn("Failed to clean up disk_volume entity: %v", derr) + } + return fmt.Errorf("updating disk to provisioned: %w", err) + } + + // All entities committed — remove leftover metadata and disarm rollback + os.Remove(filepath.Join(destPath, "metadata.json")) + committed = true + + ctx.Info("Disk restored successfully") + ctx.Info(" Disk ID: %s", diskEntityId) + ctx.Info(" Volume ID: %s", volId) + ctx.Info(" Image: %s", imagePath) + + return nil +} diff --git a/cli/commands/disk_list_deleted.go b/cli/commands/disk_list_deleted.go index 1d3943cc9..71ecd2e77 100644 --- a/cli/commands/disk_list_deleted.go +++ b/cli/commands/disk_list_deleted.go @@ -1,32 +1,29 @@ package commands import ( - "fmt" - "os" - "path/filepath" "time" - "miren.dev/runtime/components/diskio" + "miren.dev/runtime/api/disk/disk_v1alpha" ) -// DiskListDeleted lists disks that have been soft-deleted and are available -// for recovery via disk undelete. +// DiskListDeleted lists disks whose data is still recoverable, asking the +// server rather than reading its data directory, so it works from anywhere. func DiskListDeleted(ctx *Context, opts struct { FormatOptions ConfigCentric - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` }) error { - diskDataPath := filepath.Join(opts.DataPath, "disk-data") - if _, err := os.Stat(diskDataPath); err != nil { - return fmt.Errorf("data path %s not found — this command must be run on the server", diskDataPath) + client, err := ctx.RPCClient(diskBackupService) + if err != nil { + return err } - entries, err := diskio.ListDeletedVolumes(diskDataPath) + res, err := disk_v1alpha.NewDiskBackupClient(client).ListDeleted(ctx) if err != nil { - return fmt.Errorf("listing deleted volumes: %w", err) + return err } - retentionDays := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + disks := res.Disks() + retentionDays := int(res.RetentionDays()) if opts.IsJSON() { type deletedDiskJSON struct { @@ -40,16 +37,14 @@ func DiskListDeleted(ctx *Context, opts struct { } var items []deletedDiskJSON - for _, e := range entries { - meta := e.Metadata - expiresAt := meta.DeletedAt.Add(time.Duration(retentionDays) * 24 * time.Hour) + for _, d := range disks { items = append(items, deletedDiskJSON{ - DiskName: meta.DiskName, - VolumeID: meta.VolumeID, - SizeGB: meta.SizeGb, - Filesystem: meta.Filesystem, - DeletedAt: meta.DeletedAt.Format(time.RFC3339), - ExpiresAt: expiresAt.Format(time.RFC3339), + DiskName: d.DiskName(), + VolumeID: d.VolumeId(), + SizeGB: d.SizeGb(), + Filesystem: d.Filesystem(), + DeletedAt: rfc3339(d.HasDeletedAt(), d.DeletedAt()), + ExpiresAt: rfc3339(d.HasExpiresAt(), d.ExpiresAt()), RetentionDays: retentionDays, }) } @@ -57,7 +52,7 @@ func DiskListDeleted(ctx *Context, opts struct { return PrintJSON(items) } - if len(entries) == 0 { + if len(disks) == 0 { ctx.Info("No deleted disks found") return nil } @@ -65,25 +60,34 @@ func DiskListDeleted(ctx *Context, opts struct { ctx.Info("Deleted disks available for recovery:") ctx.Info("") - for _, e := range entries { - meta := e.Metadata - age := time.Since(meta.DeletedAt) - remaining := time.Duration(retentionDays)*24*time.Hour - age - - ctx.Info("Name: %s", meta.DiskName) - ctx.Info(" Volume ID: %s", meta.VolumeID) - ctx.Info(" Size: %d GB", meta.SizeGb) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Deleted: %s (%s ago)", meta.DeletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) - if remaining > 0 { - ctx.Info(" Expires in: %s", remaining.Truncate(time.Minute)) + for _, d := range disks { + deletedAt := goTime(d.HasDeletedAt(), d.DeletedAt()) + expiresAt := goTime(d.HasExpiresAt(), d.ExpiresAt()) + + ctx.Info("Name: %s", d.DiskName()) + ctx.Info(" Volume ID: %s", d.VolumeId()) + ctx.Info(" Size: %d GB", d.SizeGb()) + ctx.Info(" Filesystem: %s", d.Filesystem()) + + if deletedAt.IsZero() { + ctx.Info(" Deleted: unknown") } else { + age := time.Since(deletedAt) + ctx.Info(" Deleted: %s (%s ago)", deletedAt.Format(time.RFC3339), age.Truncate(time.Minute)) + } + + switch { + case expiresAt.IsZero(): + ctx.Info(" Expires in: unknown") + case time.Until(expiresAt) > 0: + ctx.Info(" Expires in: %s", time.Until(expiresAt).Truncate(time.Minute)) + default: ctx.Info(" Expires in: imminent (past retention period)") } ctx.Info("") } - ctx.Info("To restore: miren disk undelete --name ") + ctx.Info("To recover: miren disk undelete --name ") return nil } diff --git a/cli/commands/disk_progress.go b/cli/commands/disk_progress.go index ea4267a24..2034a1d35 100644 --- a/cli/commands/disk_progress.go +++ b/cli/commands/disk_progress.go @@ -8,11 +8,30 @@ import ( "golang.org/x/term" "miren.dev/runtime/api/disk/disk_v1alpha" "miren.dev/runtime/pkg/progress/upload" + "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/rpc/stream" ) const diskBackupService = "dev.miren.runtime/disk-backup" +// goTime converts a wire timestamp, treating an absent one as the zero time so +// callers can say "unknown" rather than print 1970. +func goTime(present bool, ts *standard.Timestamp) time.Time { + if !present || ts == nil { + return time.Time{} + } + return time.Unix(ts.Seconds(), int64(ts.Nanoseconds())) +} + +// rfc3339 renders a wire timestamp for JSON output, empty when absent. +func rfc3339(present bool, ts *standard.Timestamp) string { + t := goTime(present, ts) + if t.IsZero() { + return "" + } + return t.Format(time.RFC3339) +} + // diskProgress renders the server's progress events for backup and restore. // // Disk images are large enough that a silent wait is indistinguishable from a diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index bddabd79f..90186b0b7 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -113,9 +113,8 @@ func (r restorePointItem) ID() string { return r.point.Id() } func (r restorePointItem) Row() []string { when := "unknown" - if r.point.HasCreatedAt() { - ts := r.point.CreatedAt() - when = time.Unix(ts.Seconds(), int64(ts.Nanoseconds())).Format("2006-01-02 15:04:05") + if t := goTime(r.point.HasCreatedAt(), r.point.CreatedAt()); !t.IsZero() { + when = t.Format("2006-01-02 15:04:05") } return []string{ when, diff --git a/cli/commands/disk_undelete.go b/cli/commands/disk_undelete.go index 47f853c2e..9df461f2c 100644 --- a/cli/commands/disk_undelete.go +++ b/cli/commands/disk_undelete.go @@ -1,214 +1,36 @@ package commands import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "miren.dev/runtime/api/entityserver" - "miren.dev/runtime/api/entityserver/entityserver_v1alpha" - "miren.dev/runtime/api/storage/storage_v1alpha" - "miren.dev/runtime/components/diskio" - "miren.dev/runtime/pkg/diskresolve" - "miren.dev/runtime/pkg/entity" - "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/api/disk/disk_v1alpha" ) -// DiskUndelete restores a recently deleted disk from the soft-delete holding area. +// DiskUndelete recovers a recently deleted disk, asking the server to move the +// data back and recreate the entities, so it works from anywhere. func DiskUndelete(ctx *Context, opts struct { ConfigCentric Name string `short:"n" long:"name" description:"Disk name to undelete" required:"true"` - VolumeID string `short:"V" long:"volume-id" description:"Volume ID to restore (when multiple deleted disks share a name)"` - DataPath string `long:"data-path" description:"Path to miren data directory" default:"/var/lib/miren"` + VolumeID string `short:"V" long:"volume-id" description:"Volume ID to recover (when several deleted disks share a name)"` }) error { - diskDataPath := filepath.Join(opts.DataPath, "disk-data") - if _, err := os.Stat(diskDataPath); err != nil { - return fmt.Errorf("data path %s not found — disk undelete must be run on the server", diskDataPath) - } - - entries, err := diskio.ListDeletedVolumes(diskDataPath) - if err != nil { - return fmt.Errorf("listing deleted volumes: %w", err) - } - - // Filter by name - var matches []diskio.DeletedVolumeEntry - for _, e := range entries { - if e.Metadata.DiskName == opts.Name { - if opts.VolumeID == "" || e.Metadata.VolumeID == opts.VolumeID { - matches = append(matches, e) - } - } - } - - if len(matches) == 0 { - return fmt.Errorf("no deleted disk found with name %q", opts.Name) - } - - if len(matches) > 1 { - ctx.Info("Multiple deleted disks found with name %q:", opts.Name) - for _, m := range matches { - age := time.Since(m.Metadata.DeletedAt).Truncate(time.Minute) - ctx.Info(" Volume ID: %s (deleted %s ago, %d GB, %s)", - m.Metadata.VolumeID, age, m.Metadata.SizeGb, m.Metadata.Filesystem) - } - return fmt.Errorf("specify --volume-id to select which disk to restore") - } - - entry := matches[0] - meta := entry.Metadata - - ctx.Info("Restoring deleted disk:") - ctx.Info(" Name: %s", meta.DiskName) - ctx.Info(" Size: %d GB", meta.SizeGb) - ctx.Info(" Filesystem: %s", meta.Filesystem) - ctx.Info(" Deleted: %s", meta.DeletedAt.Format(time.RFC3339)) - - client, err := ctx.RPCClient("entities") + client, err := ctx.RPCClient(diskBackupService) if err != nil { return err } - eac := entityserver_v1alpha.NewEntityAccessClient(client) - ec := entityserver.NewClient(ctx.Log, eac) - resolver := diskresolve.New(eac, ec) - - // Check if a disk with this name already exists - if _, err := resolver.FindDisk(context.Background(), meta.DiskName); err == nil { - return fmt.Errorf("a disk named %q already exists — rename or delete it before restoring", meta.DiskName) - } - - // Normalize filesystem - filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + ctx.Info("Recovering deleted disk %q", opts.Name) - var fs storage_v1alpha.DiskFilesystem - switch filesystem { - case "ext4": - fs = storage_v1alpha.EXT4 - case "xfs": - fs = storage_v1alpha.XFS - case "btrfs": - fs = storage_v1alpha.BTRFS - default: - fs = storage_v1alpha.EXT4 - } - - // Create the disk entity in RESTORING state - diskId := idgen.GenNS("disk") - disk := &storage_v1alpha.Disk{ - Name: meta.DiskName, - SizeGb: meta.SizeGb, - Filesystem: fs, - Status: storage_v1alpha.RESTORING, - } - - diskEntityId, err := ec.Create(context.Background(), diskId, disk) + res, err := disk_v1alpha.NewDiskBackupClient(client).Undelete(ctx, opts.Name, opts.VolumeID) if err != nil { - return fmt.Errorf("creating disk entity: %w", err) - } - - ctx.Info("Created disk entity: %s", diskEntityId) - - // Move the volume directory back - volId := meta.VolumeID - destPath := filepath.Join(diskDataPath, "volumes", volId) - - if err := os.Rename(entry.Path, destPath); err != nil { - // Clean up the disk entity we just created - if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk entity: %v", derr) - } - return fmt.Errorf("moving volume back to %s: %w", destPath, err) - } - - // Deferred rollback: if we don't reach the final commit, move the - // volume back to deleted-volumes (with its metadata intact) and clean - // up any entities we created. - committed := false - defer func() { - if committed { - return - } - if rerr := os.Rename(destPath, entry.Path); rerr != nil { - ctx.Warn("Failed to move volume back to deleted-volumes: %v", rerr) - } - if _, derr := eac.Delete(context.Background(), string(diskEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk entity: %v", derr) - } - }() - - // Find the node ID - nodeId, err := resolver.FindNodeId(context.Background()) - if err != nil { - ctx.Warn("Failed to find node ID, using stored value: %v", err) - nodeId = meta.NodeID.Id() - } - - imagePath := filepath.Join(destPath, "disk.img") - - // Verify the disk image actually exists before creating entities - if _, err := os.Stat(imagePath); err != nil { - return fmt.Errorf("disk image not found at %s — deleted volume may be corrupted", imagePath) - } - - // Create disk_volume entity - volEntityId := entity.Id("disk_volume/" + volId) - vol := &storage_v1alpha.DiskVolume{ - Name: meta.DiskName, - DiskId: diskEntityId, - VolumeId: volId, - SizeGb: meta.SizeGb, - Filesystem: filesystem, - VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), - DesiredState: storage_v1alpha.DV_PRESENT, - // Start PENDING, not READY. The runner's DiskVolumeController drives - // the restored volume through the same mount-then-READY ordering as a - // fresh create (skipping the reimage, since disk.img is already in - // place). Advertising DV_READY here would let a lease bind against a - // volume that isn't registered in the runner's in-memory state yet, - // which lands the lease in a terminal FAILED (MIR-1469). - ActualState: storage_v1alpha.DV_PENDING, - ImagePath: imagePath, - NodeId: nodeId, - } - - _, err = eac.Create(context.Background(), entity.New( - entity.DBId, volEntityId, - vol.Encode, - ).Attrs()) - if err != nil { - return fmt.Errorf("creating disk_volume entity: %w", err) - } - - // Transition disk to PROVISIONING (not PROVISIONED). The DiskController - // promotes it to PROVISIONED only once the disk_volume actually reaches - // DV_READY — i.e. after the volume is registered and mounted on the node — - // so the disk never reports "provisioned" (and thus leasable) ahead of a - // real mount. - _, err = eac.Patch(context.Background(), []entity.Attr{ - entity.Ref(entity.DBId, diskEntityId), - entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), - entity.String(storage_v1alpha.DiskVolumeIdId, volId), - }, 0) - if err != nil { - // Also clean up the volume entity - if _, derr := eac.Delete(context.Background(), string(volEntityId)); derr != nil { - ctx.Warn("Failed to clean up disk_volume entity: %v", derr) - } - return fmt.Errorf("updating disk to provisioned: %w", err) + return err } - // All entities committed — remove leftover metadata and disarm rollback - os.Remove(filepath.Join(destPath, "metadata.json")) - committed = true - + out := res.Result() ctx.Info("Disk restored successfully") - ctx.Info(" Disk ID: %s", diskEntityId) - ctx.Info(" Volume ID: %s", volId) - ctx.Info(" Image: %s", imagePath) + if out == nil { + return nil + } + ctx.Info(" Disk ID: %s", out.DiskId()) + ctx.Info(" Volume ID: %s", out.VolumeId()) + ctx.Info(" Image: %s", out.ImagePath()) return nil } diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index b09b4c856..7204eb179 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -132,9 +132,11 @@ "command/debug-disk-lease-release", "command/debug-disk-lease-status", "command/debug-disk-list", + "command/debug-disk-list-deleted", "command/debug-disk-mounts", "command/debug-disk-restore", "command/debug-disk-status", + "command/debug-disk-undelete", "command/debug-entity", "command/debug-entity-create", "command/debug-entity-delete", diff --git a/docs/docs/command/debug-disk-list-deleted.md b/docs/docs/command/debug-disk-list-deleted.md new file mode 100644 index 000000000..feca867d2 --- /dev/null +++ b/docs/docs/command/debug-disk-list-deleted.md @@ -0,0 +1,33 @@ +--- +title: "miren debug disk list-deleted" +sidebar_label: "debug disk list-deleted" +description: "Read the soft-delete holding area directly (break-glass)" +--- + +# miren debug disk list-deleted + +Read the soft-delete holding area directly (break-glass) + +## Usage + +```bash +miren debug disk list-deleted [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--format` — Output format (text, json) (default: `text`) +- `--json` — Shorthand for --format json + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](/command/debug-disk) diff --git a/docs/docs/command/debug-disk-undelete.md b/docs/docs/command/debug-disk-undelete.md new file mode 100644 index 000000000..b88d15e47 --- /dev/null +++ b/docs/docs/command/debug-disk-undelete.md @@ -0,0 +1,33 @@ +--- +title: "miren debug disk undelete" +sidebar_label: "debug disk undelete" +description: "Recover a deleted disk by moving its data directly (break-glass)" +--- + +# miren debug disk undelete + +Recover a deleted disk by moving its data directly (break-glass) + +## Usage + +```bash +miren debug disk undelete [flags] +``` + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) +- `--name, -n` — Disk name to undelete +- `--volume-id, -V` — Volume ID to restore (when multiple deleted disks share a name) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## See also + +- [`miren debug disk`](/command/debug-disk) diff --git a/docs/docs/command/debug-disk.md b/docs/docs/command/debug-disk.md index 0b06873d4..e8db1b4e1 100644 --- a/docs/docs/command/debug-disk.md +++ b/docs/docs/command/debug-disk.md @@ -64,9 +64,11 @@ miren debug disk [flags] - [`miren debug disk lease-release`](/command/debug-disk-lease-release) — Release a disk lease - [`miren debug disk lease-status`](/command/debug-disk-lease-status) — Show detailed status of a disk lease - [`miren debug disk list`](/command/debug-disk-list) — List all disk entities +- [`miren debug disk list-deleted`](/command/debug-disk-list-deleted) — Read the soft-delete holding area directly (break-glass) - [`miren debug disk mounts`](/command/debug-disk-mounts) — List all mounted disks from /proc/mounts - [`miren debug disk restore`](/command/debug-disk-restore) — Restore a disk by writing its image directly (break-glass) - [`miren debug disk status`](/command/debug-disk-status) — Show status of a disk entity +- [`miren debug disk undelete`](/command/debug-disk-undelete) — Recover a deleted disk by moving its data directly (break-glass) ## See also diff --git a/docs/docs/command/disk-list-deleted.md b/docs/docs/command/disk-list-deleted.md index 8440f367e..2c0a798cf 100644 --- a/docs/docs/command/disk-list-deleted.md +++ b/docs/docs/command/disk-list-deleted.md @@ -18,7 +18,6 @@ miren disk list-deleted [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--format` — Output format (text, json) (default: `text`) - `--json` — Shorthand for --format json diff --git a/docs/docs/command/disk-undelete.md b/docs/docs/command/disk-undelete.md index ebababb2b..7dfe981ad 100644 --- a/docs/docs/command/disk-undelete.md +++ b/docs/docs/command/disk-undelete.md @@ -18,9 +18,8 @@ miren disk undelete [flags] - `--cluster, -C` — Cluster name - `--config` — Path to the config file -- `--data-path` — Path to miren data directory (default: `/var/lib/miren`) - `--name, -n` — Disk name to undelete -- `--volume-id, -V` — Volume ID to restore (when multiple deleted disks share a name) +- `--volume-id, -V` — Volume ID to recover (when several deleted disks share a name) ## Global Options diff --git a/docs/docs/commands.md b/docs/docs/commands.md index f5123aff6..f48d6795f 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -321,9 +321,11 @@ These commands are intended for advanced debugging and troubleshooting. They may | [`miren debug disk lease-release`](/command/debug-disk-lease-release) | Release a disk lease | | [`miren debug disk lease-status`](/command/debug-disk-lease-status) | Show detailed status of a disk lease | | [`miren debug disk list`](/command/debug-disk-list) | List all disk entities | +| [`miren debug disk list-deleted`](/command/debug-disk-list-deleted) | Read the soft-delete holding area directly (break-glass) | | [`miren debug disk mounts`](/command/debug-disk-mounts) | List all mounted disks from /proc/mounts | | [`miren debug disk restore`](/command/debug-disk-restore) | Restore a disk by writing its image directly (break-glass) | | [`miren debug disk status`](/command/debug-disk-status) | Show status of a disk entity | +| [`miren debug disk undelete`](/command/debug-disk-undelete) | Recover a deleted disk by moving its data directly (break-glass) | | [`miren debug entity`](/command/debug-entity) | Entity store debug commands | | [`miren debug entity create`](/command/debug-entity-create) | Create a new entity | | [`miren debug entity delete`](/command/debug-entity-delete) | Delete an entity | diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index 85da552c1..4571a9994 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -96,18 +96,7 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt // Normalize filesystem string — strip enum prefix if present filesystem = strings.TrimPrefix(strings.ToLower(filesystem), "filesystem.") - - var fs storage_v1alpha.DiskFilesystem - switch filesystem { - case "ext4": - fs = storage_v1alpha.EXT4 - case "xfs": - fs = storage_v1alpha.XFS - case "btrfs": - fs = storage_v1alpha.BTRFS - default: - fs = storage_v1alpha.EXT4 - } + fs := ParseFilesystem(filesystem) diskId := idgen.GenNS("disk") volId := idgen.GenNS("disk-vol") @@ -281,6 +270,20 @@ func (r *Resolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.Le return leases, nil } +// ParseFilesystem maps a filesystem name onto the disk enum, tolerating the +// "filesystem." prefix the enum renders with. Anything unrecognized becomes +// ext4, which is the default a disk gets when it does not ask for one. +func ParseFilesystem(fs string) storage_v1alpha.DiskFilesystem { + switch strings.TrimPrefix(strings.ToLower(fs), "filesystem.") { + case "xfs": + return storage_v1alpha.XFS + case "btrfs": + return storage_v1alpha.BTRFS + default: + return storage_v1alpha.EXT4 + } +} + func DetectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { return storage_v1alpha.VM_ACCELERATOR diff --git a/pkg/workloadroles/roles.go b/pkg/workloadroles/roles.go index 29edfe659..b389716b6 100644 --- a/pkg/workloadroles/roles.go +++ b/pkg/workloadroles/roles.go @@ -155,9 +155,11 @@ func clusterAdminPerms() perms { "internalhttp": set("dorequest"), "disks": set("new", "delete"), "addons": set("createinstance", "deleteinstance"), - // Backup and restore read and rewrite a disk's contents wholesale, - // so they sit with the other disk mutations rather than with reads. - "diskbackup": set("backup", "restore", "listbackups"), + // Backup, restore and recovery read and rewrite a disk's contents + // wholesale, so they sit with the other disk mutations rather than + // with reads — listdeleted included, since what it lists is the + // data of disks somebody deleted. + "diskbackup": set("backup", "restore", "listbackups", "listdeleted", "undelete"), }, ) } diff --git a/servers/disk/server.go b/servers/disk/server.go index 31c7af8ba..9416388d6 100644 --- a/servers/disk/server.go +++ b/servers/disk/server.go @@ -24,16 +24,20 @@ import ( "miren.dev/runtime/components/diskio" "miren.dev/runtime/pkg/cond" "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/rpc/standard" "miren.dev/runtime/pkg/rpc/stream" "miren.dev/runtime/pkg/snapshot" ) // diskEntities is what the handlers need from the entity store: which disk the -// operator means, and how to conjure one that does not exist yet. +// operator means, how to conjure one that does not exist yet, and which node +// stateful workloads run on. type diskEntities interface { snapshot.DiskResolver snapshot.DiskCreator + + FindNodeId(ctx context.Context) (entity.Id, error) } // Server implements disk_v1alpha.DiskBackup. @@ -42,6 +46,12 @@ type Server struct { disks diskEntities dataPath string + // Recovering a deleted disk rebuilds entities from what the holding area + // recorded, rather than from anything a caller supplied, so it writes to + // the entity store directly instead of through the resolver. + eac *entityserver_v1alpha.EntityAccessClient + ec *entityserver.Client + // updates is nil on a cluster with no miren.cloud registration. The // streaming paths still work in that case; only the cloud ones refuse. updates diskio.CloudUpdatesClient @@ -65,6 +75,8 @@ func NewServer( log: log, disks: diskresolve.New(eac, ec), dataPath: dataPath, + eac: eac, + ec: ec, updates: updates, mntOps: diskio.NewRealDiskMountOps(log), } diff --git a/servers/disk/server_test.go b/servers/disk/server_test.go index ff873c0e9..cd56033cf 100644 --- a/servers/disk/server_test.go +++ b/servers/disk/server_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/snapshot" ) @@ -23,6 +24,7 @@ type fakeDisks struct { volume *snapshot.VolumeState leases []snapshot.LeaseState created *snapshot.RestoreTarget + nodeErr error } func (f *fakeDisks) FindDisk(_ context.Context, name string) (*snapshot.DiskState, error) { @@ -43,6 +45,13 @@ func (f *fakeDisks) FindLeases(context.Context, string) ([]snapshot.LeaseState, return f.leases, nil } +func (f *fakeDisks) FindNodeId(context.Context) (entity.Id, error) { + if f.nodeErr != nil { + return "", f.nodeErr + } + return entity.Id("node/n1"), nil +} + func (f *fakeDisks) CreateDiskAndVolume(context.Context, string, int64, string, string) (*snapshot.RestoreTarget, error) { if f.created == nil { return nil, errors.New("creation not configured") diff --git a/servers/disk/undelete.go b/servers/disk/undelete.go new file mode 100644 index 000000000..7193f68db --- /dev/null +++ b/servers/disk/undelete.go @@ -0,0 +1,295 @@ +package disk + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/idgen" +) + +// diskDataPath is where volume directories and the soft-delete holding area +// live under the server's data directory. +func (s *Server) diskDataPath() string { + return filepath.Join(s.dataPath, "disk-data") +} + +// ListDeleted reports the disks still recoverable from the soft-delete holding +// area. +func (s *Server) ListDeleted(ctx context.Context, state *disk_v1alpha.DiskBackupListDeleted) error { + disks, retention, err := s.listDeleted() + if err != nil { + return err + } + state.Results().SetDisks(disks) + state.Results().SetRetentionDays(int32(retention)) + return nil +} + +func (s *Server) listDeleted() ([]*disk_v1alpha.DeletedDisk, int, error) { + entries, err := diskio.ListDeletedVolumes(s.diskDataPath()) + if err != nil { + return nil, 0, fmt.Errorf("listing deleted volumes: %w", err) + } + + // Newest deletion first: the disk someone wants back is almost always the + // one they just lost. + sort.Slice(entries, func(i, j int) bool { + return entries[i].Metadata.DeletedAt.After(entries[j].Metadata.DeletedAt) + }) + + retention := diskio.DefaultDeletedVolumeGCConfig().RetentionDays + + disks := make([]*disk_v1alpha.DeletedDisk, 0, len(entries)) + for _, e := range entries { + meta := e.Metadata + + d := new(disk_v1alpha.DeletedDisk) + d.SetDiskName(meta.DiskName) + d.SetVolumeId(meta.VolumeID) + d.SetSizeGb(meta.SizeGb) + d.SetFilesystem(meta.Filesystem) + d.SetVolumeMode(meta.VolumeMode) + if ts := timestamp(meta.DeletedAt); ts != nil { + d.SetDeletedAt(ts) + } + expiry := meta.DeletedAt.Add(time.Duration(retention) * 24 * time.Hour) + if ts := timestamp(expiry); ts != nil { + d.SetExpiresAt(ts) + } + disks = append(disks, d) + } + + return disks, retention, nil +} + +// Undelete moves a deleted volume's data back into place and recreates the +// entities that point at it. +func (s *Server) Undelete(ctx context.Context, state *disk_v1alpha.DiskBackupUndelete) error { + args := state.Args() + + out, err := s.undelete(ctx, args.Disk(), args.VolumeId()) + if err != nil { + return err + } + + res := new(disk_v1alpha.UndeleteResult) + res.SetDisk(out.name) + res.SetDiskId(string(out.diskID)) + res.SetVolumeId(out.volumeID) + res.SetImagePath(out.imagePath) + state.Results().SetResult(&res) + return nil +} + +// undeleteResult is what a recovery produced, for the caller to report. +type undeleteResult struct { + name string + diskID entity.Id + volumeID string + imagePath string +} + +func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undeleteResult, retErr error) { + if name == "" { + return nil, refuse("disk name is required") + } + + entry, err := s.findDeleted(name, volumeID) + if err != nil { + return nil, err + } + meta := entry.Metadata + + // A live disk already owning this name would end up with two entities + // answering to it, and every lookup by name is ambiguous from then on. + if _, err := s.disks.FindDisk(ctx, name); err == nil { + return nil, refuse("a disk named %q already exists — rename or delete it before recovering this one", name) + } + + filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") + + diskEntityId, err := s.createRestoringDisk(ctx, meta, filesystem) + if err != nil { + return nil, err + } + + volID := meta.VolumeID + destPath := filepath.Join(s.diskDataPath(), "volumes", volID) + + // The runner creates this on startup, but a recovery can be the first thing + // that happens on a rebuilt host, and rename will not create it. + if err := os.MkdirAll(filepath.Dir(destPath), 0700); err != nil { + s.deleteEntity(ctx, string(diskEntityId), "disk") + return nil, fmt.Errorf("creating volumes directory: %w", err) + } + + if err := os.Rename(entry.Path, destPath); err != nil { + s.deleteEntity(ctx, string(diskEntityId), "disk") + return nil, fmt.Errorf("moving volume back to %s: %w", destPath, err) + } + + // Until the last entity write lands, put everything back: the data returns + // to the holding area with its metadata intact, so a failed recovery can be + // retried rather than leaving a volume directory nothing references. + committed := false + defer func() { + if committed { + return + } + // The request's context is already cancelled when a client disconnects + // mid-recovery, and rolling back is exactly what has to happen then. + rctx := context.WithoutCancel(ctx) + if rerr := os.Rename(destPath, entry.Path); rerr != nil { + s.log.Warn("failed to move volume back to deleted-volumes", "volume_id", volID, "error", rerr) + } + s.deleteEntity(rctx, string(diskEntityId), "disk") + }() + + imagePath := filepath.Join(destPath, "disk.img") + if _, err := os.Stat(imagePath); err != nil { + return nil, refuse("disk image not found at %s — the deleted volume may be corrupted", imagePath) + } + + nodeId, err := s.disks.FindNodeId(ctx) + if err != nil { + s.log.Warn("could not find node, using the one recorded at deletion", + "disk", name, "node_id", meta.NodeID, "error", err) + nodeId = meta.NodeID.Id() + } + + volEntityId := entity.Id("disk_volume/" + volID) + vol := &storage_v1alpha.DiskVolume{ + Name: meta.DiskName, + DiskId: diskEntityId, + VolumeId: volID, + SizeGb: meta.SizeGb, + Filesystem: filesystem, + VolumeMode: storage_v1alpha.DiskVolumeVolumeMode(meta.VolumeMode), + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY. The runner's DiskVolumeController drives + // the recovered volume through the same mount-then-READY ordering as a + // fresh create (skipping the reimage, since disk.img is already in + // place). Advertising DV_READY here would let a lease bind against a + // volume that isn't registered in the runner's in-memory state yet, + // which lands the lease in a terminal FAILED (MIR-1469). + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, + } + + if _, err := s.eac.Create(ctx, entity.New(entity.DBId, volEntityId, vol.Encode).Attrs()); err != nil { + return nil, fmt.Errorf("creating disk_volume entity: %w", err) + } + + // PROVISIONING, not PROVISIONED. The DiskController promotes it only once + // the disk_volume actually reaches DV_READY — after the volume is + // registered and mounted on the node — so the disk never reports itself + // leasable ahead of a real mount. + _, err = s.eac.Patch(ctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volID), + }, 0) + if err != nil { + s.deleteEntity(ctx, string(volEntityId), "disk_volume") + return nil, fmt.Errorf("updating disk to provisioning: %w", err) + } + + // Committed. The metadata file only exists to describe a volume while it + // sits in the holding area, so it goes now that the volume is live again. + if rerr := os.Remove(filepath.Join(destPath, "metadata.json")); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove deleted-volume metadata", "volume_id", volID, "error", rerr) + } + committed = true + + s.log.Info("recovered deleted disk", + "disk", name, "disk_id", diskEntityId, "volume_id", volID) + + return &undeleteResult{ + name: name, + diskID: diskEntityId, + volumeID: volID, + imagePath: imagePath, + }, nil +} + +// findDeleted picks the one deleted volume the caller meant. +func (s *Server) findDeleted(name, volumeID string) (*diskio.DeletedVolumeEntry, error) { + entries, err := diskio.ListDeletedVolumes(s.diskDataPath()) + if err != nil { + return nil, fmt.Errorf("listing deleted volumes: %w", err) + } + + var matches []diskio.DeletedVolumeEntry + for _, e := range entries { + if e.Metadata.DiskName != name { + continue + } + if volumeID != "" && e.Metadata.VolumeID != volumeID { + continue + } + matches = append(matches, e) + } + + switch len(matches) { + case 0: + if volumeID != "" { + return nil, refuse("no deleted disk named %q with volume %s", name, volumeID) + } + return nil, refuse("no deleted disk found named %q", name) + case 1: + return &matches[0], nil + } + + ids := make([]string, 0, len(matches)) + for _, m := range matches { + ids = append(ids, m.Metadata.VolumeID) + } + return nil, refuse( + "several deleted disks are named %q — pass one of these volume ids to say which: %s", + name, strings.Join(ids, ", "), + ) +} + +// createRestoringDisk makes the disk entity in RESTORING so the disk controller +// leaves it alone while the volume is moved back into place. +func (s *Server) createRestoringDisk( + ctx context.Context, + meta *diskio.DeletedVolumeMetadata, + filesystem string, +) (entity.Id, error) { + disk := &storage_v1alpha.Disk{ + Name: meta.DiskName, + SizeGb: meta.SizeGb, + Filesystem: diskresolve.ParseFilesystem(filesystem), + Status: storage_v1alpha.RESTORING, + } + + // A fresh id, not the one the disk had before it was deleted. Anything that + // still holds a reference to the old id is holding a reference to something + // the operator deleted, and reusing the id would silently reconnect it. + id, err := s.ec.Create(ctx, idgen.GenNS("disk"), disk) + if err != nil { + return "", fmt.Errorf("creating disk entity: %w", err) + } + return id, nil +} + +// deleteEntity is best-effort cleanup on a path that is already failing, so it +// reports rather than returns: the error that got us here is the one to keep. +func (s *Server) deleteEntity(ctx context.Context, id, kind string) { + if _, err := s.eac.Delete(ctx, id); err != nil { + s.log.Warn("failed to clean up entity after a failed recovery", + "kind", kind, "id", id, "error", err) + } +} diff --git a/servers/disk/undelete_test.go b/servers/disk/undelete_test.go new file mode 100644 index 000000000..d06aa5315 --- /dev/null +++ b/servers/disk/undelete_test.go @@ -0,0 +1,248 @@ +package disk + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + compute "miren.dev/runtime/api/compute/compute_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/api/storage/storage_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/diskresolve" + "miren.dev/runtime/pkg/entity" + "miren.dev/runtime/pkg/entity/testutils" +) + +// newUndeleteServer wires a server over a real in-memory entity store, since +// undelete's whole job is entity writes and a fake would not exercise them. +func newUndeleteServer(t *testing.T) (*Server, *testutils.InMemEntityServer, string) { + t.Helper() + + es, cleanup := testutils.NewInMemEntityServer(t) + t.Cleanup(cleanup) + + // One coordinator node, so FindNodeId resolves rather than falling back to + // whatever was recorded at deletion. + _, err := es.EAC.Create(context.Background(), entity.New( + entity.DBId, entity.Id("node/n1"), + entity.Ref(entity.EntityKind, compute.KindNode), + ).Attrs()) + require.NoError(t, err) + + dataPath := t.TempDir() + ec := entityserver.NewClient(testutils.TestLogger(t), es.EAC) + + s := &Server{ + log: slog.Default(), + disks: diskresolve.New(es.EAC, ec), + dataPath: dataPath, + eac: es.EAC, + ec: ec, + mntOps: fakeMountOps{}, + } + return s, es, dataPath +} + +// seedDeleted puts a volume into the soft-delete holding area the way +// deleteVolume does: the directory, its image, and the metadata describing it. +func seedDeleted(t *testing.T, dataPath, diskName, volumeID string, deletedAt time.Time) string { + t.Helper() + + diskData := filepath.Join(dataPath, "disk-data") + dir := filepath.Join(diskio.DeletedVolumesPath(diskData), volumeID) + require.NoError(t, os.MkdirAll(dir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "disk.img"), []byte("disk contents"), 0644)) + + require.NoError(t, diskio.SaveDeletedVolumeMetadata(dir, &diskio.DeletedVolumeMetadata{ + DiskID: "disk/old-" + volumeID, + DiskName: diskName, + SizeGb: 1, + Filesystem: "ext4", + VolumeID: volumeID, + VolumeMode: string(storage_v1alpha.VM_UNIVERSAL), + NodeID: compute.NodeId("node/n1"), + DeletedAt: deletedAt, + })) + return dir +} + +func TestListDeletedIsEmptyWhenNothingWasDeleted(t *testing.T) { + s, _, _ := newUndeleteServer(t) + + disks, retention, err := s.listDeleted() + require.NoError(t, err) + assert.Empty(t, disks) + assert.Positive(t, retention, "the client shows an expiry, so retention has to come back") +} + +func TestListDeletedReportsNewestFirstWithAnExpiry(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + + older := time.Now().Add(-48 * time.Hour).Truncate(time.Second) + newer := time.Now().Add(-1 * time.Hour).Truncate(time.Second) + seedDeleted(t, dataPath, "old-disk", "vol-old", older) + seedDeleted(t, dataPath, "new-disk", "vol-new", newer) + + disks, retention, err := s.listDeleted() + require.NoError(t, err) + require.Len(t, disks, 2) + + // The disk someone wants back is almost always the one they just lost. + assert.Equal(t, "new-disk", disks[0].DiskName()) + assert.Equal(t, "old-disk", disks[1].DiskName()) + + assert.Equal(t, "vol-new", disks[0].VolumeId()) + assert.Equal(t, int64(1), disks[0].SizeGb()) + assert.Equal(t, "ext4", disks[0].Filesystem()) + + require.True(t, disks[0].HasDeletedAt()) + assert.Equal(t, newer.Unix(), disks[0].DeletedAt().Seconds()) + + // Expiry is what tells an operator how long they have to act on this. + require.True(t, disks[0].HasExpiresAt()) + wantExpiry := newer.Add(time.Duration(retention) * 24 * time.Hour) + assert.Equal(t, wantExpiry.Unix(), disks[0].ExpiresAt().Seconds()) +} + +func TestUndeleteMovesTheVolumeBackAndRecreatesEntities(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + holding := seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now().Add(-time.Hour)) + + res, err := s.undelete(context.Background(), "mydisk", "") + require.NoError(t, err) + + // The data is live again, and gone from the holding area. + imagePath := filepath.Join(dataPath, "disk-data", "volumes", "vol-1", "disk.img") + assert.FileExists(t, imagePath) + assert.NoDirExists(t, holding) + assert.Equal(t, imagePath, res.imagePath) + + // The metadata only described the volume while it sat in the holding area. + _, statErr := os.Stat(filepath.Join(filepath.Dir(imagePath), "metadata.json")) + assert.True(t, os.IsNotExist(statErr), "metadata.json should be gone once the volume is live") + + disks := listDisks(t, es) + require.Len(t, disks, 1) + // PROVISIONING, not PROVISIONED: the DiskController promotes it only once + // the volume is actually mounted, so it never reports itself leasable early. + assert.Equal(t, storage_v1alpha.PROVISIONING, disks[0].Status) + assert.Equal(t, "vol-1", disks[0].VolumeId) + + vols := listVolumes(t, es) + require.Len(t, vols, 1) + assert.Equal(t, storage_v1alpha.DV_PENDING, vols[0].ActualState) + assert.Equal(t, storage_v1alpha.DV_PRESENT, vols[0].DesiredState) + assert.Equal(t, imagePath, vols[0].ImagePath) + assert.Equal(t, entity.Id("node/n1"), vols[0].NodeId) +} + +// The recovered disk must not reuse the id of the disk that was deleted, or +// anything still pointing at that id gets silently reconnected. +func TestUndeleteGivesTheDiskAFreshId(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.NoError(t, err) + + disks := listDisks(t, es) + require.Len(t, disks, 1) + assert.NotEqual(t, entity.Id("disk/old-vol-1"), disks[0].ID) +} + +func TestUndeleteReportsAnUnknownName(t *testing.T) { + s, _, _ := newUndeleteServer(t) + + _, err := s.undelete(context.Background(), "nope", "") + require.Error(t, err) + assert.Contains(t, err.Error(), `no deleted disk found named "nope"`) +} + +// Two deletions of the same name are ordinary — delete, recreate, delete again +// — so the ambiguity has to be reported with the ids needed to resolve it. +func TestUndeleteNamesTheChoicesWhenSeveralShareAName(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now().Add(-2*time.Hour)) + seedDeleted(t, dataPath, "mydisk", "vol-2", time.Now().Add(-time.Hour)) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "vol-1") + assert.Contains(t, err.Error(), "vol-2") + + // Naming one resolves it. + res, err := s.undelete(context.Background(), "mydisk", "vol-2") + require.NoError(t, err) + assert.Equal(t, "vol-2", res.volumeID) +} + +func TestUndeleteRefusesWhenALiveDiskHoldsTheName(t *testing.T) { + s, _, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + // A live disk already answering to the name. + _, err := s.ec.Create(context.Background(), "disk/live", &storage_v1alpha.Disk{ + Name: "mydisk", + SizeGb: 1, + Filesystem: storage_v1alpha.EXT4, + Status: storage_v1alpha.PROVISIONED, + }) + require.NoError(t, err) + + _, err = s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") +} + +// A holding-area directory with no image is corrupt. Recovery must roll back +// rather than leave entities pointing at data that is not there. +func TestUndeleteRollsBackWhenTheImageIsMissing(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + holding := seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + require.NoError(t, os.Remove(filepath.Join(holding, "disk.img"))) + + _, err := s.undelete(context.Background(), "mydisk", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "may be corrupted") + + // The data went back to the holding area, so the operator can try again. + assert.DirExists(t, holding) + assert.NoDirExists(t, filepath.Join(dataPath, "disk-data", "volumes", "vol-1")) + + // And no disk entity was left behind. + assert.Empty(t, listDisks(t, es)) +} + +func listDisks(t *testing.T, es *testutils.InMemEntityServer) []storage_v1alpha.Disk { + t.Helper() + resp, err := es.EAC.List(context.Background(), entity.Ref(entity.EntityKind, storage_v1alpha.KindDisk)) + require.NoError(t, err) + + var out []storage_v1alpha.Disk + for _, v := range resp.Values() { + var d storage_v1alpha.Disk + d.Decode(v.Entity()) + out = append(out, d) + } + return out +} + +func listVolumes(t *testing.T, es *testutils.InMemEntityServer) []storage_v1alpha.DiskVolume { + t.Helper() + resp, err := es.EAC.List(context.Background(), entity.Ref(entity.EntityKind, storage_v1alpha.KindDiskVolume)) + require.NoError(t, err) + + var out []storage_v1alpha.DiskVolume + for _, v := range resp.Values() { + var vol storage_v1alpha.DiskVolume + vol.Decode(v.Entity()) + out = append(out, vol) + } + return out +} From 3d7a49ac90cf287c0b71078ff04ff2cf45ae8ee6 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 21:20:34 -0700 Subject: [PATCH 11/19] Resume interrupted disk backups and restores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFD-108 settles that a multi-gigabyte transfer has to checkpoint and resume, because a stream that fails near the end and restarts from zero is not acceptable at that size. It matters most over the Miren Anywhere relay (RFD-101), where a dropped cluster uplink ends every in-flight session and the cluster can take a minute to come back, so the failure is structural rather than rare. The transfer keeps the existing stream transport and gains a transfer id and an offset. The id comes from the client, not the server: an id the server handed back would be lost along with the call that carried it, which is precisely the call that failed. Each direction checkpoints where the bytes actually land. A backup stages its compressed snapshot under the transfer id and resumes by seeking into it, which also means a resumed backup does not re-read and re-compress the image — and must not, since a live disk is no longer the image the client already holds half of. An upload appends to a file the server fsyncs, and that file's length is the resume point, which transferOffset reports. The retry loop lives in the CLI and refuses to retry a refusal. "The disk is in use" means the same thing on all six attempts, and retrying it only delays the operator reading it. The server does the matching thing on its side: a refused upload is dropped rather than left to the sweep, since the client is never coming back for it. Abandoned transfers are reclaimed after 24 hours, swept when the next transfer starts rather than by a timer that would have to be owned and shut down. Not covered, deliberately: the cloud paths. Those bytes move between the server and miren.cloud over its own connection, so an interruption there is not on the client's session and re-fetching costs only time. --- api/disk/disk_v1alpha/rpc.gen.go | 206 ++++++++++++++++++++++- api/disk/rpc.yml | 36 ++++ cli/commands/disk_backup.go | 53 +++++- cli/commands/disk_restore.go | 38 ++++- cli/commands/disk_transfer.go | 96 +++++++++++ cli/commands/disk_transfer_test.go | 61 +++++++ pkg/workloadroles/roles.go | 5 +- servers/disk/backup.go | 181 ++++++++++++++++----- servers/disk/restore.go | 151 +++++++++++++++-- servers/disk/resume_test.go | 215 ++++++++++++++++++++++++ servers/disk/transfer.go | 253 +++++++++++++++++++++++++++++ servers/disk/transfer_test.go | 173 ++++++++++++++++++++ 12 files changed, 1391 insertions(+), 77 deletions(-) create mode 100644 cli/commands/disk_transfer.go create mode 100644 cli/commands/disk_transfer_test.go create mode 100644 servers/disk/resume_test.go create mode 100644 servers/disk/transfer.go create mode 100644 servers/disk/transfer_test.go diff --git a/api/disk/disk_v1alpha/rpc.gen.go b/api/disk/disk_v1alpha/rpc.gen.go index baaea6a11..b2987f41f 100644 --- a/api/disk/disk_v1alpha/rpc.gen.go +++ b/api/disk/disk_v1alpha/rpc.gen.go @@ -708,11 +708,13 @@ func (v *RestoreResult) UnmarshalJSON(data []byte) error { } type diskBackupBackupArgsData struct { - Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` - ToCloud *bool `cbor:"1,keyasint,omitempty" json:"to_cloud,omitempty"` - Pin *string `cbor:"2,keyasint,omitempty" json:"pin,omitempty"` - Data *rpc.Capability `cbor:"3,keyasint,omitempty" json:"data,omitempty"` - Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` + Disk *string `cbor:"0,keyasint,omitempty" json:"disk,omitempty"` + ToCloud *bool `cbor:"1,keyasint,omitempty" json:"to_cloud,omitempty"` + Pin *string `cbor:"2,keyasint,omitempty" json:"pin,omitempty"` + Data *rpc.Capability `cbor:"3,keyasint,omitempty" json:"data,omitempty"` + Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` + TransferId *string `cbor:"5,keyasint,omitempty" json:"transfer_id,omitempty"` + Offset *int64 `cbor:"6,keyasint,omitempty" json:"offset,omitempty"` } type DiskBackupBackupArgs struct { @@ -775,6 +777,28 @@ func (v *DiskBackupBackupArgs) Progress() *stream.SendStreamClient[*Progress] { return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} } +func (v *DiskBackupBackupArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupBackupArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupBackupArgs) HasOffset() bool { + return v.data.Offset != nil +} + +func (v *DiskBackupBackupArgs) Offset() int64 { + if v.data.Offset == nil { + return 0 + } + return *v.data.Offset +} + func (v *DiskBackupBackupArgs) MarshalCBOR() ([]byte, error) { return cbor.Marshal(v.data) } @@ -892,6 +916,8 @@ type diskBackupRestoreArgsData struct { Data *rpc.Capability `cbor:"2,keyasint,omitempty" json:"data,omitempty"` Force *bool `cbor:"3,keyasint,omitempty" json:"force,omitempty"` Progress *rpc.Capability `cbor:"4,keyasint,omitempty" json:"progress,omitempty"` + TransferId *string `cbor:"5,keyasint,omitempty" json:"transfer_id,omitempty"` + Offset *int64 `cbor:"6,keyasint,omitempty" json:"offset,omitempty"` } type DiskBackupRestoreArgs struct { @@ -954,6 +980,28 @@ func (v *DiskBackupRestoreArgs) Progress() *stream.SendStreamClient[*Progress] { return &stream.SendStreamClient[*Progress]{Client: v.call.NewClient(v.data.Progress)} } +func (v *DiskBackupRestoreArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupRestoreArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupRestoreArgs) HasOffset() bool { + return v.data.Offset != nil +} + +func (v *DiskBackupRestoreArgs) Offset() int64 { + if v.data.Offset == nil { + return 0 + } + return *v.data.Offset +} + func (v *DiskBackupRestoreArgs) MarshalCBOR() ([]byte, error) { return cbor.Marshal(v.data) } @@ -999,6 +1047,71 @@ func (v *DiskBackupRestoreResults) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &v.data) } +type diskBackupTransferOffsetArgsData struct { + TransferId *string `cbor:"0,keyasint,omitempty" json:"transfer_id,omitempty"` +} + +type DiskBackupTransferOffsetArgs struct { + call rpc.Call + data diskBackupTransferOffsetArgsData +} + +func (v *DiskBackupTransferOffsetArgs) HasTransferId() bool { + return v.data.TransferId != nil +} + +func (v *DiskBackupTransferOffsetArgs) TransferId() string { + if v.data.TransferId == nil { + return "" + } + return *v.data.TransferId +} + +func (v *DiskBackupTransferOffsetArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupTransferOffsetArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type diskBackupTransferOffsetResultsData struct { + ReceivedBytes *int64 `cbor:"0,keyasint,omitempty" json:"received_bytes,omitempty"` +} + +type DiskBackupTransferOffsetResults struct { + call rpc.Call + data diskBackupTransferOffsetResultsData +} + +func (v *DiskBackupTransferOffsetResults) SetReceivedBytes(received_bytes int64) { + v.data.ReceivedBytes = &received_bytes +} + +func (v *DiskBackupTransferOffsetResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *DiskBackupTransferOffsetResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *DiskBackupTransferOffsetResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type diskBackupListDeletedArgsData struct{} type DiskBackupListDeletedArgs struct { @@ -1212,6 +1325,32 @@ func (t *DiskBackupRestore) Results() *DiskBackupRestoreResults { return results } +type DiskBackupTransferOffset struct { + rpc.Call + args DiskBackupTransferOffsetArgs + results DiskBackupTransferOffsetResults +} + +func (t *DiskBackupTransferOffset) Args() *DiskBackupTransferOffsetArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *DiskBackupTransferOffset) Results() *DiskBackupTransferOffsetResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type DiskBackupListDeleted struct { rpc.Call args DiskBackupListDeletedArgs @@ -1268,6 +1407,7 @@ type DiskBackup interface { Backup(ctx context.Context, state *DiskBackupBackup) error ListBackups(ctx context.Context, state *DiskBackupListBackups) error Restore(ctx context.Context, state *DiskBackupRestore) error + TransferOffset(ctx context.Context, state *DiskBackupTransferOffset) error ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error Undelete(ctx context.Context, state *DiskBackupUndelete) error } @@ -1288,6 +1428,10 @@ func (reexportDiskBackup) Restore(ctx context.Context, state *DiskBackupRestore) panic("not implemented") } +func (reexportDiskBackup) TransferOffset(ctx context.Context, state *DiskBackupTransferOffset) error { + panic("not implemented") +} + func (reexportDiskBackup) ListDeleted(ctx context.Context, state *DiskBackupListDeleted) error { panic("not implemented") } @@ -1307,7 +1451,7 @@ func AdaptDiskBackup(t DiskBackup) *rpc.Interface { InterfaceName: "DiskBackup", Index: 0, Public: false, - Params: []string{"disk", "to_cloud", "pin", "data", "progress"}, + Params: []string{"disk", "to_cloud", "pin", "data", "progress", "transfer_id", "offset"}, Handler: func(ctx context.Context, call rpc.Call) error { return t.Backup(ctx, &DiskBackupBackup{Call: call}) }, @@ -1327,11 +1471,21 @@ func AdaptDiskBackup(t DiskBackup) *rpc.Interface { InterfaceName: "DiskBackup", Index: 0, Public: false, - Params: []string{"disk", "restore_point", "data", "force", "progress"}, + Params: []string{"disk", "restore_point", "data", "force", "progress", "transfer_id", "offset"}, Handler: func(ctx context.Context, call rpc.Call) error { return t.Restore(ctx, &DiskBackupRestore{Call: call}) }, }, + { + Name: "transferOffset", + InterfaceName: "DiskBackup", + Index: 0, + Public: false, + Params: []string{"transfer_id"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.TransferOffset(ctx, &DiskBackupTransferOffset{Call: call}) + }, + }, { Name: "listDeleted", InterfaceName: "DiskBackup", @@ -1385,7 +1539,7 @@ func (v *DiskBackupClientBackupResults) Result() *BackupResult { return *v.data.Result } -func (v DiskBackupClient) Backup(ctx context.Context, disk string, to_cloud bool, pin string, data stream.SendStream[[]byte], progress stream.SendStream[*Progress]) (*DiskBackupClientBackupResults, error) { +func (v DiskBackupClient) Backup(ctx context.Context, disk string, to_cloud bool, pin string, data stream.SendStream[[]byte], progress stream.SendStream[*Progress], transfer_id string, offset int64) (*DiskBackupClientBackupResults, error) { args := DiskBackupBackupArgs{} caps := map[rpc.OID]*rpc.InlineCapability{} args.data.Disk = &disk @@ -1401,6 +1555,8 @@ func (v DiskBackupClient) Backup(ctx context.Context, disk string, to_cloud bool args.data.Progress = c caps[oid] = ic } + args.data.TransferId = &transfer_id + args.data.Offset = &offset var ret diskBackupBackupResultsData @@ -1458,7 +1614,7 @@ func (v *DiskBackupClientRestoreResults) Result() *RestoreResult { return *v.data.Result } -func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_point string, data stream.RecvStream[[]byte], force bool, progress stream.SendStream[*Progress]) (*DiskBackupClientRestoreResults, error) { +func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_point string, data stream.RecvStream[[]byte], force bool, progress stream.SendStream[*Progress], transfer_id string, offset int64) (*DiskBackupClientRestoreResults, error) { args := DiskBackupRestoreArgs{} caps := map[rpc.OID]*rpc.InlineCapability{} args.data.Disk = &disk @@ -1474,6 +1630,8 @@ func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_poin args.data.Progress = c caps[oid] = ic } + args.data.TransferId = &transfer_id + args.data.Offset = &offset var ret diskBackupRestoreResultsData @@ -1485,6 +1643,36 @@ func (v DiskBackupClient) Restore(ctx context.Context, disk string, restore_poin return &DiskBackupClientRestoreResults{client: v.Client, data: ret}, nil } +type DiskBackupClientTransferOffsetResults struct { + client rpc.Client + data diskBackupTransferOffsetResultsData +} + +func (v *DiskBackupClientTransferOffsetResults) HasReceivedBytes() bool { + return v.data.ReceivedBytes != nil +} + +func (v *DiskBackupClientTransferOffsetResults) ReceivedBytes() int64 { + if v.data.ReceivedBytes == nil { + return 0 + } + return *v.data.ReceivedBytes +} + +func (v DiskBackupClient) TransferOffset(ctx context.Context, transfer_id string) (*DiskBackupClientTransferOffsetResults, error) { + args := DiskBackupTransferOffsetArgs{} + args.data.TransferId = &transfer_id + + var ret diskBackupTransferOffsetResultsData + + err := v.Call(ctx, "transferOffset", &args, &ret) + if err != nil { + return nil, err + } + + return &DiskBackupClientTransferOffsetResults{client: v.Client, data: ret}, nil +} + type DiskBackupClientListDeletedResults struct { client rpc.Client data diskBackupListDeletedResultsData diff --git a/api/disk/rpc.yml b/api/disk/rpc.yml index ca058a978..eb246c2a5 100644 --- a/api/disk/rpc.yml +++ b/api/disk/rpc.yml @@ -212,6 +212,18 @@ interfaces: - name: progress type: stream.SendStream[*Progress] doc: Stream the client receives progress events on + - name: transfer_id + type: string + doc: > + Names this transfer so a dropped connection can pick it up again. + The client chooses it, because a value the server handed back + would be lost with the call that carried it. Retrying with the + same id resumes; a new id starts over. + - name: offset + type: int64 + doc: > + Byte of the compressed snapshot to start sending from. A resuming + client passes what it already has on disk. results: - name: result type: '*BackupResult' @@ -252,10 +264,34 @@ interfaces: - name: progress type: stream.SendStream[*Progress] doc: Stream the client receives progress events on + - name: transfer_id + type: string + doc: > + Names this transfer so a dropped connection can pick it up again. + Required when streaming a snapshot up; ignored for a restore point, + which the server fetches itself. + - name: offset + type: int64 + doc: > + Byte of the snapshot the client is sending from. It must match + what the server already holds, which transferOffset reports. results: - name: result type: '*RestoreResult' + - name: transferOffset + doc: > + How much of a transfer the server already holds, so a client resuming + an interrupted restore knows where to start sending. Unknown ids + report zero, which is what makes the first attempt and a retry the + same code path on the client. + parameters: + - name: transfer_id + type: string + results: + - name: received_bytes + type: int64 + - name: listDeleted doc: > List disks whose data is still recoverable from the soft-delete diff --git a/cli/commands/disk_backup.go b/cli/commands/disk_backup.go index 6a0f6bc66..881443267 100644 --- a/cli/commands/disk_backup.go +++ b/cli/commands/disk_backup.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "io" "os" "time" @@ -35,10 +36,17 @@ func DiskBackup(ctx *Context, opts struct { start := time.Now() progress := diskProgress(ctx) + transferID, err := newTransferID() + if err != nil { + return err + } + if opts.Cloud { ctx.Info("Backing up disk %q to miren.cloud", opts.Name) - res, err := dc.Backup(ctx, opts.Name, true, opts.Pin, nil, progress) + // No resume here: the bytes go from the server straight to the cloud, + // so nothing is riding on this connection but the request itself. + res, err := dc.Backup(ctx, opts.Name, true, opts.Pin, nil, progress, transferID, 0) if err != nil { return err } @@ -73,17 +81,56 @@ func DiskBackup(ctx *Context, opts struct { ctx.Info("Backing up disk %q", opts.Name) ctx.Info("Output: %s", outputPath) - res, err := dc.Backup(ctx, opts.Name, false, "", stream.ServeWriter(ctx, outFile), progress) + var result *disk_v1alpha.BackupResult + + err = runTransfer(ctx, "Backup", func(try int) error { + // What is durably in the file is the resume point, and it is the + // client's to know: it is the only end that can see how much of the + // stream actually reached disk. + offset, oerr := syncedSize(outFile) + if oerr != nil { + return oerr + } + + res, berr := dc.Backup(ctx, opts.Name, false, "", stream.ServeWriter(ctx, outFile), progress, transferID, offset) + if berr != nil { + return berr + } + result = res.Result() + return nil + }) if err != nil { return err } complete = true - reportBackup(ctx, res.Result(), time.Since(start)) + reportBackup(ctx, result, time.Since(start)) ctx.Info(" Snapshot: %s", outputPath) return nil } +// syncedSize flushes what has been written and reports how much of it is +// durably on disk. +// +// This is the number the server is told to continue from, so it has to be what +// survived rather than what was handed to the kernel: asking to resume past +// bytes that a crash could still take back would leave a hole in the snapshot. +func syncedSize(f *os.File) (int64, error) { + if err := f.Sync(); err != nil { + return 0, fmt.Errorf("flushing snapshot: %w", err) + } + info, err := f.Stat() + if err != nil { + return 0, fmt.Errorf("stat snapshot: %w", err) + } + // Writes go on at the end, so the offset has to follow the length: a retry + // that left the file position short would overwrite good bytes. + if _, err := f.Seek(info.Size(), io.SeekStart); err != nil { + return 0, fmt.Errorf("seeking snapshot: %w", err) + } + return info.Size(), nil +} + func reportBackup(ctx *Context, res *disk_v1alpha.BackupResult, took time.Duration) { ctx.Info("Backup complete") if res == nil { diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index 90186b0b7..565468184 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -61,7 +61,9 @@ func DiskRestore(ctx *Context, opts struct { ctx.Info("Restoring disk %q from restore point %s", opts.Name, point) - res, err := dc.Restore(ctx, opts.Name, point, nil, opts.Force, progress) + // No resume here: the server fetches from the cloud over its own + // connection, so nothing rides on this one but the request. + res, err := dc.Restore(ctx, opts.Name, point, nil, opts.Force, progress, "", 0) if err != nil { return err } @@ -96,11 +98,41 @@ func DiskRestore(ctx *Context, opts struct { ctx.Info("Restoring disk %q from %s", name, opts.Snapshot) - res, err := dc.Restore(ctx, name, "", stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), opts.Force, progress) + transferID, err := newTransferID() if err != nil { return err } - reportRestore(ctx, res.Result(), time.Since(start)) + + var result *disk_v1alpha.RestoreResult + + err = runTransfer(ctx, "Restore", func(try int) error { + // The server is the only end that knows how much of the upload it + // durably has, so ask rather than assume. An id it has never seen + // reports zero, which makes the first attempt and a retry identical. + off, oerr := dc.TransferOffset(ctx, transferID) + if oerr != nil { + return oerr + } + offset := off.ReceivedBytes() + + if _, serr := snapFile.Seek(offset, io.SeekStart); serr != nil { + return fmt.Errorf("seeking snapshot to %d: %w", offset, serr) + } + + res, rerr := dc.Restore(ctx, name, "", + stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), + opts.Force, progress, transferID, offset) + if rerr != nil { + return rerr + } + result = res.Result() + return nil + }) + if err != nil { + return err + } + + reportRestore(ctx, result, time.Since(start)) return nil } diff --git a/cli/commands/disk_transfer.go b/cli/commands/disk_transfer.go new file mode 100644 index 000000000..de7882fc7 --- /dev/null +++ b/cli/commands/disk_transfer.go @@ -0,0 +1,96 @@ +package commands + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "miren.dev/runtime/pkg/cond" +) + +const ( + // transferAttempts is how many times a backup or restore will pick itself + // up after the connection drops. + // + // This is not a tight retry loop around a flaky call — each attempt resumes + // from where the last one stopped, so the work already done is kept and the + // cost of another attempt is only the time to reconnect. Six covers a + // cluster uplink that flaps repeatedly during one long transfer, which is + // the case that motivated resume in the first place. + transferAttempts = 6 + + // transferRetryDelay is the wait before picking a transfer back up. A + // cluster's link to the cloud can take up to a minute to come back + // (RFD-101), so this backs off toward that rather than hammering. + transferRetryDelay = 5 * time.Second + transferRetryMax = 60 * time.Second +) + +// newTransferID names one backup or restore so an interrupted one can be +// resumed. +// +// The client picks it, not the server: an id the server handed back would be +// lost along with the call that carried it, which is precisely the call that +// failed. +func newTransferID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("generating a transfer id: %w", err) + } + return hex.EncodeToString(b), nil +} + +// resumable reports whether an error is worth another attempt. +// +// A refusal is not. The server declining to overwrite a mounted disk, or +// reporting that no such disk exists, means the same thing on every attempt, +// and retrying it six times only makes the operator wait longer to read it. +func resumable(err error) bool { + if err == nil { + return false + } + + var validation cond.ErrValidationFailure + var notFound cond.ErrNotFound + return !errors.As(err, &validation) && !errors.As(err, ¬Found) +} + +// runTransfer calls attempt until it succeeds, the error turns out not to be +// worth retrying, or the attempts run out. +// +// Each attempt is expected to resume rather than restart: this loop exists to +// survive a dropped connection, not to paper over a call that fails the same +// way every time. +func runTransfer(ctx *Context, what string, attempt func(try int) error) error { + delay := transferRetryDelay + + for try := 1; ; try++ { + err := attempt(try) + if err == nil { + return nil + } + if !resumable(err) || try >= transferAttempts { + return err + } + // A cancelled command is the operator's decision, not a failure to + // retry around. + if ctx.Err() != nil { + return err + } + + ctx.Warn("%s was interrupted: %v", what, err) + ctx.Info("Picking up where it stopped in %s (attempt %d of %d)", delay, try+1, transferAttempts) + + select { + case <-time.After(delay): + case <-ctx.Done(): + return err + } + + if delay *= 2; delay > transferRetryMax { + delay = transferRetryMax + } + } +} diff --git a/cli/commands/disk_transfer_test.go b/cli/commands/disk_transfer_test.go new file mode 100644 index 000000000..c5a424cf3 --- /dev/null +++ b/cli/commands/disk_transfer_test.go @@ -0,0 +1,61 @@ +package commands + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" +) + +// Retrying a refusal six times only makes the operator wait longer to read a +// message that will say the same thing every time. +func TestRefusalsAreNotRetried(t *testing.T) { + refusals := []error{ + cond.ValidationFailure("disk-backup", `disk "data" is in use`), + cond.NotFound("disk", "data"), + fmt.Errorf("wrapped: %w", cond.ValidationFailure("disk-backup", "no cloud")), + } + for _, err := range refusals { + assert.False(t, resumable(err), "should not retry: %v", err) + } +} + +// A dropped connection is exactly what resume exists for. +func TestTransportFailuresAreRetried(t *testing.T) { + assert.True(t, resumable(errors.New("connection reset by peer"))) + assert.True(t, resumable(errors.New("the cluster's link to the cloud dropped"))) +} + +func TestNilIsNotRetried(t *testing.T) { + assert.False(t, resumable(nil)) +} + +// Two transfers must not collide in the server's staging directory. +func TestTransferIDsAreDistinct(t *testing.T) { + seen := map[string]bool{} + for i := 0; i < 100; i++ { + id, err := newTransferID() + require.NoError(t, err) + require.NotEmpty(t, id) + assert.False(t, seen[id], "transfer ids must not repeat") + seen[id] = true + } +} + +// The id names a file on the server, which validates it — so the ids we +// generate have to be ones it accepts. +func TestTransferIDsUseOnlySafeCharacters(t *testing.T) { + id, err := newTransferID() + require.NoError(t, err) + + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + default: + t.Fatalf("transfer id %q contains %q, which the server rejects", id, r) + } + } +} diff --git a/pkg/workloadroles/roles.go b/pkg/workloadroles/roles.go index b389716b6..4331db7d7 100644 --- a/pkg/workloadroles/roles.go +++ b/pkg/workloadroles/roles.go @@ -159,7 +159,10 @@ func clusterAdminPerms() perms { // wholesale, so they sit with the other disk mutations rather than // with reads — listdeleted included, since what it lists is the // data of disks somebody deleted. - "diskbackup": set("backup", "restore", "listbackups", "listdeleted", "undelete"), + // transferoffset only reports how far an interrupted transfer got, + // but it is part of doing a backup or a restore, so it is granted + // with them rather than to every reader. + "diskbackup": set("backup", "restore", "listbackups", "listdeleted", "undelete", "transferoffset"), }, ) } diff --git a/servers/disk/backup.go b/servers/disk/backup.go index 83035dd04..3d7997a11 100644 --- a/servers/disk/backup.go +++ b/servers/disk/backup.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "path/filepath" "time" "miren.dev/runtime/api/disk/disk_v1alpha" @@ -97,58 +98,53 @@ func (s *Server) backupToCloud( // backupToClient compresses the image and streams it down to the caller, which // is how a cluster with no miren.cloud still produces a backup. // -// The snapshot is staged to a temp file rather than compressed straight into -// the stream: snapshot.Backup rewrites the header with the image's checksum -// once it has read the whole image, so it needs somewhere it can seek back to. +// The snapshot is staged to a file rather than compressed straight into the +// stream for two reasons. snapshot.Backup rewrites the header with the image's +// checksum once it has read the whole image, so it needs somewhere it can seek +// back to. And a staged snapshot is what makes the transfer resumable: a client +// that loses the connection at 90% comes back for the last 10% rather than +// making the server read and compress a multi-gigabyte image all over again. func (s *Server) backupToClient( ctx context.Context, state *disk_v1alpha.DiskBackupBackup, prog progressSink, target *snapshot.BackupTarget, ) error { - out := state.Args().Data() + args := state.Args() + + out := args.Data() if out == nil { return refuse("backup needs either --cloud or somewhere to write the snapshot") } - img, err := os.Open(target.ImagePath) - if err != nil { - return fmt.Errorf("opening disk image: %w", err) - } - defer img.Close() - - info, err := img.Stat() - if err != nil { - return fmt.Errorf("stat disk image: %w", err) - } - - prog.Message("Compressing %s (%d bytes)", target.Name, info.Size()) - - staged, err := os.CreateTemp(s.stagingDir(target.ImagePath), ".disk-backup-*") - if err != nil { - return fmt.Errorf("creating staging file: %w", err) - } - defer os.Remove(staged.Name()) - defer staged.Close() - - checksum, err := snapshot.Backup(staged, img, target.Name, info.Size(), target.Filesystem) + staged, meta, err := s.stageForClient(args.TransferId(), prog, target) if err != nil { return err } + defer staged.Close() - stagedInfo, err := staged.Stat() - if err != nil { - return fmt.Errorf("stat staging file: %w", err) + offset := args.Offset() + if offset < 0 || offset > meta.CompressedSize { + return refuse( + "this snapshot is %d bytes and the client offered to continue from %d", + meta.CompressedSize, offset, + ) } - if _, err := staged.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("rewind staging file: %w", err) + if _, err := staged.Seek(offset, io.SeekStart); err != nil { + return fmt.Errorf("seeking staged snapshot: %w", err) } - prog.Message("Sending %d bytes", stagedInfo.Size()) + if offset > 0 { + prog.Message("Resuming at %d of %d bytes", offset, meta.CompressedSize) + } else { + prog.Message("Sending %d bytes", meta.CompressedSize) + } w := stream.ToWriter(ctx, out) - sent, err := io.Copy(w, s.trackReads(staged, prog, stagedInfo.Size())) + sent, err := io.Copy(w, s.trackReads(staged, prog, offset, meta.CompressedSize)) if err != nil { + // The staged snapshot deliberately survives: the whole point is that + // the client can come back for the rest of it. return fmt.Errorf("sending snapshot: %w", err) } // The stream belongs to the caller, so closing our writer flushes what we @@ -156,42 +152,141 @@ func (s *Server) backupToClient( if err := w.Close(); err != nil { return fmt.Errorf("finishing snapshot stream: %w", err) } - if sent != stagedInfo.Size() { - return fmt.Errorf("sent %d bytes of a %d byte snapshot", sent, stagedInfo.Size()) + if want := meta.CompressedSize - offset; sent != want { + return fmt.Errorf("sent %d bytes of the %d remaining", sent, want) } + // Delivered in full, so there is nothing left to resume. + s.discardStaging(args.TransferId()) + s.log.Info("backed up disk to client", "disk", target.Name, - "image_size", info.Size(), - "compressed_size", stagedInfo.Size(), + "image_size", meta.ImageSize, + "compressed_size", meta.CompressedSize, ) res := new(disk_v1alpha.BackupResult) - res.SetImageSizeBytes(info.Size()) - res.SetCompressedSizeBytes(stagedInfo.Size()) - res.SetChecksum(checksum) + res.SetImageSizeBytes(meta.ImageSize) + res.SetCompressedSizeBytes(meta.CompressedSize) + res.SetChecksum(meta.Checksum) state.Results().SetResult(&res) return nil } -// trackReads reports progress as bytes are pulled out of r. -func (s *Server) trackReads(r io.Reader, prog progressSink, total int64) io.Reader { +// stageForClient returns the compressed snapshot to send, compressing it only +// if this transfer has not already produced one. +// +// Reusing it is not just an optimization. A second compression pass would read +// the image as it is now, which for a live disk is not the image the client has +// already half-downloaded, and stitching the two together would produce a file +// that is the right length and not a valid snapshot of anything. +func (s *Server) stageForClient( + transferID string, + prog progressSink, + target *snapshot.BackupTarget, +) (*os.File, *stagedSnapshot, error) { + if meta, err := s.loadStaging(transferID); err != nil { + return nil, nil, err + } else if meta != nil { + path, _ := s.transferPath(transferID) + f, oerr := os.Open(path) + if oerr != nil { + return nil, nil, fmt.Errorf("reopening staged snapshot: %w", oerr) + } + return f, meta, nil + } + + s.sweepTransfers() + + img, err := os.Open(target.ImagePath) + if err != nil { + return nil, nil, fmt.Errorf("opening disk image: %w", err) + } + defer img.Close() + + info, err := img.Stat() + if err != nil { + return nil, nil, fmt.Errorf("stat disk image: %w", err) + } + + prog.Message("Compressing %s (%d bytes)", target.Name, info.Size()) + + path, err := s.transferPath(transferID) + if err != nil { + return nil, nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, nil, fmt.Errorf("creating transfer directory: %w", err) + } + + staged, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) + if err != nil { + return nil, nil, fmt.Errorf("creating staged snapshot: %w", err) + } + + checksum, err := snapshot.Backup(staged, img, target.Name, info.Size(), target.Filesystem) + if err != nil { + staged.Close() + os.Remove(path) + return nil, nil, err + } + + stagedInfo, err := staged.Stat() + if err != nil { + staged.Close() + os.Remove(path) + return nil, nil, fmt.Errorf("stat staged snapshot: %w", err) + } + + meta := &stagedSnapshot{ + Disk: target.Name, + ImageSize: info.Size(), + CompressedSize: stagedInfo.Size(), + Checksum: checksum, + } + // Written only now, so a staging file left behind by a server that died + // mid-compression has no marker and is never mistaken for a finished one. + if err := s.saveStaging(transferID, meta); err != nil { + staged.Close() + os.Remove(path) + return nil, nil, err + } + + // Compression left the file at its end. Rewind so a freshly staged snapshot + // and one reopened for a resume are handed back the same way, rather than + // leaving every caller to remember which it got. + if _, err := staged.Seek(0, io.SeekStart); err != nil { + staged.Close() + return nil, nil, fmt.Errorf("rewinding staged snapshot: %w", err) + } + + return staged, meta, nil +} + +// trackReads reports progress as bytes are pulled out of r, counting from what +// the client already has so a resumed transfer shows a bar that keeps going up +// rather than restarting. +func (s *Server) trackReads(r io.Reader, prog progressSink, done, total int64) io.Reader { start := time.Now() - var done int64 + sentThisCall := int64(0) var lastReport time.Time return readerFunc(func(p []byte) (int, error) { n, err := r.Read(p) done += int64(n) + sentThisCall += int64(n) // One event per 250ms. The client renders a bar from these, and a // report per read would be thousands of RPCs for one backup. if now := time.Now(); now.Sub(lastReport) >= 250*time.Millisecond || err == io.EOF { lastReport = now + // Rate is measured over this attempt only. Counting bytes a + // previous attempt moved against this attempt's clock would + // report a throughput nothing achieved. elapsed := now.Sub(start).Seconds() var perSecond, eta int64 if elapsed > 0 { - perSecond = int64(float64(done) / elapsed) + perSecond = int64(float64(sentThisCall) / elapsed) } if perSecond > 0 && total > done { eta = (total - done) / perSecond diff --git a/servers/disk/restore.go b/servers/disk/restore.go index 0eb5040bb..5e486df36 100644 --- a/servers/disk/restore.go +++ b/servers/disk/restore.go @@ -2,6 +2,7 @@ package disk import ( "context" + "errors" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "miren.dev/runtime/api/disk/disk_v1alpha" "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/cond" "miren.dev/runtime/pkg/rpc/stream" "miren.dev/runtime/pkg/snapshot" ) @@ -28,7 +30,7 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest return refuse("disk name is required") } - src, compressedSize, closeSrc, err := s.restoreSource(ctx, name, args, prog) + src, total, closeSrc, err := s.restoreSource(ctx, name, args, prog) if err != nil { return err } @@ -48,11 +50,19 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest // Roll the entities back if anything below fails, but only when this // restore is what created them. defer func() { - if retErr != nil && target.Created && target.Cleanup != nil { + if retErr == nil { + return + } + if target.Created && target.Cleanup != nil { if cerr := target.Cleanup(ctx); cerr != nil { s.log.Warn("failed to clean up after restore", "disk", name, "error", cerr) } } + // A refusal is the one failure the client will not come back from, so + // the uploaded bytes are already dead. Drop them now rather than leave + // a copy of the snapshot sitting there until the sweep, which a + // repeatedly-refused restore would do once per attempt. + s.discardRefusedUpload(args, retErr) }() if err := s.refuseLiveImage(target, name); err != nil { @@ -70,7 +80,7 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest prog.Message("Restoring %s (%d bytes)", name, meta.SizeBytes) - if err := s.writeImage(target.ImagePath, src, compressedSize, meta, prog); err != nil { + if err := s.writeImage(target.ImagePath, src, total, meta, prog); err != nil { return err } @@ -80,6 +90,13 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest } } + // Installed, so the uploaded copy has served its purpose. Only now: until + // the image is in place, those bytes are the only copy on this host and a + // retry would have to send them again. + if id := args.TransferId(); id != "" && args.RestorePoint() == "" { + s.discardTransfer(id) + } + s.log.Info("restored disk", "disk", name, "image_size", meta.SizeBytes, @@ -94,31 +111,129 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest return nil } -// restoreSource opens the snapshot to restore from, whichever end it came from. +// discardRefusedUpload drops an uploaded snapshot the server has decided it +// will not use. +// +// The distinction that matters is refusal versus interruption. An interrupted +// upload is exactly what the transfer file is for and must survive; a refused +// one will never be picked up, because the client does not retry a refusal. +func (s *Server) discardRefusedUpload(args *disk_v1alpha.DiskBackupRestoreArgs, err error) { + if shouldDiscardUpload(args.TransferId(), args.RestorePoint(), err) { + s.discardTransfer(args.TransferId()) + } +} + +func shouldDiscardUpload(transferID, restorePoint string, err error) bool { + if transferID == "" || restorePoint != "" { + return false + } + var refusal cond.ErrValidationFailure + return errors.As(err, &refusal) +} + +// restoreSource resolves the snapshot to restore from and hands back a reader +// positioned at its start. // // The second return is the compressed size when it is known, and 0 when it is -// not. Progress is measured in compressed bytes because that is what actually -// moves; a client streaming a snapshot up has not told us how big it is, so -// there is nothing honest to show a percentage against. +// not, for progress reporting. func (s *Server) restoreSource( ctx context.Context, name string, args *disk_v1alpha.DiskBackupRestoreArgs, prog progressSink, ) (io.Reader, int64, func(), error) { - point := args.RestorePoint() - if point == "" { - in := args.Data() - if in == nil { - return nil, 0, nil, refuse("restore needs either a restore point or a snapshot to read") - } + if point := args.RestorePoint(); point != "" { + return s.downloadRestorePoint(ctx, name, point, prog) + } + + in := args.Data() + if in == nil { + return nil, 0, nil, refuse("restore needs either a restore point or a snapshot to read") + } + + // Land the whole snapshot on disk before touching the image. It is what + // makes an interrupted upload resumable, and it means a transfer that dies + // halfway cannot leave a half-decompressed image behind. + path, err := s.receiveSnapshot(ctx, args.TransferId(), args.Offset(), in, prog) + if err != nil { + return nil, 0, nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, 0, nil, fmt.Errorf("reopening uploaded snapshot: %w", err) + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, 0, nil, fmt.Errorf("stat uploaded snapshot: %w", err) + } + return f, info.Size(), func() { f.Close() }, nil +} + +// receiveSnapshot appends the client's bytes to this transfer's file and +// returns where it landed. +// +// The file is the checkpoint. Every byte that reaches it is a byte the client +// never has to send again, and fsync before returning is what makes that true +// across a server restart rather than only across a dropped connection. +func (s *Server) receiveSnapshot( + ctx context.Context, + transferID string, + offset int64, + in *stream.RecvStreamClient[[]byte], + prog progressSink, +) (string, error) { + if transferID == "" { + return "", refuse("uploading a snapshot needs a transfer id, so an interrupted upload can be resumed") + } + + s.sweepTransfers() + + f, err := s.openTransfer(transferID, offset) + if err != nil { + return "", err + } + defer f.Close() + + if offset > 0 { + prog.Message("Resuming upload at %d bytes", offset) + } else { prog.Message("Reading snapshot from client") - r := stream.ToReader(ctx, in) - // The stream belongs to the caller; closing our reader would tear - // their client down, so leave it to them. - return r, 0, func() {}, nil } + // The stream belongs to the caller; closing our reader would tear their + // client down, so leave it to them. + r := stream.ToReader(ctx, in) + + // Total is unknown — the client has not said how big its snapshot is — so + // progress shows bytes moved rather than a percentage. + written, err := io.Copy(f, s.trackReads(r, prog, offset, 0)) + + // Flush whatever did arrive before reporting either way. On the failure + // path this is the entire point: unsynced bytes would be re-sent for no + // reason, and worse, a length the client trusts might not survive a crash. + if serr := f.Sync(); serr != nil && err == nil { + err = fmt.Errorf("flushing uploaded snapshot: %w", serr) + } + if err != nil { + return "", fmt.Errorf("receiving snapshot after %d bytes: %w", written, err) + } + + return f.Name(), nil +} + +// downloadRestorePoint opens a restore point from miren.cloud. +// +// This one is not resumable here: the bytes come over the cloud's own HTTPS +// connection rather than the client's RPC session, so an interruption is +// between the server and the cloud, and re-fetching costs the operator nothing +// but time. +func (s *Server) downloadRestorePoint( + ctx context.Context, + name, point string, + prog progressSink, +) (io.Reader, int64, func(), error) { if s.updates == nil { return nil, 0, nil, errNoCloud("restoring from a restore point") } @@ -218,7 +333,7 @@ func (s *Server) writeImage(imagePath string, src io.Reader, compressedSize int6 return fmt.Errorf("preallocating image: %w", err) } - if err := snapshot.RestoreImage(out, s.trackReads(src, prog, compressedSize), meta); err != nil { + if err := snapshot.RestoreImage(out, s.trackReads(src, prog, 0, compressedSize), meta); err != nil { return err } diff --git a/servers/disk/resume_test.go b/servers/disk/resume_test.go new file mode 100644 index 000000000..3a21fa974 --- /dev/null +++ b/servers/disk/resume_test.go @@ -0,0 +1,215 @@ +package disk + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/cond" + "miren.dev/runtime/pkg/snapshot" +) + +// diskImage builds content that compresses to something big enough to be worth +// resuming, and that a wrong byte would show up in. +func diskImage(size int) []byte { + b := make([]byte, size) + for i := range b { + b[i] = byte(i*7 + i/251) + } + return b +} + +// A resumed backup must reuse the snapshot it already compressed. Compressing +// again would read the image as it is now, which for a live disk is not the +// image the client already holds half of, and the two halves would not be a +// snapshot of anything. +func TestBackupResumeReusesTheStagedSnapshot(t *testing.T) { + content := diskImage(256 * 1024) + s, _, _, imagePath := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + first, meta, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, first.Close()) + require.NotZero(t, meta.CompressedSize) + + // Change the image underneath, as a live disk would. + require.NoError(t, os.WriteFile(imagePath, diskImage(256*1024+7777), 0644)) + + second, meta2, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, second.Close()) + + assert.Equal(t, *meta, *meta2, "a resumed transfer must describe the same snapshot") + + // A different transfer id is a different backup, and does see the new image. + third, meta3, err := s.stageForClient("t2", prog, target) + require.NoError(t, err) + require.NoError(t, third.Close()) + assert.NotEqual(t, meta.Checksum, meta3.Checksum) +} + +// The point of the whole exercise: bytes delivered in two pieces, split at an +// arbitrary offset, must restore to exactly the original image. +func TestBackupSplitAcrossTwoAttemptsRestoresIdentically(t *testing.T) { + content := diskImage(512 * 1024) + s, _, _, _ := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + staged, meta, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + defer staged.Close() + + // First attempt gets a third of the way and dies. + cut := meta.CompressedSize / 3 + require.Positive(t, cut) + + firstHalf := make([]byte, cut) + _, err = io.ReadFull(staged, firstHalf) + require.NoError(t, err) + + // Second attempt resumes from exactly what arrived. + resumed, meta2, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + defer resumed.Close() + require.Equal(t, meta.CompressedSize, meta2.CompressedSize) + + _, err = resumed.Seek(cut, io.SeekStart) + require.NoError(t, err) + secondHalf, err := io.ReadAll(resumed) + require.NoError(t, err) + + stitched := append(append([]byte{}, firstHalf...), secondHalf...) + require.Len(t, stitched, int(meta.CompressedSize)) + + // And what the client ends up with is a snapshot that restores to the + // original image, checksum and all. + src := bytes.NewReader(stitched) + hdr, err := snapshot.ReadHeader(src) + require.NoError(t, err) + assert.Equal(t, meta.Checksum, hdr.Checksum) + + restored := filepath.Join(t.TempDir(), "restored.img") + require.NoError(t, s.writeImage(restored, src, 0, hdr, prog)) + + got, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, got, "a resumed backup must restore byte for byte") +} + +// The mirror case: an upload that arrives in two pieces leaves the server +// holding exactly the file the client sent. +func TestUploadSplitAcrossTwoAttemptsIsReassembled(t *testing.T) { + content := diskImage(256 * 1024) + s, _, _, _ := newTestServer(t, content) + prog := s.newProgress(context.Background(), nil) + + // A real snapshot, so what is reassembled can be checked by restoring it. + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + staged, meta, err := s.stageForClient("src", prog, target) + require.NoError(t, err) + snapBytes, err := io.ReadAll(staged) + require.NoError(t, err) + require.NoError(t, staged.Close()) + require.Len(t, snapBytes, int(meta.CompressedSize)) + + cut := len(snapBytes) / 2 + + // First attempt delivers half, then the connection dies. + f, err := s.openTransfer("up1", 0) + require.NoError(t, err) + _, err = f.Write(snapBytes[:cut]) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, f.Close()) + + // The client asks where to pick up, and is told what actually landed. + path, err := s.transferPath("up1") + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(cut), info.Size()) + + // Second attempt sends the rest from there. + f, err = s.openTransfer("up1", int64(cut)) + require.NoError(t, err) + _, err = f.Write(snapBytes[cut:]) + require.NoError(t, err) + require.NoError(t, f.Sync()) + require.NoError(t, f.Close()) + + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, snapBytes, got, "the reassembled upload must match what was sent") + + // And it really is a usable snapshot. + src := bytes.NewReader(got) + hdr, err := snapshot.ReadHeader(src) + require.NoError(t, err) + + restored := filepath.Join(t.TempDir(), "restored.img") + require.NoError(t, s.writeImage(restored, src, 0, hdr, prog)) + + back, err := os.ReadFile(restored) + require.NoError(t, err) + assert.Equal(t, content, back) +} + +// A completed backup releases its staging, so a transfer that finished does not +// keep a copy of the disk around. +func TestFinishedBackupReleasesItsStaging(t *testing.T) { + s, _, _, _ := newTestServer(t, diskImage(64*1024)) + prog := s.newProgress(context.Background(), nil) + + target, err := s.prepareBackup(context.Background(), "data") + require.NoError(t, err) + + staged, _, err := s.stageForClient("t1", prog, target) + require.NoError(t, err) + require.NoError(t, staged.Close()) + + path, err := s.transferPath("t1") + require.NoError(t, err) + metaPath, err := s.stagingMetaPath("t1") + require.NoError(t, err) + require.FileExists(t, path) + require.FileExists(t, metaPath) + + s.discardStaging("t1") + + _, err = os.Stat(path) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(metaPath) + assert.True(t, os.IsNotExist(err)) +} + +// An interrupted upload is exactly what the transfer file exists for and must +// survive. A refused one never will be picked up, because the client does not +// retry a refusal, so leaving it would keep a copy of the snapshot around for +// every attempt until the sweep. +func TestOnlyRefusedUploadsAreDiscarded(t *testing.T) { + interrupted := errors.New("connection reset by peer") + refused := cond.ValidationFailure("disk-backup", `disk "data" is in use`) + + assert.True(t, shouldDiscardUpload("t1", "", refused)) + assert.False(t, shouldDiscardUpload("t1", "", interrupted)) + + // A restore point came from the cloud, so there is no upload to discard. + assert.False(t, shouldDiscardUpload("t1", "point-1", refused)) + + // And nothing to discard without an id. + assert.False(t, shouldDiscardUpload("", "", refused)) +} diff --git a/servers/disk/transfer.go b/servers/disk/transfer.go new file mode 100644 index 000000000..45607fc4a --- /dev/null +++ b/servers/disk/transfer.go @@ -0,0 +1,253 @@ +package disk + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "miren.dev/runtime/api/disk/disk_v1alpha" +) + +const ( + // transfersDir holds the partial snapshots of transfers that are still in + // flight, beside the volumes they belong to so they land on the filesystem + // sized for disk data. + transfersDir = "transfers" + + // transferTTL is how long an abandoned transfer's bytes are kept. Long + // enough to outlive an outage and a night's sleep, short enough that a + // client that never comes back does not hoard a multi-gigabyte file + // forever. + transferTTL = 24 * time.Hour +) + +// transferPath is where a transfer's bytes accumulate. +// +// The id comes from the client, so it is checked rather than trusted: it names +// a file under a directory this server owns, and a caller must not be able to +// aim that at something else. +func (s *Server) transferPath(id string) (string, error) { + if id == "" { + return "", refuse("a transfer id is required to resume an interrupted transfer") + } + if len(id) > 128 { + return "", refuse("transfer id is too long") + } + for _, r := range id { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_': + default: + return "", refuse("transfer id may only contain letters, digits, dashes and underscores") + } + } + return filepath.Join(s.diskDataPath(), transfersDir, id+".part"), nil +} + +// TransferOffset reports how many bytes of a transfer the server already holds. +// +// An id it has never seen reports zero rather than failing, so a client's first +// attempt and its retries take the same path. +func (s *Server) TransferOffset(ctx context.Context, state *disk_v1alpha.DiskBackupTransferOffset) error { + path, err := s.transferPath(state.Args().TransferId()) + if err != nil { + return err + } + + info, err := os.Stat(path) + switch { + case os.IsNotExist(err): + state.Results().SetReceivedBytes(0) + return nil + case err != nil: + return fmt.Errorf("checking transfer progress: %w", err) + } + + state.Results().SetReceivedBytes(info.Size()) + return nil +} + +// openTransfer opens a transfer's file for appending, creating it if this is +// the first attempt, and checks the caller is resuming from where the server +// actually is. +// +// Refusing a mismatched offset rather than seeking to it is deliberate. The two +// ends disagreeing about how much arrived is exactly the situation where +// writing anyway produces a file that is the right length and the wrong bytes, +// and nothing would notice until a restore months later. +func (s *Server) openTransfer(id string, offset int64) (*os.File, error) { + path, err := s.transferPath(id) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, fmt.Errorf("creating transfer directory: %w", err) + } + + var have int64 + if info, serr := os.Stat(path); serr == nil { + have = info.Size() + } else if !os.IsNotExist(serr) { + return nil, fmt.Errorf("checking transfer progress: %w", serr) + } + + if offset != have { + return nil, refuse( + "this transfer is at %d bytes but the client offered to continue from %d — ask transferOffset where to resume, or use a new transfer id to start over", + have, offset, + ) + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return nil, fmt.Errorf("opening transfer file: %w", err) + } + return f, nil +} + +// stagedSnapshot describes a compressed snapshot waiting to be collected. +// +// It exists because the checksum and the image's size are known only while the +// snapshot is being compressed, and a resumed transfer skips that step. Without +// it, a client that reconnected for the last megabyte would be told the backup +// had no checksum. +type stagedSnapshot struct { + Disk string `json:"disk"` + ImageSize int64 `json:"image_size"` + CompressedSize int64 `json:"compressed_size"` + Checksum string `json:"checksum"` +} + +func (s *Server) stagingMetaPath(id string) (string, error) { + path, err := s.transferPath(id) + if err != nil { + return "", err + } + return strings.TrimSuffix(path, ".part") + ".meta", nil +} + +// loadStaging reports the finished snapshot waiting under this transfer id, or +// nil when there is none. +// +// The metadata file doubles as the completion marker: it is written only after +// compression finishes, so bytes left behind by a server that died partway +// through have no metadata and are correctly treated as nothing at all. +func (s *Server) loadStaging(id string) (*stagedSnapshot, error) { + metaPath, err := s.stagingMetaPath(id) + if err != nil { + return nil, err + } + + data, err := os.ReadFile(metaPath) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading staged snapshot metadata: %w", err) + } + + var meta stagedSnapshot + if err := json.Unmarshal(data, &meta); err != nil { + // Unreadable metadata means we cannot say what the staged bytes are, so + // treat the transfer as absent rather than guess. + s.log.Warn("discarding a staged snapshot with unreadable metadata", "transfer_id", id, "error", err) + s.discardStaging(id) + return nil, nil + } + + path, err := s.transferPath(id) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if os.IsNotExist(err) { + s.discardStaging(id) + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("checking staged snapshot: %w", err) + } + if info.Size() != meta.CompressedSize { + s.log.Warn("discarding a staged snapshot that does not match its metadata", + "transfer_id", id, "on_disk", info.Size(), "recorded", meta.CompressedSize) + s.discardStaging(id) + return nil, nil + } + + return &meta, nil +} + +func (s *Server) saveStaging(id string, meta *stagedSnapshot) error { + metaPath, err := s.stagingMetaPath(id) + if err != nil { + return err + } + data, err := json.Marshal(meta) + if err != nil { + return fmt.Errorf("encoding staged snapshot metadata: %w", err) + } + if err := os.WriteFile(metaPath, data, 0600); err != nil { + return fmt.Errorf("writing staged snapshot metadata: %w", err) + } + return nil +} + +// discardStaging removes a staged snapshot and its metadata. +func (s *Server) discardStaging(id string) { + if metaPath, err := s.stagingMetaPath(id); err == nil { + if rerr := os.Remove(metaPath); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove staged snapshot metadata", "transfer_id", id, "error", rerr) + } + } + s.discardTransfer(id) +} + +// discardTransfer removes a transfer's bytes once they are no longer needed. +func (s *Server) discardTransfer(id string) { + path, err := s.transferPath(id) + if err != nil { + return + } + if rerr := os.Remove(path); rerr != nil && !os.IsNotExist(rerr) { + s.log.Warn("failed to remove finished transfer", "transfer_id", id, "error", rerr) + } +} + +// sweepTransfers deletes transfers nobody came back for. +// +// This runs when a transfer starts rather than on a timer: it is the moment a +// transfer directory is known to be in use, and it keeps the cleanup on the +// path that creates the mess rather than in a goroutine that has to be owned +// and shut down. +func (s *Server) sweepTransfers() { + dir := filepath.Join(s.diskDataPath(), transfersDir) + entries, err := os.ReadDir(dir) + if err != nil { + if !os.IsNotExist(err) { + s.log.Warn("could not sweep abandoned transfers", "error", err) + } + return + } + + cutoff := time.Now().Add(-transferTTL) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".part") { + continue + } + info, ierr := e.Info() + if ierr != nil || info.ModTime().After(cutoff) { + continue + } + path := filepath.Join(dir, e.Name()) + if rerr := os.Remove(path); rerr != nil { + s.log.Warn("failed to remove abandoned transfer", "path", path, "error", rerr) + continue + } + s.log.Info("removed abandoned transfer", + "path", path, "size", info.Size(), "idle_for", time.Since(info.ModTime()).Truncate(time.Minute)) + } +} diff --git a/servers/disk/transfer_test.go b/servers/disk/transfer_test.go new file mode 100644 index 000000000..4b33a181b --- /dev/null +++ b/servers/disk/transfer_test.go @@ -0,0 +1,173 @@ +package disk + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A transfer id names a file under a directory the server owns, and it comes +// from the client, so it must not be usable to point somewhere else. +func TestTransferPathRejectsIdsThatEscapeTheDirectory(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + for _, id := range []string{ + "../../etc/passwd", + "a/b", + "..", + ".", + `a\b`, + "a b", + "a.part", + "", + } { + _, err := s.transferPath(id) + require.Error(t, err, "id %q should be rejected", id) + } +} + +func TestTransferPathAcceptsOrdinaryIds(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("0a1b2c3d-4e5f_6789") + require.NoError(t, err) + assert.Equal(t, + filepath.Join(s.diskDataPath(), transfersDir, "0a1b2c3d-4e5f_6789.part"), + path) +} + +func TestTransferOffsetIsZeroForAnUnknownId(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + // Zero rather than an error is what lets a client's first attempt and its + // retries be the same code path. + f, err := s.openTransfer("never-seen", 0) + require.NoError(t, err) + require.NoError(t, f.Close()) +} + +// The two ends disagreeing about how much arrived is exactly when writing +// anyway produces a file of the right length and the wrong bytes. +func TestOpenTransferRefusesAMismatchedOffset(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + f, err := s.openTransfer("t1", 0) + require.NoError(t, err) + _, err = f.Write([]byte("0123456789")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // Resuming from where the server actually is works. + f, err = s.openTransfer("t1", 10) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // Claiming to be further along does not. + _, err = s.openTransfer("t1", 25) + require.Error(t, err) + assert.Contains(t, err.Error(), "at 10 bytes") + + // Neither does claiming to be behind, which would duplicate bytes. + _, err = s.openTransfer("t1", 4) + require.Error(t, err) +} + +// Appending is what makes a resumed upload continue rather than overwrite. +func TestOpenTransferAppends(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + f, err := s.openTransfer("t1", 0) + require.NoError(t, err) + _, err = f.Write([]byte("abc")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = s.openTransfer("t1", 3) + require.NoError(t, err) + _, err = f.Write([]byte("def")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + path, err := s.transferPath("t1") + require.NoError(t, err) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "abcdef", string(got)) +} + +func TestStagingRoundTrips(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("compressed"), 0600)) + + want := &stagedSnapshot{Disk: "data", ImageSize: 4096, CompressedSize: 10, Checksum: "abc123"} + require.NoError(t, s.saveStaging("t1", want)) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, *want, *got) +} + +// Bytes left behind by a server that died mid-compression have no metadata, and +// must not be mistaken for a snapshot that is ready to send. +func TestStagingWithoutMetadataIsNotUsable(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("half a snapshot"), 0600)) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + assert.Nil(t, got) +} + +// A file that does not match what the metadata says it should be cannot be +// resumed from: the offsets the client is working with would be meaningless. +func TestStagingThatDoesNotMatchItsMetadataIsDiscarded(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + path, err := s.transferPath("t1") + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0700)) + require.NoError(t, os.WriteFile(path, []byte("short"), 0600)) + require.NoError(t, s.saveStaging("t1", &stagedSnapshot{CompressedSize: 999})) + + got, err := s.loadStaging("t1") + require.NoError(t, err) + assert.Nil(t, got) + + // And it is gone, rather than left to be reconsidered next time. + _, statErr := os.Stat(path) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSweepRemovesAbandonedTransfersAndKeepsFreshOnes(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + dir := filepath.Join(s.diskDataPath(), transfersDir) + require.NoError(t, os.MkdirAll(dir, 0700)) + + stale := filepath.Join(dir, "stale.part") + fresh := filepath.Join(dir, "fresh.part") + require.NoError(t, os.WriteFile(stale, []byte("old"), 0600)) + require.NoError(t, os.WriteFile(fresh, []byte("new"), 0600)) + + old := time.Now().Add(-transferTTL - time.Hour) + require.NoError(t, os.Chtimes(stale, old, old)) + + s.sweepTransfers() + + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err), "a transfer nobody came back for should be reclaimed") + assert.FileExists(t, fresh, "a transfer still within its window must survive") +} From f01ad5d1bc9342ceeb2e4c93241421b1dad3d113 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 21:38:09 -0700 Subject: [PATCH 12/19] Stop adopting a loop device that holds a deleted image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FindAllLoopBackings stripped the kernel's " (deleted)" marker off a loop device's backing file so callers could compare it against a live path. That is right for the orphan sweep, which matches on a prefix and wants to reclaim stale devices either way. It is wrong for the two callers that ask "is this image already attached, so I should mount that device instead of attaching a second one" — because a stripped ghost answers yes for an image that is no longer there. The result was silent and bad. ensureVolumeMount adopted the device, found the old filesystem on it, decided no formatting was needed, and mounted the contents the operator had replaced. A restore is exactly how you get there: renaming a new image over the old path unlinks the old inode, and the loop device pinning it goes on reporting that path. So the marker is now reported rather than stripped, and the two questions are answered separately: FindLoopByBacking ignores deleted backings, the orphan sweep still reclaims them. Confirmed against a live loop device, and worth writing down because it is not what the name suggests: the kernel tracks the backing inode's current name, so a rename simply changes what backing_file reports, with no marker. The marker appears only once the inode has no name left. The restore refusal added earlier reads better as a result. It asked FindLoopByBacking whether the image was in use, and a ghost used to make that a false positive; now that adoption cannot mount one, a ghost is genuinely not a reason to refuse. --- components/diskio/disk_ops.go | 20 +++- components/diskio/disk_ops_darwin.go | 2 +- components/diskio/disk_ops_linux.go | 55 +++++++-- components/diskio/disk_ops_linux_test.go | 56 +++++++++ components/diskio/disk_ops_mock_test.go | 19 ++- components/diskio/disk_volume_controller.go | 44 ++++++- .../diskio/disk_volume_controller_test.go | 112 ++++++++++++++++++ controllers/integration/mock_ops.go | 6 +- 8 files changed, 290 insertions(+), 24 deletions(-) create mode 100644 components/diskio/disk_ops_linux_test.go diff --git a/components/diskio/disk_ops.go b/components/diskio/disk_ops.go index 44ac98896..139c31b41 100644 --- a/components/diskio/disk_ops.go +++ b/components/diskio/disk_ops.go @@ -22,6 +22,24 @@ type ActiveMount struct { MountPath string } +// LoopBacking describes what a loop device is attached to. +// +// Path and Deleted are reported separately because the two questions callers +// ask are genuinely different. "Is anything holding a file under my volumes +// directory?" wants both, so a stale device can be reclaimed. "Is this image +// already attached, so I should mount that device instead of attaching a second +// one?" wants only live attachments — a loop holding an unlinked inode is not +// this image, it is the file that used to be at this path, and mounting it +// serves data the operator replaced. +type LoopBacking struct { + // Path is the backing file as the kernel reports it. + Path string + // Deleted reports that the backing inode has been unlinked. The file lives + // on because the loop device holds a reference, but nothing can reach it by + // name any more: something renamed or removed the path out from under it. + Deleted bool +} + // DiskMountOps abstracts OS operations for disk mount management. // This interface enables testing without requiring actual loop device or mount operations. type DiskMountOps interface { @@ -36,7 +54,7 @@ type DiskMountOps interface { // FindAllLoopBackings returns a map of loop device path to the backing // file currently attached to it, for every loop device in the kernel. // Used by boot-time orphan reconciliation to find stale attachments. - FindAllLoopBackings() (map[string]string, error) + FindAllLoopBackings() (map[string]LoopBacking, error) LbdAttach(ctx context.Context, imagePath, logDir string) (devicePath string, err error) LbdDetach(ctx context.Context, devicePath string) error LbdAvailable() bool diff --git a/components/diskio/disk_ops_darwin.go b/components/diskio/disk_ops_darwin.go index 4866c192f..d5a7341bf 100644 --- a/components/diskio/disk_ops_darwin.go +++ b/components/diskio/disk_ops_darwin.go @@ -68,7 +68,7 @@ func (s *stubDiskMountOps) FindLoopByBacking(_ string) (string, error) { return "", nil } -func (s *stubDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (s *stubDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { return nil, nil } diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 2c300ab36..08c019648 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -174,7 +174,8 @@ func (r *realDiskMountOps) LoopDetach(devicePath string) error { } // FindLoopByBacking walks /sys/block/loop*/loop/backing_file and returns the -// loop device path currently backing imagePath, or "" if none is attached. +// loop device path currently backing the file at imagePath, or "" if none is +// attached. // // The kernel never deletes a stale loop device just because miren restarted, // so finding a match here means a previous miren (or an uncleanly shut down @@ -182,6 +183,15 @@ func (r *realDiskMountOps) LoopDetach(devicePath string) error { // produce two loop devices with independent, incoherent page caches and // corrupt the filesystem. Callers should reuse the returned device, fail // loudly, or detach it explicitly. +// +// A loop whose backing inode has been unlinked is deliberately not a match, +// even though the kernel still reports this exact path for it. That device is +// holding the file that used to be here, not the one here now — after a restore +// wrote a new image and renamed it into place, say. Treating it as a match is +// how stale data gets mounted over good data: the caller adopts the device, +// finds the old filesystem on it, decides no formatting is needed, and serves +// the contents the operator just replaced. Use FindAllLoopBackings, which +// reports the distinction, to find such devices and reclaim them. func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { absPath, err := filepath.Abs(imagePath) if err != nil { @@ -193,7 +203,7 @@ func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { return "", err } for dev, backing := range all { - if backing == absPath { + if backing.Path == absPath && !backing.Deleted { return dev, nil } } @@ -203,13 +213,13 @@ func (r *realDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { // FindAllLoopBackings walks /sys/block/loop*/loop/backing_file and returns // a map of loop device path → backing file path for every loop device in // the kernel. Devices that race with a concurrent detach are skipped. -func (r *realDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (r *realDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { entries, err := filepath.Glob("/sys/block/loop*/loop/backing_file") if err != nil { return nil, fmt.Errorf("failed to glob loop backing files: %w", err) } - result := make(map[string]string, len(entries)) + result := make(map[string]LoopBacking, len(entries)) for _, entry := range entries { data, err := os.ReadFile(entry) if err != nil { @@ -220,19 +230,44 @@ func (r *realDiskMountOps) FindAllLoopBackings() (map[string]string, error) { return nil, fmt.Errorf("failed to read %s: %w", entry, err) } - backing := strings.TrimSpace(string(data)) - // The kernel appends " (deleted)" when the backing inode is - // unlinked; strip it so callers can compare against a live path. - backing = strings.TrimSuffix(backing, " (deleted)") - // entry is /sys/block/loopN/loop/backing_file — extract loopN. loopName := filepath.Base(filepath.Dir(filepath.Dir(entry))) - result["/dev/"+loopName] = backing + result["/dev/"+loopName] = parseLoopBacking(string(data)) } return result, nil } +// deletedBackingSuffix is what the kernel appends to a loop device's backing +// file path once that file has been unlinked. +const deletedBackingSuffix = " (deleted)" + +// parseLoopBacking reads one backing_file line. +// +// The deleted marker is reported rather than stripped: to a caller comparing +// against a live path, a deleted backing that still reads as that path is a +// trap, because the file at the path now is a different file entirely. +// +// What the kernel actually does, confirmed against a live loop device: it +// tracks the backing inode's current name, so renaming the file simply changes +// what this reports, with no marker. The marker appears only once the inode has +// no name left. That is exactly what a restore does to the image it replaces — +// renaming a new image over the old path unlinks the old inode — which is how a +// loop device ends up reporting a path whose file it is no longer holding. +// +// The marker is ambiguous for a file whose own name ends in " (deleted)", and +// the kernel offers no way to tell the two apart. Reading it as deleted is the +// safe way to be wrong: the cost is attaching a second loop device rather than +// adopting one, where the other way round serves stale data. +func parseLoopBacking(raw string) LoopBacking { + path := strings.TrimSpace(raw) + deleted := strings.HasSuffix(path, deletedBackingSuffix) + return LoopBacking{ + Path: strings.TrimSuffix(path, deletedBackingSuffix), + Deleted: deleted, + } +} + func (r *realDiskMountOps) LbdAttach(ctx context.Context, imagePath, logDir string) (string, error) { cmd := exec.CommandContext(ctx, "lbdctl", "add", "--json", imagePath, "--log-dir", logDir) output, err := cmd.CombinedOutput() diff --git a/components/diskio/disk_ops_linux_test.go b/components/diskio/disk_ops_linux_test.go new file mode 100644 index 000000000..f58d0c4c3 --- /dev/null +++ b/components/diskio/disk_ops_linux_test.go @@ -0,0 +1,56 @@ +package diskio + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseLoopBacking(t *testing.T) { + cases := []struct { + name string + raw string + want LoopBacking + }{ + { + name: "a live backing file", + raw: "/var/lib/miren/disk-data/volumes/vol-1/disk.img\n", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/disk.img"}, + }, + { + // What the kernel reports once the file has lost its last name. + // The path is where it was when that happened. + name: "a backing file that has been unlinked", + raw: "/var/lib/miren/disk-data/volumes/vol-1/disk.img (deleted)\n", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/disk.img", Deleted: true}, + }, + { + // A renamed file is followed to its new name with no marker: the + // kernel tracks the inode, and the inode still has a name. Only + // losing its last name marks it. + name: "a backing file that was renamed rather than removed", + raw: "/var/lib/miren/disk-data/volumes/vol-1/renamed.img", + want: LoopBacking{Path: "/var/lib/miren/disk-data/volumes/vol-1/renamed.img"}, + }, + { + name: "an unlinked file whose own name ended in the marker", + raw: "/tmp/weird (deleted) (deleted)", + want: LoopBacking{Path: "/tmp/weird (deleted)", Deleted: true}, + }, + { + // Genuinely ambiguous — a live file named "weird (deleted)" reads + // identically to a deleted file named "weird", and the kernel gives + // no way to tell. Read as deleted, which is the safe way to be + // wrong: it costs an extra loop device rather than stale data. + name: "the ambiguous case is read as deleted", + raw: "/tmp/weird (deleted)", + want: LoopBacking{Path: "/tmp/weird", Deleted: true}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, parseLoopBacking(tc.raw)) + }) + } +} diff --git a/components/diskio/disk_ops_mock_test.go b/components/diskio/disk_ops_mock_test.go index 98bd56a3f..3cef842b1 100644 --- a/components/diskio/disk_ops_mock_test.go +++ b/components/diskio/disk_ops_mock_test.go @@ -102,6 +102,11 @@ type mockDiskMountOps struct { mountDevices map[string]string // mount path → device, for FindMounts formattedDevs map[string]string formatCalls []diskMockFormat + // deletedBacking marks image paths whose loop device holds an unlinked + // inode — the kernel still reports the path, but the file there now is a + // different file. Set it to model a ghost loop. + deletedBacking map[string]bool + // loopBacking maps imagePath → existing loop device, for FindLoopByBacking. // Tests populate this to simulate a pre-existing attachment. loopBacking map[string]string @@ -209,19 +214,21 @@ func (m *mockDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { if m.loopBacking == nil { return "", nil } + // A device holding an unlinked inode is not this image, so it is not a + // match — the same rule the real implementation follows. + if m.deletedBacking[imagePath] { + return "", nil + } return m.loopBacking[imagePath], nil } -func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]string, error) { +func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]LoopBacking, error) { if m.findLoopErr != nil { return nil, m.findLoopErr } - if m.loopBacking == nil { - return map[string]string{}, nil - } - result := make(map[string]string, len(m.loopBacking)) + result := make(map[string]LoopBacking, len(m.loopBacking)) for imagePath, dev := range m.loopBacking { - result[dev] = imagePath + result[dev] = LoopBacking{Path: imagePath, Deleted: m.deletedBacking[imagePath]} } return result, nil } diff --git a/components/diskio/disk_volume_controller.go b/components/diskio/disk_volume_controller.go index f5cf496e7..37fa372ab 100644 --- a/components/diskio/disk_volume_controller.go +++ b/components/diskio/disk_volume_controller.go @@ -648,6 +648,12 @@ func (c *DiskVolumeController) ensureVolumeMount(ctx context.Context, entityId s ) devicePath = existing } else { + // A loop device may still be holding the file that used to be at this + // path — after a restore replaced the image, say. It is not adopted, + // because it would serve the old contents, but it does leak a device + // until the orphan sweep reclaims it, so say so. + c.warnAboutStaleLoop(entityId, imagePath) + // No existing loop. Any stale volState.DevicePath is meaningless // (the kernel has no loop backing this image), so we don't touch // it — the loop index it names may have been reallocated to some @@ -810,6 +816,32 @@ func (c *DiskVolumeController) Shutdown() { } } +// warnAboutStaleLoop reports a loop device still holding a file that used to be +// at imagePath. +// +// Best effort, and never fatal: this is only here to make a leaked device +// visible. The mount that follows attaches the live image on its own device and +// is correct either way, and the orphan sweep reclaims the leftover. +func (c *DiskVolumeController) warnAboutStaleLoop(entityId, imagePath string) { + abs, err := filepath.Abs(imagePath) + if err != nil { + return + } + backings, err := c.mntOps.FindAllLoopBackings() + if err != nil { + return + } + for dev, backing := range backings { + if backing.Deleted && backing.Path == abs { + c.log.Warn("a loop device still holds the file that used to be at this path", + "entity_id", entityId, + "image_path", imagePath, + "device", dev, + ) + } + } +} + // reconcileOrphanKernelState runs once at boot and tears down any kernel // loop devices or mounts that are rooted in miren's volumes directory but // that no longer correspond to a known volume in local state. @@ -874,14 +906,20 @@ func (c *DiskVolumeController) reconcileOrphanKernelState() { } // Only touch loops backing files inside miren's volumes dir. // Anything else is not ours to manage. - if !strings.HasPrefix(backing, volumesDir+string(filepath.Separator)) && - !strings.HasPrefix(backing, volumesDir+"/") { + // + // A deleted backing counts: the path it reports is where the file used + // to be, which is still enough to say the device is ours, and a device + // holding an unlinked image is exactly the kind of leftover this sweep + // exists to reclaim. + if !strings.HasPrefix(backing.Path, volumesDir+string(filepath.Separator)) && + !strings.HasPrefix(backing.Path, volumesDir+"/") { continue } c.log.Warn("orphan sweep: detaching stale loop device", "device", dev, - "backing_file", backing, + "backing_file", backing.Path, + "backing_deleted", backing.Deleted, ) if err := c.mntOps.LoopDetach(dev); err != nil { c.log.Warn("orphan sweep: LoopDetach failed", diff --git a/components/diskio/disk_volume_controller_test.go b/components/diskio/disk_volume_controller_test.go index 3caeb219e..3e266bc9e 100644 --- a/components/diskio/disk_volume_controller_test.go +++ b/components/diskio/disk_volume_controller_test.go @@ -1080,3 +1080,115 @@ func TestDiskVolumeControllerAcceleratorNoMountAtCreation(t *testing.T) { require.NotNil(t, volState) assert.False(t, volState.Mounted) } + +// TestDiskVolumeControllerDoesNotAdoptADeletedBacking is the counterpart to the +// adoption test above, for the case where the loop device is holding a file +// that is no longer at that path. +// +// The kernel reports such a device's backing_file as the original path with +// " (deleted)" appended. Matching on the path alone makes it look like the +// image is already attached, so the controller adopts it, finds the old +// filesystem, decides no formatting is needed, and mounts the contents that +// were replaced — after a restore wrote a new image and renamed it into place, +// exactly the data the operator had just discarded. +// +// The live image must get its own loop device instead. +func TestDiskVolumeControllerDoesNotAdoptADeletedBacking(t *testing.T) { + ctx := t.Context() + log := testutils.TestLogger(t) + + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + nodeId := "test-node-1" + state := NewState() + volOps := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + + volPath := filepath.Join(dataPath, "volumes", "vol-ghost") + mountPath := filepath.Join(dataPath, "vol-ghost") + imagePath := filepath.Join(volPath, "disk.img") + + state.SetVolume("disk_volume/vol-ghost", &VolumeState{ + EntityId: "disk_volume/vol-ghost", + VolumeId: "vol-ghost", + DiskPath: volPath, + SizeBytes: 10 * 1024 * 1024 * 1024, + Filesystem: "ext4", + Mode: storage_v1alpha.VM_UNIVERSAL, + Mounted: false, + MountPath: mountPath, + }) + volOps.existingPaths[volPath] = true + + // A loop device still pinning the image that used to be at this path. + const ghostLoopDev = "/dev/loop9" + mntOps.loopBacking = map[string]string{imagePath: ghostLoopDev} + mntOps.deletedBacking = map[string]bool{imagePath: true} + mntOps.formattedDevs[ghostLoopDev] = "ext4" + + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId(nodeId), state, volOps, mntOps) + vc.SetEAC(es.EAC) + + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-ghost", + NodeId: compute.NewNodeId(nodeId).Id(), + SizeGb: 10, + Filesystem: "ext4", + VolumeMode: storage_v1alpha.VM_UNIVERSAL, + DesiredState: storage_v1alpha.DV_PRESENT, + ActualState: storage_v1alpha.DV_READY, + } + createDiskVolumeEntity(ctx, t, es, vol) + + require.NoError(t, vc.ReconcileWithEntities(ctx)) + + // The live image gets a loop of its own rather than the ghost. + require.Len(t, mntOps.attachedLoops, 1, + "the live image must be attached rather than the device holding the deleted one") + + require.Len(t, mntOps.mounts, 1) + assert.NotEqual(t, ghostLoopDev, mntOps.mounts[0].device, + "mounting the device that holds the deleted inode serves the data the operator replaced") + + volState := state.GetVolume("disk_volume/vol-ghost") + require.NotNil(t, volState) + assert.True(t, volState.Mounted) + assert.NotEqual(t, ghostLoopDev, volState.DevicePath) +} + +// TestDiskVolumeControllerOrphanSweepReclaimsDeletedBackings is the other half +// of the deleted-backing distinction. +// +// Adoption must ignore a loop holding an unlinked inode, but the sweep must +// still reclaim it: a device pinning a file that no longer has a name is +// precisely the leftover this sweep exists for, and it is what keeps the +// non-adoption above from leaking a device forever. +func TestDiskVolumeControllerOrphanSweepReclaimsDeletedBackings(t *testing.T) { + ctx := t.Context() + log := testutils.TestLogger(t) + + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + nodeId := "test-node-1" + state := NewState() + volOps := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + + ghostImage := filepath.Join(dataPath, "volumes", "vol-ghost", "disk.img") + const ghostLoopDev = "/dev/loop11" + + mntOps.loopBacking = map[string]string{ghostImage: ghostLoopDev} + mntOps.deletedBacking = map[string]bool{ghostImage: true} + + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId(nodeId), state, volOps, mntOps) + vc.SetEAC(es.EAC) + + require.NoError(t, vc.ReconcileWithEntities(ctx)) + + assert.Contains(t, mntOps.detachedLoops, ghostLoopDev, + "a loop holding an unlinked image under the volumes dir must still be reclaimed") +} diff --git a/controllers/integration/mock_ops.go b/controllers/integration/mock_ops.go index b5a9ead2a..081fc84ab 100644 --- a/controllers/integration/mock_ops.go +++ b/controllers/integration/mock_ops.go @@ -103,10 +103,10 @@ func (m *mockDiskMountOps) FindLoopByBacking(imagePath string) (string, error) { return "", nil } -func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]string, error) { - result := make(map[string]string, len(m.loopDevices)) +func (m *mockDiskMountOps) FindAllLoopBackings() (map[string]diskio.LoopBacking, error) { + result := make(map[string]diskio.LoopBacking, len(m.loopDevices)) for img, dev := range m.loopDevices { - result[dev] = img + result[dev] = diskio.LoopBacking{Path: img} } return result, nil } From 450bb344a68e5b551a686bd3d70f03b77433d2c9 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 21:57:01 -0700 Subject: [PATCH 13/19] Make was_attached mean something MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metadata key recording that a restore point was taken from a live disk was fed by BackupTarget.IsAttached, which was set from disk.status == ATTACHED — a status nothing in the runtime ever writes. So it was always false, and every restore point claimed to be a clean point-in-time copy whether or not it was. The fact is worth recording. It is the difference between a restore point you can rely on and one you are choosing to gamble on, and the operator who took it is not the one who will read it at three in the morning. So rather than drop the key, it now comes from the kernel: the same FindLoopByBacking check that produces the warning, in both the RPC path and the break-glass one, which now warns at all. Not knowing counts as in use. The flag only ever warns a reader off relying on the snapshot, so "possibly" and "yes" call for the same caution. IsAttached is gone from BackupTarget. A field that cannot be true is worse than no field: it reads as a supported signal. --- cli/commands/debug_disk_backup.go | 34 +++++++++++++++++------- components/diskio/image_snapshot.go | 9 +++++++ components/diskio/image_snapshot_test.go | 23 ++++++++++++++++ pkg/snapshot/disk.go | 3 --- pkg/snapshot/disk_test.go | 11 -------- servers/disk/backup.go | 9 ++++++- 6 files changed, 64 insertions(+), 25 deletions(-) diff --git a/cli/commands/debug_disk_backup.go b/cli/commands/debug_disk_backup.go index 6b67bd8ee..2c766eb27 100644 --- a/cli/commands/debug_disk_backup.go +++ b/cli/commands/debug_disk_backup.go @@ -53,15 +53,25 @@ func DebugDiskBackup(ctx *Context, opts struct { return fmt.Errorf("disk image not found at %s: %w", target.ImagePath, err) } - if target.IsAttached { - // Nothing here freezes the filesystem or takes a copy-on-write clone, so - // this is a sequential read of a file the loop device is still writing. - // The head and tail of the image come from different moments, which is - // weaker than the power-loss state fsck and Postgres recovery are built - // for. Say so plainly: the operator is the one deciding this is safe. - ctx.Warn("Disk is attached and may be written during the backup.") - ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") - ctx.Warn("Detach the disk first for a backup you can rely on.") + // Nothing here freezes the filesystem or takes a copy-on-write clone, so a + // backup of a live disk is a sequential read of a file the loop device is + // still writing. The head and tail of the image come from different moments, + // which is weaker than the power-loss state fsck and Postgres recovery are + // built for. Say so plainly: the operator is the one deciding this is safe. + // + // The kernel is the only thing that actually knows, so ask it. Not knowing + // counts as in use, since this flag only ever warns a reader off relying on + // the snapshot. + inUse := true + if dev, lerr := diskio.NewRealDiskMountOps(ctx.Log).FindLoopByBacking(target.ImagePath); lerr != nil { + ctx.Warn("Could not tell whether the disk is in use: %v", lerr) + } else { + inUse = dev != "" + if inUse { + ctx.Warn("Disk is in use (%s) and may be written during the backup.", dev) + ctx.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") + ctx.Warn("Detach the disk first for a backup you can rely on.") + } } outputPath := opts.Output @@ -128,6 +138,7 @@ func DebugDiskBackup(ctx *Context, opts struct { if opts.Cloud { updateID, err := uploadSnapshotToCloud(ctx, opts.DataPath, target, outputPath, cloudSnapshotDetails{ Pin: opts.Pin, + InUse: inUse, ImageSize: imgInfo.Size(), ImageChecksum: checksum, CompressedSize: outInfo.Size(), @@ -152,6 +163,9 @@ type cloudSnapshotDetails struct { ImageSize int64 ImageChecksum string CompressedSize int64 + // InUse records that the image was attached when it was read, so the + // snapshot is a smear across the read rather than a point-in-time copy. + InUse bool } // uploadSnapshotToCloud sends a finished snapshot file to miren.cloud as a @@ -218,7 +232,7 @@ func uploadSnapshotToCloud(ctx *Context, dataPath string, target *snapshot.Backu "compressed_size": details.CompressedSize, // Records that this was taken from a live disk, so the image is a // smear across the read rather than a point-in-time copy. - "was_attached": target.IsAttached, + "was_attached": details.InUse, }, }, file, info.Size()) } diff --git a/components/diskio/image_snapshot.go b/components/diskio/image_snapshot.go index 732f4a463..da3d9d1cb 100644 --- a/components/diskio/image_snapshot.go +++ b/components/diskio/image_snapshot.go @@ -53,6 +53,14 @@ type SnapshotRequest struct { Filesystem string // SnapshotName optionally pins this as a named restore point. SnapshotName string + // InUse records that the image was attached when it was read, so the + // snapshot is a smear across the read rather than a point-in-time copy. + // Worth knowing later: it is the difference between a restore point you can + // rely on and one you are choosing to gamble on. + // + // The caller decides, because "attached" is a question about kernel state + // that the snapshotter has no handle on. + InUse bool // LeaseNonce is required when the volume has an active lease. LeaseNonce string // StagingDir is where the compressed snapshot is written before upload. @@ -119,6 +127,7 @@ func (s *ImageSnapshotter) Snapshot(ctx context.Context, req SnapshotRequest) (* "image_size": info.Size(), "image_sha256": checksum, "compressed_size": stagedInfo.Size(), + "was_attached": req.InUse, }, LeaseNonce: req.LeaseNonce, }, staged, stagedInfo.Size()) diff --git a/components/diskio/image_snapshot_test.go b/components/diskio/image_snapshot_test.go index 47c019424..6060e5774 100644 --- a/components/diskio/image_snapshot_test.go +++ b/components/diskio/image_snapshot_test.go @@ -62,6 +62,29 @@ func TestImageSnapshotUploadsCompressedImage(t *testing.T) { assert.Equal(t, "data", meta.Name) assert.Equal(t, int64(len(content)), meta.SizeBytes) assert.Equal(t, meta.Checksum, up.Request.Metadata["image_sha256"]) + + // The fixture did not say the image was in use, so the restore point does + // not claim it was. + assert.Equal(t, false, up.Request.Metadata["was_attached"]) +} + +// A snapshot taken from a live image is a smear across the read rather than a +// point-in-time copy, and the restore point has to say so — it is the +// difference between one you can rely on and one you are gambling on. +func TestImageSnapshotRecordsThatTheImageWasInUse(t *testing.T) { + snapshotter, fake, imagePath := newSnapshotFixture(t, []byte("written while we read")) + + _, err := snapshotter.Snapshot(context.Background(), SnapshotRequest{ + VolumeID: "vol-1", + ImagePath: imagePath, + Name: "data", + Filesystem: "ext4", + InUse: true, + }) + require.NoError(t, err) + + require.Len(t, fake.uploads, 1) + assert.Equal(t, true, fake.uploads[0].Request.Metadata["was_attached"]) } // Every snapshot is a deliberate act, so two in a row both upload. There is no diff --git a/pkg/snapshot/disk.go b/pkg/snapshot/disk.go index 7295a5708..a254359fd 100644 --- a/pkg/snapshot/disk.go +++ b/pkg/snapshot/disk.go @@ -8,7 +8,6 @@ import ( const ( StatusDeleting = "DELETING" - StatusAttached = "ATTACHED" LeaseStatusBound = "BOUND" ) @@ -56,7 +55,6 @@ type BackupTarget struct { Name string Filesystem string ImagePath string - IsAttached bool // VolumeID is the node-local volume identifier. VolumeID string // CloudVolumeID identifies the volume in miren.cloud, for backups sent @@ -96,7 +94,6 @@ func PrepareBackup(ctx context.Context, resolver DiskResolver, name string, data Name: name, Filesystem: disk.Filesystem, ImagePath: resolveImagePath(vol, dataPath), - IsAttached: disk.Status == StatusAttached, VolumeID: vol.VolumeID, CloudVolumeID: vol.CloudVolumeID, diff --git a/pkg/snapshot/disk_test.go b/pkg/snapshot/disk_test.go index a56d306dd..ebb8a6f19 100644 --- a/pkg/snapshot/disk_test.go +++ b/pkg/snapshot/disk_test.go @@ -54,17 +54,6 @@ func TestPrepareBackup(t *testing.T) { assert.Equal(t, "mydb", target.Name) assert.Equal(t, "ext4", target.Filesystem) assert.Equal(t, "/data/disk.img", target.ImagePath) - assert.False(t, target.IsAttached) - }) - - t.Run("attached disk sets flag", func(t *testing.T) { - r := &mockResolver{ - disk: &DiskState{ID: "d1", Name: "mydb", Status: StatusAttached, Filesystem: "ext4"}, - volume: &VolumeState{VolumeID: "v1", ImagePath: "/data/disk.img"}, - } - target, err := PrepareBackup(ctx, r, "mydb", "/var/lib/miren") - require.NoError(t, err) - assert.True(t, target.IsAttached) }) t.Run("deleting disk rejected", func(t *testing.T) { diff --git a/servers/disk/backup.go b/servers/disk/backup.go index 3d7997a11..8f69ec899 100644 --- a/servers/disk/backup.go +++ b/servers/disk/backup.go @@ -32,10 +32,15 @@ func (s *Server) Backup(ctx context.Context, state *disk_v1alpha.DiskBackupBacku // Unlike restore, this check is best effort. Backup only reads, so being // unable to tell whether the disk is in use is a reason to say less, not a // reason to refuse. + inUse := false if dev, err := s.liveImageDevice(target.ImagePath); err != nil { s.log.Warn("could not tell whether disk image is in use", "disk", target.Name, "error", err) prog.Warn("Could not tell whether %q is in use, so this backup may not be a point-in-time copy.", target.Name) + // Recorded as in use, because the honest answer is "possibly" and this + // flag only ever warns a future reader off relying on the snapshot. + inUse = true } else if dev != "" { + inUse = true s.log.Info("backing up a disk that is in use", "disk", target.Name, "device", dev) prog.Warn("Disk %q is in use (%s) and may be written during the backup.", target.Name, dev) prog.Warn("The image is read while in use, so it is not a point-in-time copy and may not mount cleanly.") @@ -43,7 +48,7 @@ func (s *Server) Backup(ctx context.Context, state *disk_v1alpha.DiskBackupBacku } if args.ToCloud() { - return s.backupToCloud(ctx, state, prog, target) + return s.backupToCloud(ctx, state, prog, target, inUse) } return s.backupToClient(ctx, state, prog, target) } @@ -53,6 +58,7 @@ func (s *Server) backupToCloud( state *disk_v1alpha.DiskBackupBackup, prog progressSink, target *snapshot.BackupTarget, + inUse bool, ) error { if s.updates == nil { return errNoCloud("backing up to miren.cloud") @@ -74,6 +80,7 @@ func (s *Server) backupToCloud( Filesystem: target.Filesystem, SnapshotName: state.Args().Pin(), StagingDir: s.stagingDir(target.ImagePath), + InUse: inUse, }) if err != nil { return err From 07a9e47086c7851001e34168e1354eaf8cb3afb6 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 22:39:28 -0700 Subject: [PATCH 14/19] Address review feedback Both reviewers landed on the same gap: nothing serialized work on a transfer id. Two calls carrying the same one could both pass openTransfer's offset check and interleave their appends, or one could truncate a staged snapshot while another was streaming it. A well-behaved client never does this, since it picks a fresh id per invocation and retries in sequence, but the server should not have been relying on that. There is now a reference-counted per-id lock, held across the whole staging-and-stream or receive-and-install rather than just the file opens, because the window that matters spans the copy. The rest are smaller, and mostly cases of a check that could not tell two situations apart. FindDisk returning any error was read as "no such disk", so an entity store that merely failed to answer looked like a free name. Restore would create a second disk, and undelete would let a recovery proceed into a name that was actually taken. Absence is now its own error type, and only absence means create. Disk size rounded down, so restoring a 1.5 GiB image produced a disk claiming 1 GiB. Backup opened its output with os.Create, which truncates. Since a failed backup then deletes that file, reusing an -o path destroyed a good snapshot to produce nothing, which is the one outcome a backup command must not have. It refuses an existing path now. Undelete's cleanup ran on the request context, so a client disconnect could cancel the very writes that unwind a half-finished recovery. It uses a detached context, matching what the deferred rollback already did. The sweep took a transfer's bytes and left its metadata, restore did not sync the directory entry after renaming an image into place, --pin was validated after building the RPC client so a flag error needed a reachable cluster, and the break-glass listing pointed at the command that needs the RPC listener it exists to work without. The blackbox readiness poll asked whether the listing mentioned the disk and mentioned "provisioned", which any other provisioned disk in the cluster answered. It reads the disk's own entry now. Worth noting the suggested fix was to match both on one line; the two fields sit on different lines of an entry, so that would have matched nothing and hung the poll until timeout. --- blackbox/disk_backup_test.go | 30 +++++++- cli/commands/debug_disk_list_deleted.go | 7 +- cli/commands/disk_backup.go | 20 ++++-- cli/commands/disk_list_deleted.go | 4 +- pkg/diskresolve/resolver.go | 9 ++- pkg/diskresolve/resolver_test.go | 32 +++++++++ pkg/snapshot/disk.go | 23 +++++- pkg/snapshot/disk_test.go | 39 +++++++++++ servers/disk/backup.go | 6 ++ servers/disk/restore.go | 32 +++++++++ servers/disk/server.go | 19 +++-- servers/disk/transfer.go | 61 +++++++++++++++- servers/disk/transfer_test.go | 93 +++++++++++++++++++++++++ servers/disk/undelete.go | 28 +++++--- 14 files changed, 374 insertions(+), 29 deletions(-) diff --git a/blackbox/disk_backup_test.go b/blackbox/disk_backup_test.go index 9dcf56f06..d2cd5f045 100644 --- a/blackbox/disk_backup_test.go +++ b/blackbox/disk_backup_test.go @@ -4,6 +4,7 @@ package blackbox import ( "fmt" + "strings" "testing" "time" @@ -83,10 +84,37 @@ func waitDiskProvisioned(t *testing.T, m *harness.Miren, name string) { if !r.Success() { return false, "debug disk list failed" } - if r.OutputContains(name) && r.OutputContains("provisioned") { + if diskIsProvisioned(r.Stdout+r.Stderr, name) { return true, "" } return false, "disk not yet provisioned" }, ) } + +// diskIsProvisioned reports whether the named disk's own entry says provisioned. +// +// Asking whether the whole listing mentions the name and mentions "provisioned" +// is not the same question: any other provisioned disk in the cluster answers +// the second half, so the poll would return while this disk was still coming +// up. The name and the status sit on different lines of one entry, so the match +// has to run over the entry rather than a line. +func diskIsProvisioned(output, name string) bool { + lines := strings.Split(output, "\n") + + for i, line := range lines { + if !strings.Contains(line, "Name:") || !strings.Contains(line, name) { + continue + } + // Scan this entry's fields, stopping where the next entry begins. + for _, field := range lines[i+1:] { + if strings.HasPrefix(strings.TrimSpace(field), "ID:") { + break + } + if strings.Contains(field, "Status:") { + return strings.Contains(field, "provisioned") + } + } + } + return false +} diff --git a/cli/commands/debug_disk_list_deleted.go b/cli/commands/debug_disk_list_deleted.go index 765df229b..435e5ad79 100644 --- a/cli/commands/debug_disk_list_deleted.go +++ b/cli/commands/debug_disk_list_deleted.go @@ -41,7 +41,9 @@ func DebugDiskListDeleted(ctx *Context, opts struct { RetentionDays int `json:"retention_days"` } - var items []deletedDiskJSON + // Non-nil, so an empty result marshals as [] rather than null and + // callers can iterate it without a special case. + items := make([]deletedDiskJSON, 0, len(entries)) for _, e := range entries { meta := e.Metadata expiresAt := meta.DeletedAt.Add(time.Duration(retentionDays) * 24 * time.Hour) @@ -85,7 +87,8 @@ func DebugDiskListDeleted(ctx *Context, opts struct { ctx.Info("") } - ctx.Info("To restore: miren disk undelete --name ") + ctx.Info("To recover: miren disk undelete --name ") + ctx.Info(" or, if the server's RPC listener is down: miren debug disk undelete --name ") return nil } diff --git a/cli/commands/disk_backup.go b/cli/commands/disk_backup.go index 881443267..114b9da77 100644 --- a/cli/commands/disk_backup.go +++ b/cli/commands/disk_backup.go @@ -1,6 +1,7 @@ package commands import ( + "errors" "fmt" "io" "os" @@ -23,16 +24,18 @@ func DiskBackup(ctx *Context, opts struct { Cloud bool `long:"cloud" description:"Upload the snapshot to miren.cloud as a restore point instead of writing a local file"` Pin string `long:"pin" description:"Name the uploaded restore point, pinning it against cleanup"` }) (retErr error) { + // Checked before the client is built, so a flag mistake reports the flag + // rather than whatever the network had to say about reaching the cluster. + if opts.Pin != "" && !opts.Cloud { + return fmt.Errorf("--pin names a restore point in miren.cloud, so it only applies with --cloud") + } + client, err := ctx.RPCClient(diskBackupService) if err != nil { return err } dc := disk_v1alpha.NewDiskBackupClient(client) - if opts.Pin != "" && !opts.Cloud { - return fmt.Errorf("--pin names a restore point in miren.cloud, so it only applies with --cloud") - } - start := time.Now() progress := diskProgress(ctx) @@ -60,8 +63,15 @@ func DiskBackup(ctx *Context, opts struct { outputPath = fmt.Sprintf("%s-%s.miren.zst", opts.Name, time.Now().Format("20060102-150405")) } - outFile, err := os.Create(outputPath) + // Deliberately not os.Create. This command cleans up a snapshot that never + // finished, so truncating an existing file and then failing would destroy a + // good backup to produce nothing, which is the one outcome a backup command + // must not have. + outFile, err := os.OpenFile(outputPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("%s already exists — choose another --output path, or move it aside", outputPath) + } return fmt.Errorf("creating output file: %w", err) } diff --git a/cli/commands/disk_list_deleted.go b/cli/commands/disk_list_deleted.go index 71ecd2e77..ec01ab8b6 100644 --- a/cli/commands/disk_list_deleted.go +++ b/cli/commands/disk_list_deleted.go @@ -36,7 +36,9 @@ func DiskListDeleted(ctx *Context, opts struct { RetentionDays int `json:"retention_days"` } - var items []deletedDiskJSON + // Non-nil, so an empty result marshals as [] rather than null and + // callers can iterate it without a special case. + items := make([]deletedDiskJSON, 0, len(disks)) for _, d := range disks { items = append(items, deletedDiskJSON{ DiskName: d.DiskName(), diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index 4571a9994..ea259d8ca 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -51,7 +51,9 @@ func (r *Resolver) FindDisk(ctx context.Context, name string) (*snapshot.DiskSta switch len(matches) { case 0: - return nil, fmt.Errorf("disk %q not found", name) + // Typed, so callers can tell "no such disk" from "the lookup failed". + // PrepareRestore creates a disk on the first and must not on the second. + return nil, snapshot.DiskNotFoundError{Name: name} case 1: return &matches[0], nil default: @@ -89,7 +91,10 @@ func (r *Resolver) FindVolume(ctx context.Context, diskID string) (*snapshot.Vol // RestoreTarget includes a Finalize callback that creates the disk_volume // entity and transitions the disk to PROVISIONED. func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { - sizeGb := sizeBytes / (1 << 30) + // Round up. Truncating would hand back a disk that claims less capacity + // than the image it is about to be given, so a 1.5 GiB image would land on + // a disk reporting 1 GiB. + sizeGb := (sizeBytes + (1 << 30) - 1) / (1 << 30) if sizeGb == 0 { sizeGb = 1 } diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index 8eeb03f5b..728a1c6f5 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -373,3 +373,35 @@ func TestCreateDiskAndVolume_CleanupRunsAfterCancellation(t *testing.T) { _, statErr := os.Stat(target.ImagePath) assert.True(t, os.IsNotExist(statErr), "the restored image must be removed, got %v", statErr) } + +// A disk that claims less capacity than the image it is given is wrong on its +// face, so the size rounds up rather than truncating. +func TestCreateDiskAndVolumeRoundsSizeUp(t *testing.T) { + ctx := t.Context() + + cases := []struct { + name string + sizeBytes int64 + wantGb int64 + }{ + {"exactly one GiB", 1 << 30, 1}, + {"a byte over one GiB", (1 << 30) + 1, 2}, + {"one and a half GiB", (1 << 30) * 3 / 2, 2}, + {"exactly two GiB", 2 << 30, 2}, + {"smaller than a GiB still gets one", 4096, 1}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + es, resolver := setupResolver(t, nil) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", tc.sizeBytes, "ext4", t.TempDir()) + require.NoError(t, err) + require.NotNil(t, target) + + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, tc.wantGb, disks[0].SizeGb) + }) + } +} diff --git a/pkg/snapshot/disk.go b/pkg/snapshot/disk.go index a254359fd..63b4caf0a 100644 --- a/pkg/snapshot/disk.go +++ b/pkg/snapshot/disk.go @@ -2,6 +2,7 @@ package snapshot import ( "context" + "errors" "fmt" "path/filepath" ) @@ -12,6 +13,20 @@ const ( LeaseStatusBound = "BOUND" ) +// DiskNotFoundError reports that no disk answers to a name. +// +// It is a distinct type because "there is no such disk" and "I could not find +// out" call for opposite responses. Callers act on the first by creating the +// disk, or by reporting that there is nothing to recover. Acting on the second +// the same way turns a moment of entity-store trouble into a duplicate disk. +type DiskNotFoundError struct { + Name string +} + +func (e DiskNotFoundError) Error() string { + return fmt.Sprintf("disk %q not found", e.Name) +} + // DiskState holds the state of a disk entity as returned by a DiskResolver. type DiskState struct { ID string @@ -113,10 +128,14 @@ func PrepareRestore(ctx context.Context, resolver DiskResolver, name string, dat disk, err := resolver.FindDisk(ctx, name) if err != nil { - if cfg.creator == nil { + var notFound DiskNotFoundError + if cfg.creator == nil || !errors.As(err, ¬Found) { + // Only an actual absence means "create it". A lookup that failed + // says nothing about whether the disk exists, and creating one on + // that basis is how a brief entity-store outage turns into two + // disks answering to the same name. return nil, err } - // Disk not found — create it return cfg.creator.CreateDiskAndVolume(ctx, name, cfg.sizeBytes, cfg.filesystem, dataPath) } diff --git a/pkg/snapshot/disk_test.go b/pkg/snapshot/disk_test.go index ebb8a6f19..ca9b68d8a 100644 --- a/pkg/snapshot/disk_test.go +++ b/pkg/snapshot/disk_test.go @@ -2,6 +2,7 @@ package snapshot import ( "context" + "errors" "fmt" "testing" @@ -218,3 +219,41 @@ func TestPrepareRestore(t *testing.T) { assert.Equal(t, "/var/lib/miren/disk-data/volumes/vol-xyz/disk.img", target.ImagePath) }) } + +// mockCreator records whether PrepareRestore decided to create a disk. +type mockCreator struct { + called bool +} + +func (m *mockCreator) CreateDiskAndVolume(_ context.Context, name string, _ int64, _ string, _ string) (*RestoreTarget, error) { + m.called = true + return &RestoreTarget{Name: name, ImagePath: "/data/new.img", Created: true}, nil +} + +// "There is no such disk" and "I could not find out" call for opposite +// responses, and only the first one means create it. Treating a failed lookup +// as an absence is how a brief entity-store outage becomes two disks answering +// to the same name. +func TestPrepareRestoreOnlyCreatesOnAGenuineAbsence(t *testing.T) { + ctx := context.Background() + + t.Run("absent disk is created", func(t *testing.T) { + c := &mockCreator{} + r := &mockResolver{diskErr: DiskNotFoundError{Name: "mydb"}} + + target, err := PrepareRestore(ctx, r, "mydb", "/var/lib/miren", WithCreator(c, 1<<30, "ext4")) + require.NoError(t, err) + assert.True(t, c.called, "an absent disk should be created") + assert.True(t, target.Created) + }) + + t.Run("a failed lookup is reported, not papered over", func(t *testing.T) { + c := &mockCreator{} + r := &mockResolver{diskErr: errors.New("listing disks: etcd unreachable")} + + _, err := PrepareRestore(ctx, r, "mydb", "/var/lib/miren", WithCreator(c, 1<<30, "ext4")) + require.Error(t, err) + assert.Contains(t, err.Error(), "etcd unreachable") + assert.False(t, c.called, "a lookup failure must not create a disk") + }) +} diff --git a/servers/disk/backup.go b/servers/disk/backup.go index 8f69ec899..efc34fb5d 100644 --- a/servers/disk/backup.go +++ b/servers/disk/backup.go @@ -124,6 +124,12 @@ func (s *Server) backupToClient( return refuse("backup needs either --cloud or somewhere to write the snapshot") } + // Held across staging and the whole stream, not just the file opens. The + // window that matters is the one where another call could truncate the + // staged snapshot while this one is reading it. + release := s.transfers.acquire(args.TransferId()) + defer release() + staged, meta, err := s.stageForClient(args.TransferId(), prog, target) if err != nil { return err diff --git a/servers/disk/restore.go b/servers/disk/restore.go index 5e486df36..167f079ca 100644 --- a/servers/disk/restore.go +++ b/servers/disk/restore.go @@ -30,6 +30,16 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest return refuse("disk name is required") } + // An uploaded snapshot lives in a file keyed by transfer id, and it stays + // there until the image is installed. Hold the id for the whole handler, so + // an overlapping call cannot append into the same file or pull it out from + // under the install. A restore point needs none of this: it comes from the + // cloud and touches no transfer file. + if id := args.TransferId(); id != "" && args.RestorePoint() == "" { + release := s.transfers.acquire(id) + defer release() + } + src, total, closeSrc, err := s.restoreSource(ctx, name, args, prog) if err != nil { return err @@ -347,6 +357,28 @@ func (s *Server) writeImage(imagePath string, src io.Reader, compressedSize int6 if err := os.Rename(tmpPath, imagePath); err != nil { return fmt.Errorf("moving image into place: %w", err) } + + // Syncing the file only promised its contents survive a power loss, not the + // directory entry that gives them this name. Losing the rename would leave + // the old image in place while the disk was already reported restored, and + // on a recovery path that is exactly the report you cannot have be wrong. + if err := syncDir(filepath.Dir(imagePath)); err != nil { + return err + } + cleanup = false return nil } + +func syncDir(path string) error { + d, err := os.Open(path) + if err != nil { + return fmt.Errorf("opening volume directory: %w", err) + } + defer d.Close() + + if err := d.Sync(); err != nil { + return fmt.Errorf("flushing volume directory: %w", err) + } + return nil +} diff --git a/servers/disk/server.go b/servers/disk/server.go index 9416388d6..da19fdbdc 100644 --- a/servers/disk/server.go +++ b/servers/disk/server.go @@ -59,6 +59,10 @@ type Server struct { // mntOps answers "is this image currently in use". It reads kernel state // directly, so it needs no handle on the runner's controllers. mntOps diskio.DiskMountOps + + // transfers serializes concurrent work on one transfer id, which is the + // only shared mutable state these handlers have. + transfers *transferLocks } // NewServer builds the disk backup service. updates may be nil, which means the @@ -72,13 +76,14 @@ func NewServer( ) *Server { log = log.With("module", "disk-backup") return &Server{ - log: log, - disks: diskresolve.New(eac, ec), - dataPath: dataPath, - eac: eac, - ec: ec, - updates: updates, - mntOps: diskio.NewRealDiskMountOps(log), + log: log, + disks: diskresolve.New(eac, ec), + dataPath: dataPath, + eac: eac, + ec: ec, + updates: updates, + mntOps: diskio.NewRealDiskMountOps(log), + transfers: newTransferLocks(), } } diff --git a/servers/disk/transfer.go b/servers/disk/transfer.go index 45607fc4a..ce7e924b5 100644 --- a/servers/disk/transfer.go +++ b/servers/disk/transfer.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" "miren.dev/runtime/api/disk/disk_v1alpha" @@ -25,6 +26,58 @@ const ( transferTTL = 24 * time.Hour ) +// transferLocks serializes work on any one transfer id. +// +// The transfer files are shared mutable state, and nothing else guards them. +// Two calls carrying the same id can otherwise both pass openTransfer's offset +// check and interleave their appends, or one can truncate a staged snapshot +// while the other is streaming it. A well-behaved client never does this, since +// it picks a fresh id per invocation and retries in sequence, but the server +// should not be relying on that. +// +// Locks are reference counted so the map does not grow by one entry per backup +// forever. Dropping an entry while a caller still held a pointer to it would be +// worse than the leak: the next caller would mint a second lock for the same id +// and the two would run concurrently anyway. +type transferLocks struct { + mu sync.Mutex + locks map[string]*transferLock +} + +type transferLock struct { + mu sync.Mutex + refs int +} + +func newTransferLocks() *transferLocks { + return &transferLocks{locks: map[string]*transferLock{}} +} + +// acquire blocks until this id is free and returns the release function. +func (t *transferLocks) acquire(id string) func() { + t.mu.Lock() + l, ok := t.locks[id] + if !ok { + l = &transferLock{} + t.locks[id] = l + } + l.refs++ + t.mu.Unlock() + + l.mu.Lock() + + return func() { + l.mu.Unlock() + + t.mu.Lock() + l.refs-- + if l.refs == 0 { + delete(t.locks, id) + } + t.mu.Unlock() + } +} + // transferPath is where a transfer's bytes accumulate. // // The id comes from the client, so it is checked rather than trusted: it names @@ -235,7 +288,13 @@ func (s *Server) sweepTransfers() { cutoff := time.Now().Add(-transferTTL) for _, e := range entries { - if e.IsDir() || !strings.HasSuffix(e.Name(), ".part") { + if e.IsDir() { + continue + } + // Both halves of a staged backup are swept. Taking only the .part would + // leave its .meta behind forever, since nothing else ever looks at a + // metadata file whose bytes are gone. + if !strings.HasSuffix(e.Name(), ".part") && !strings.HasSuffix(e.Name(), ".meta") { continue } info, ierr := e.Info() diff --git a/servers/disk/transfer_test.go b/servers/disk/transfer_test.go index 4b33a181b..6b9818491 100644 --- a/servers/disk/transfer_test.go +++ b/servers/disk/transfer_test.go @@ -171,3 +171,96 @@ func TestSweepRemovesAbandonedTransfersAndKeepsFreshOnes(t *testing.T) { assert.True(t, os.IsNotExist(err), "a transfer nobody came back for should be reclaimed") assert.FileExists(t, fresh, "a transfer still within its window must survive") } + +// The lock is what stops two calls carrying the same id from interleaving their +// appends or truncating each other's staging. +func TestTransferLockSerializesTheSameId(t *testing.T) { + locks := newTransferLocks() + + release := locks.acquire("t1") + + entered := make(chan struct{}) + go func() { + r := locks.acquire("t1") + close(entered) + r() + }() + + select { + case <-entered: + t.Fatal("a second holder of the same id must wait") + case <-time.After(50 * time.Millisecond): + } + + release() + + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("releasing the id should have let the waiter through") + } +} + +// Different transfers must not queue behind each other; two operators backing +// up different disks are not related. +func TestTransferLockDoesNotSerializeDifferentIds(t *testing.T) { + locks := newTransferLocks() + + release := locks.acquire("t1") + defer release() + + done := make(chan struct{}) + go func() { + r := locks.acquire("t2") + r() + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("a different transfer id should not have blocked") + } +} + +// Reference counting keeps the map from growing by one entry per backup, and +// must not drop an entry another caller is still holding. +func TestTransferLockReleasesItsBookkeeping(t *testing.T) { + locks := newTransferLocks() + + for range 50 { + locks.acquire("t1")() + } + + locks.mu.Lock() + n := len(locks.locks) + locks.mu.Unlock() + assert.Zero(t, n, "a fully released id should leave no entry behind") +} + +// The sweep has to take both halves of a staged backup. Taking only the bytes +// would leave the metadata behind forever, since nothing looks at a metadata +// file whose .part is gone. +func TestSweepRemovesStagingMetadataToo(t *testing.T) { + s, _, _, _ := newTestServer(t, []byte("hello")) + + dir := filepath.Join(s.diskDataPath(), transfersDir) + require.NoError(t, os.MkdirAll(dir, 0700)) + + part := filepath.Join(dir, "stale.part") + meta := filepath.Join(dir, "stale.meta") + require.NoError(t, os.WriteFile(part, []byte("bytes"), 0600)) + require.NoError(t, os.WriteFile(meta, []byte("{}"), 0600)) + + old := time.Now().Add(-transferTTL - time.Hour) + for _, p := range []string{part, meta} { + require.NoError(t, os.Chtimes(p, old, old)) + } + + s.sweepTransfers() + + for _, p := range []string{part, meta} { + _, err := os.Stat(p) + assert.True(t, os.IsNotExist(err), "%s should have been reclaimed", p) + } +} diff --git a/servers/disk/undelete.go b/servers/disk/undelete.go index 7193f68db..0f5d395ab 100644 --- a/servers/disk/undelete.go +++ b/servers/disk/undelete.go @@ -2,6 +2,7 @@ package disk import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -15,6 +16,7 @@ import ( "miren.dev/runtime/pkg/diskresolve" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/snapshot" ) // diskDataPath is where volume directories and the soft-delete holding area @@ -112,8 +114,16 @@ func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undele // A live disk already owning this name would end up with two entities // answering to it, and every lookup by name is ambiguous from then on. - if _, err := s.disks.FindDisk(ctx, name); err == nil { + // + // Only an actual absence clears the way. A lookup that merely failed says + // nothing about whether the name is taken, and treating it as free is how + // a moment of entity-store trouble becomes the duplicate this check exists + // to prevent. + switch _, err := s.disks.FindDisk(ctx, name); { + case err == nil: return nil, refuse("a disk named %q already exists — rename or delete it before recovering this one", name) + case !errors.As(err, &snapshot.DiskNotFoundError{}): + return nil, fmt.Errorf("checking whether %q is already taken: %w", name, err) } filesystem := strings.TrimPrefix(strings.ToLower(meta.Filesystem), "filesystem.") @@ -123,18 +133,23 @@ func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undele return nil, err } + // From here on there are entities to unwind, and unwinding has to survive + // the thing that made it necessary. A client that disconnects mid-recovery + // cancels ctx, which is exactly when the cleanup below must still run. + cleanupCtx := context.WithoutCancel(ctx) + volID := meta.VolumeID destPath := filepath.Join(s.diskDataPath(), "volumes", volID) // The runner creates this on startup, but a recovery can be the first thing // that happens on a rebuilt host, and rename will not create it. if err := os.MkdirAll(filepath.Dir(destPath), 0700); err != nil { - s.deleteEntity(ctx, string(diskEntityId), "disk") + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") return nil, fmt.Errorf("creating volumes directory: %w", err) } if err := os.Rename(entry.Path, destPath); err != nil { - s.deleteEntity(ctx, string(diskEntityId), "disk") + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") return nil, fmt.Errorf("moving volume back to %s: %w", destPath, err) } @@ -146,13 +161,10 @@ func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undele if committed { return } - // The request's context is already cancelled when a client disconnects - // mid-recovery, and rolling back is exactly what has to happen then. - rctx := context.WithoutCancel(ctx) if rerr := os.Rename(destPath, entry.Path); rerr != nil { s.log.Warn("failed to move volume back to deleted-volumes", "volume_id", volID, "error", rerr) } - s.deleteEntity(rctx, string(diskEntityId), "disk") + s.deleteEntity(cleanupCtx, string(diskEntityId), "disk") }() imagePath := filepath.Join(destPath, "disk.img") @@ -201,7 +213,7 @@ func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undele entity.String(storage_v1alpha.DiskVolumeIdId, volID), }, 0) if err != nil { - s.deleteEntity(ctx, string(volEntityId), "disk_volume") + s.deleteEntity(cleanupCtx, string(volEntityId), "disk_volume") return nil, fmt.Errorf("updating disk to provisioning: %w", err) } From 8725de329cfccac2e48bb574e8e2718f5692c594 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 23:10:18 -0700 Subject: [PATCH 15/19] Address the second review round Rounding the disk size up, from the first round, introduced an overflow: adding gib-1 to a size near the top of the int64 range wraps to a negative capacity, and the entity store would have persisted it. The size comes out of a snapshot header, which is a file the caller handed us, so it is now checked rather than trusted, and the rounding uses a remainder instead so it cannot wrap. Recovering a deleted disk checked that the name was free and then created a disk under it, two steps with a gap. Two recoveries of the same name could both find it free, leaving two disks answering to one name and every lookup by that name ambiguous from then on. Recoveries now serialize on the name. Process-local is the right granularity for that rather than a shortcut, and the reasoning is in the code: disk volumes are pinned to the coordinator, so one process serves every recovery on a cluster. The reviewer asked for a conditional unique-name create in the entity store, which is what this would need if disks ever schedule anywhere else. --- pkg/diskresolve/resolver.go | 43 +++++++++++++++++++++++++++----- pkg/diskresolve/resolver_test.go | 32 ++++++++++++++++++++++++ servers/disk/server.go | 11 +++++--- servers/disk/server_test.go | 3 +++ servers/disk/transfer.go | 18 ++++++------- servers/disk/transfer_test.go | 6 ++--- servers/disk/undelete.go | 13 ++++++++++ servers/disk/undelete_test.go | 36 ++++++++++++++++++++++++++ 8 files changed, 140 insertions(+), 22 deletions(-) diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index ea259d8ca..4a92e856d 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -3,6 +3,7 @@ package diskresolve import ( "context" "fmt" + "math" "os" "os/exec" "path/filepath" @@ -91,12 +92,9 @@ func (r *Resolver) FindVolume(ctx context.Context, diskID string) (*snapshot.Vol // RestoreTarget includes a Finalize callback that creates the disk_volume // entity and transitions the disk to PROVISIONED. func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { - // Round up. Truncating would hand back a disk that claims less capacity - // than the image it is about to be given, so a 1.5 GiB image would land on - // a disk reporting 1 GiB. - sizeGb := (sizeBytes + (1 << 30) - 1) / (1 << 30) - if sizeGb == 0 { - sizeGb = 1 + sizeGb, err := diskSizeGb(sizeBytes) + if err != nil { + return nil, err } // Normalize filesystem string — strip enum prefix if present @@ -275,6 +273,39 @@ func (r *Resolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.Le return leases, nil } +// gib is the unit disks are sized in. +const gib = 1 << 30 + +// diskSizeGb converts an image size into the capacity to record on the disk. +// +// It rounds up, because a disk claiming less capacity than the image it is +// about to be given is wrong on its face: a 1.5 GiB image would land on a disk +// reporting 1 GiB. +// +// The size comes from a snapshot header, which is a file the caller handed us, +// so it is checked rather than trusted. The rounding is done with a remainder +// rather than by adding gib-1 so that a size near the top of the range cannot +// overflow into a negative capacity, and the upper bound is where the +// controller's own conversion back to bytes would overflow. +func diskSizeGb(sizeBytes int64) (int64, error) { + switch { + case sizeBytes < 0: + return 0, fmt.Errorf("snapshot reports a negative image size (%d bytes)", sizeBytes) + case sizeBytes > math.MaxInt64-gib: + return 0, fmt.Errorf("snapshot reports an image size too large to be real (%d bytes)", sizeBytes) + } + + sizeGb := sizeBytes / gib + if sizeBytes%gib != 0 { + sizeGb++ + } + // Even an empty image gets a disk, and a disk has to be at least 1 GB. + if sizeGb == 0 { + sizeGb = 1 + } + return sizeGb, nil +} + // ParseFilesystem maps a filesystem name onto the disk enum, tolerating the // "filesystem." prefix the enum renders with. Anything unrecognized becomes // ext4, which is the default a disk gets when it does not ask for one. diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index 728a1c6f5..5333265e7 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -3,6 +3,7 @@ package diskresolve import ( "context" "fmt" + "math" "os" "path/filepath" "sync" @@ -389,6 +390,7 @@ func TestCreateDiskAndVolumeRoundsSizeUp(t *testing.T) { {"one and a half GiB", (1 << 30) * 3 / 2, 2}, {"exactly two GiB", 2 << 30, 2}, {"smaller than a GiB still gets one", 4096, 1}, + {"zero still gets one", 0, 1}, } for _, tc := range cases { @@ -405,3 +407,33 @@ func TestCreateDiskAndVolumeRoundsSizeUp(t *testing.T) { }) } } + +// The size comes out of a snapshot header, which is a file the caller handed +// us, so it is checked rather than trusted. Rounding by adding gib-1 would let +// a size near the top of the range overflow into a negative capacity, which the +// entity store would happily persist. +func TestDiskSizeGbRejectsSizesItCannotRepresent(t *testing.T) { + for _, tc := range []struct { + name string + sizeBytes int64 + }{ + {"negative", -1}, + {"very negative", math.MinInt64}, + {"large enough to overflow the rounding", math.MaxInt64}, + {"just past the representable ceiling", math.MaxInt64 - (1 << 30) + 1}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := diskSizeGb(tc.sizeBytes) + require.Error(t, err, "size %d should have been rejected", tc.sizeBytes) + }) + } +} + +// And the largest size it does accept round-trips back to bytes without +// overflowing, which is what the disk volume controller does with it. +func TestDiskSizeGbCeilingRoundTrips(t *testing.T) { + sizeGb, err := diskSizeGb(math.MaxInt64 - (1 << 30)) + require.NoError(t, err) + assert.Positive(t, sizeGb) + assert.Positive(t, sizeGb*(1<<30)/(1<<30), "the byte conversion must not overflow") +} diff --git a/servers/disk/server.go b/servers/disk/server.go index da19fdbdc..355e02671 100644 --- a/servers/disk/server.go +++ b/servers/disk/server.go @@ -60,9 +60,11 @@ type Server struct { // directly, so it needs no handle on the runner's controllers. mntOps diskio.DiskMountOps - // transfers serializes concurrent work on one transfer id, which is the - // only shared mutable state these handlers have. - transfers *transferLocks + // transfers serializes concurrent work on one transfer id, and names does + // the same for one disk name, so a check followed by a create cannot be + // overtaken between the two steps. + transfers *keyedLocks + names *keyedLocks } // NewServer builds the disk backup service. updates may be nil, which means the @@ -83,7 +85,8 @@ func NewServer( ec: ec, updates: updates, mntOps: diskio.NewRealDiskMountOps(log), - transfers: newTransferLocks(), + transfers: newKeyedLocks(), + names: newKeyedLocks(), } } diff --git a/servers/disk/server_test.go b/servers/disk/server_test.go index cd56033cf..192417713 100644 --- a/servers/disk/server_test.go +++ b/servers/disk/server_test.go @@ -134,6 +134,9 @@ func newTestServer(t *testing.T, content []byte) (*Server, *fakeDisks, *fakeUpda dataPath: dataPath, updates: updates, mntOps: fakeMountOps{}, + + transfers: newKeyedLocks(), + names: newKeyedLocks(), } return s, disks, updates, imagePath } diff --git a/servers/disk/transfer.go b/servers/disk/transfer.go index ce7e924b5..348cbd688 100644 --- a/servers/disk/transfer.go +++ b/servers/disk/transfer.go @@ -26,7 +26,7 @@ const ( transferTTL = 24 * time.Hour ) -// transferLocks serializes work on any one transfer id. +// keyedLocks serializes work on any one key: a transfer id, or a disk name. // // The transfer files are shared mutable state, and nothing else guards them. // Two calls carrying the same id can otherwise both pass openTransfer's offset @@ -39,26 +39,26 @@ const ( // forever. Dropping an entry while a caller still held a pointer to it would be // worse than the leak: the next caller would mint a second lock for the same id // and the two would run concurrently anyway. -type transferLocks struct { +type keyedLocks struct { mu sync.Mutex - locks map[string]*transferLock + locks map[string]*keyedLock } -type transferLock struct { +type keyedLock struct { mu sync.Mutex refs int } -func newTransferLocks() *transferLocks { - return &transferLocks{locks: map[string]*transferLock{}} +func newKeyedLocks() *keyedLocks { + return &keyedLocks{locks: map[string]*keyedLock{}} } -// acquire blocks until this id is free and returns the release function. -func (t *transferLocks) acquire(id string) func() { +// acquire blocks until this key is free and returns the release function. +func (t *keyedLocks) acquire(id string) func() { t.mu.Lock() l, ok := t.locks[id] if !ok { - l = &transferLock{} + l = &keyedLock{} t.locks[id] = l } l.refs++ diff --git a/servers/disk/transfer_test.go b/servers/disk/transfer_test.go index 6b9818491..e32a48b2f 100644 --- a/servers/disk/transfer_test.go +++ b/servers/disk/transfer_test.go @@ -175,7 +175,7 @@ func TestSweepRemovesAbandonedTransfersAndKeepsFreshOnes(t *testing.T) { // The lock is what stops two calls carrying the same id from interleaving their // appends or truncating each other's staging. func TestTransferLockSerializesTheSameId(t *testing.T) { - locks := newTransferLocks() + locks := newKeyedLocks() release := locks.acquire("t1") @@ -204,7 +204,7 @@ func TestTransferLockSerializesTheSameId(t *testing.T) { // Different transfers must not queue behind each other; two operators backing // up different disks are not related. func TestTransferLockDoesNotSerializeDifferentIds(t *testing.T) { - locks := newTransferLocks() + locks := newKeyedLocks() release := locks.acquire("t1") defer release() @@ -226,7 +226,7 @@ func TestTransferLockDoesNotSerializeDifferentIds(t *testing.T) { // Reference counting keeps the map from growing by one entry per backup, and // must not drop an entry another caller is still holding. func TestTransferLockReleasesItsBookkeeping(t *testing.T) { - locks := newTransferLocks() + locks := newKeyedLocks() for range 50 { locks.acquire("t1")() diff --git a/servers/disk/undelete.go b/servers/disk/undelete.go index 0f5d395ab..00ac5a562 100644 --- a/servers/disk/undelete.go +++ b/servers/disk/undelete.go @@ -106,6 +106,19 @@ func (s *Server) undelete(ctx context.Context, name, volumeID string) (_ *undele return nil, refuse("disk name is required") } + // The name check below and the create that follows it are two steps, so two + // recoveries of the same name could both find it free and both take it, + // leaving two disks answering to one name and every lookup by that name + // ambiguous from then on. + // + // Process-local is the right granularity here rather than a shortcut: disk + // volumes are pinned to the coordinator, so one process serves every + // recovery on a cluster. Making this safe across processes would mean a + // conditional unique-name create in the entity store, which is worth doing + // if disks ever schedule anywhere else. + releaseName := s.names.acquire(name) + defer releaseName() + entry, err := s.findDeleted(name, volumeID) if err != nil { return nil, err diff --git a/servers/disk/undelete_test.go b/servers/disk/undelete_test.go index d06aa5315..780afb561 100644 --- a/servers/disk/undelete_test.go +++ b/servers/disk/undelete_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "sync" "testing" "time" @@ -45,6 +46,9 @@ func newUndeleteServer(t *testing.T) (*Server, *testutils.InMemEntityServer, str eac: es.EAC, ec: ec, mntOps: fakeMountOps{}, + + transfers: newKeyedLocks(), + names: newKeyedLocks(), } return s, es, dataPath } @@ -246,3 +250,35 @@ func listVolumes(t *testing.T, es *testutils.InMemEntityServer) []storage_v1alph } return out } + +// The name check and the create that follows it are two steps, so two +// recoveries of the same name must not both find it free. Two disks answering +// to one name make every lookup by that name ambiguous from then on. +func TestUndeleteSerializesRecoveriesOfTheSameName(t *testing.T) { + s, es, dataPath := newUndeleteServer(t) + seedDeleted(t, dataPath, "mydisk", "vol-1", time.Now()) + + var wg sync.WaitGroup + errs := make([]error, 2) + for i := range errs { + wg.Add(1) + go func() { + defer wg.Done() + _, errs[i] = s.undelete(context.Background(), "mydisk", "") + }() + } + wg.Wait() + + // One recovers it; the other finds nothing left in the holding area, or + // finds the name taken. Either way it must not create a second disk. + succeeded := 0 + for _, err := range errs { + if err == nil { + succeeded++ + } + } + assert.Equal(t, 1, succeeded, "exactly one recovery should have succeeded") + + disks := listDisks(t, es) + assert.Len(t, disks, 1, "a name must never end up on two disks") +} From 19611641cdf2dc08ca8ddcc87e834fb889827fe0 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 8 Sep 2026 14:28:54 -0700 Subject: [PATCH 16/19] Address review: three restore correctness fixes A fresh restore was leaving two volumes on one disk, both mounted, and the next backup failed on the ambiguity. Finalize announced the disk as PROVISIONED before creating its disk_volume, and DiskController answers a provisioned disk with no volume by provisioning a blank one. The restore's own volume then landed beside it. Finalize now creates the volume first and hands the disk over as PROVISIONING, which is the handshake a recovered disk already used. DiskController ignores a RESTORING disk, so there is no longer any window where the disk is visible without its volume, and one path owns the transition instead of two. Cleanup moves with it. It used to remove the image before touching the disk, on the reasoning that no volume could exist yet; a partial Finalize can now leave one the controller is already mounting, and pulling an image out from under a loop device leaves it holding an unlinked inode rather than releasing it. Marking the disk DELETING first lets the teardown path unmount and detach before the image goes. Retrying an upload that had already been delivered in full failed on the file rather than on anything real. The stream helper closes what it reads from once it reaches the end, so the first complete upload closed the snapshot, and every later attempt died seeking it. The file is wrapped in something with no Close for the stream to find, and ownership stays with the caller. Overlapping restores to the same name are serialized, the way recoveries already were. Restore creates the disk when it does not exist, so two of them racing to one name could both create it, or one could catch the other's disk half-built and fail looking for a volume that was not there yet. Two smaller ones. The size ceiling rejected the largest snapshot it could actually represent, since that value is exactly divisible by a GiB and converts straight back without overflowing. And the docs told operators to scale the app to zero before restoring, which is not a thing that can be done: disk-backed services run at fixed concurrency, num_instances = 0 is rejected, and a manual scale to zero is reconciled back. They now say in-place restore of a live disk is not supported, and document restoring into a new disk and moving the app across, which is what works. --- cli/commands/disk_restore.go | 6 +- cli/commands/disk_transfer.go | 16 ++++ cli/commands/disk_transfer_test.go | 67 ++++++++++++++++ docs/docs/addons.md | 31 ++++++-- pkg/diskresolve/resolver.go | 124 ++++++++++++++++------------- pkg/diskresolve/resolver_test.go | 87 ++++++++++++++------ servers/disk/restore.go | 22 +++-- 7 files changed, 261 insertions(+), 92 deletions(-) diff --git a/cli/commands/disk_restore.go b/cli/commands/disk_restore.go index 565468184..05d5041cc 100644 --- a/cli/commands/disk_restore.go +++ b/cli/commands/disk_restore.go @@ -119,8 +119,12 @@ func DiskRestore(ctx *Context, opts struct { return fmt.Errorf("seeking snapshot to %d: %w", offset, serr) } + // readerOnly, not snapFile itself: ServeReader closes what it is given + // once it reaches the end of the stream, and a retry after a fully + // delivered upload has to seek this same file again. Ownership stays + // with the defer above. res, rerr := dc.Restore(ctx, name, "", - stream.ServeReader(ctx, snapFile, stream.WithBulkBatching()), + stream.ServeReader(ctx, readerOnly{snapFile}, stream.WithBulkBatching()), opts.Force, progress, transferID, offset) if rerr != nil { return rerr diff --git a/cli/commands/disk_transfer.go b/cli/commands/disk_transfer.go index de7882fc7..1bdd04d05 100644 --- a/cli/commands/disk_transfer.go +++ b/cli/commands/disk_transfer.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "time" "miren.dev/runtime/pkg/cond" @@ -28,6 +29,21 @@ const ( transferRetryMax = 60 * time.Second ) +// readerOnly hides a reader's Close from whoever it is handed to. +// +// The stream helper closes what it reads from once it reaches the end, which is +// right when it owns the reader and wrong when the caller still needs it. A +// retry has to seek the same file again, and an upload that got all the way to +// the end before the server rejected it is exactly when that happens: the file +// would already be closed and every later attempt would fail on the seek rather +// than on anything real. +// +// Embedding only io.Reader is what does the hiding: the wrapper has Read and +// nothing else, so a type assertion for io.Closer finds nothing to call. +type readerOnly struct { + io.Reader +} + // newTransferID names one backup or restore so an interrupted one can be // resumed. // diff --git a/cli/commands/disk_transfer_test.go b/cli/commands/disk_transfer_test.go index c5a424cf3..f81eb9353 100644 --- a/cli/commands/disk_transfer_test.go +++ b/cli/commands/disk_transfer_test.go @@ -3,6 +3,9 @@ package commands import ( "errors" "fmt" + "io" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -59,3 +62,67 @@ func TestTransferIDsUseOnlySafeCharacters(t *testing.T) { } } } + +// closeCountingFile stands in for the snapshot file a restore uploads from, +// recording whether anything closed it out from under us. +type closeCountingFile struct { + *os.File + closes int +} + +func (f *closeCountingFile) Close() error { + f.closes++ + return f.File.Close() +} + +// A retry has to be able to seek the snapshot again, and the case that catches +// this is the one where the upload fully succeeded and the server then rejected +// it: the stream reaches EOF, closes what it was reading from, and every later +// attempt fails on the seek rather than on anything real. +// +// readerOnly is what prevents that, by handing the stream something with no +// Close to find. +func TestReaderOnlyHidesCloseFromTheStream(t *testing.T) { + path := filepath.Join(t.TempDir(), "snap.miren.zst") + require.NoError(t, os.WriteFile(path, []byte("a snapshot's worth of bytes"), 0600)) + + f, err := os.Open(path) + require.NoError(t, err) + tracked := &closeCountingFile{File: f} + defer tracked.File.Close() + + // What the stream helper does when it reaches the end: close the reader if + // it can. Wrapped, there is nothing to close. + var wrapped io.Reader = readerOnly{tracked} + _, isCloser := wrapped.(io.Closer) + assert.False(t, isCloser, "the stream must not be able to close the caller's file") + + // Read it to the end, the way a completed upload does. + _, err = io.ReadAll(wrapped) + require.NoError(t, err) + assert.Zero(t, tracked.closes, "reading to EOF must not have closed the file") + + // And a retry can still rewind and re-send it. + _, err = tracked.Seek(0, io.SeekStart) + require.NoError(t, err, "a retry after a complete upload must still be able to seek") + + again, err := io.ReadAll(readerOnly{tracked}) + require.NoError(t, err) + assert.Equal(t, "a snapshot's worth of bytes", string(again)) +} + +// The unwrapped file is the shape that caused the bug, so pin the difference: +// handed the file directly, the stream helper would find a Closer to call. +func TestAnUnwrappedFileWouldBeClosedByTheStream(t *testing.T) { + path := filepath.Join(t.TempDir(), "snap.miren.zst") + require.NoError(t, os.WriteFile(path, []byte("bytes"), 0600)) + + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + + var plain io.Reader = f + _, isCloser := plain.(io.Closer) + assert.True(t, isCloser, + "an *os.File is a Closer, which is why it has to be wrapped before it is streamed") +} diff --git a/docs/docs/addons.md b/docs/docs/addons.md index e0d65beaa..856ca8c1f 100644 --- a/docs/docs/addons.md +++ b/docs/docs/addons.md @@ -439,26 +439,43 @@ miren disk restore -s myapp-db.miren.zst --force ``` -To restore to a different disk name: +:::warning[Restore into a new disk, not over the running one] +Restoring on top of a disk your app is using is not supported today, and the command refuses it rather than pretending. A disk in use is held open by the kernel, so writing a new image into it would leave the running database on the old data while reporting success. + +There is currently no way to release a disk while keeping the app configured for it. Disk-backed services run at fixed concurrency, `num_instances = 0` is rejected, and scaling a pool to zero by hand is reconciled straight back to one. Even in that brief window the disk stays mounted. + +So the working recovery is to restore into a **new** disk and move the app across. +::: + +Restore into a new disk name: ```miren -miren disk restore -s myapp-db.miren.zst -n new-disk-name +miren disk restore -s myapp-db.miren.zst -n myapp-db-restored ``` -:::warning[Stop the app before restoring] -Restore refuses to write over a disk that is still attached. A disk in use is held open by the kernel, so a restore into it would leave the running database on the old data while reporting success — the command stops rather than let that happen. Scale the app to zero, or restore into a new disk name and switch over. -::: +Then point the service at it in `.miren/app.toml` and deploy: -After restoring, restart your app to pick up the restored data: +```toml +[[services.db.disks]] +name = "myapp-db-restored" +mount_path = "/var/lib/postgresql/data" +size_gb = 20 +``` ```miren -miren app restart myapp +miren deploy ``` +The one case that does restore in place is the one where nothing is holding the disk: a rebuilt host whose disk image is gone. There the image is absent rather than mounted, so `miren disk restore` writes it and the app picks it up on the next start. + +:::note[`--from-cloud` restores into the disk the point came from] +A cloud restore point is looked up through its own disk's cloud volume, so `--from-cloud` needs that disk to already exist and be registered. It cannot restore into a new disk name, which means the move-the-app-across path above needs a local snapshot file. Keep one for a disk you would want to recover onto a running cluster. +::: + ### Backup Recommendations - **Schedule regular backups** for production databases, especially before destructive operations like `addon destroy` diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index 4a92e856d..54dc43631 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -90,7 +90,8 @@ func (r *Resolver) FindVolume(ctx context.Context, diskID string) (*snapshot.Vol // CreateDiskAndVolume creates a new disk entity in RESTORING state so the disk // controller ignores it while restore writes the image. The returned // RestoreTarget includes a Finalize callback that creates the disk_volume -// entity and transitions the disk to PROVISIONED. +// entity and hands the disk to the controller as PROVISIONING, which promotes +// it once the volume is ready. func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeBytes int64, filesystem string, dataPath string) (*snapshot.RestoreTarget, error) { sizeGb, err := diskSizeGb(sizeBytes) if err != nil { @@ -133,34 +134,25 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt // the failure it is cleaning up after. cctx = context.WithoutCancel(cctx) - // Finalize is the last fallible step of a restore, so cleanup - // only ever runs when it did not complete — and its - // disk_volume Create is its last write. Nothing owns the image - // at this point, and the restore already renamed it into - // place, so the temp-file removal in disk_restore.go is a - // no-op for it. Remove it before touching the disk: a - // PROVISIONED disk still carrying its VolumeId can be - // self-healed into a disk_volume by the controller, and there - // is no reason to let that adopt an image on its way to being - // torn down. Tolerate "not present" — the rename may never - // have happened. - imageErr := os.Remove(imagePath) - if os.IsNotExist(imageErr) { - imageErr = nil - } - // Transition the disk to DELETING rather than deleting the // entity outright. A direct Delete bypasses the disk // controller's DELETING-driven handleDeletion path, which is - // the only writer of disk_volume.desired_state=DV_ABSENT; once - // the disk is gone, a disk_volume the controller self-healed - // from the PROVISIONED disk has no reaper and is left + // the only writer of disk_volume.desired_state=DV_ABSENT; a + // disk_volume whose disk is gone has no reaper and is left // reconciled by the coordinator as a live mount / volume // directory / phantom cloud volume. Marking the disk DELETING // keeps it alive to drive that existing contract, which tears // the disk_volume down via the coordinator's // DiskVolumeController. Idempotent: patching an already-DELETING // disk is a no-op. + // + // This runs before the image is touched, because Finalize + // creates the disk_volume first now: cleanup after a partial + // Finalize can find a volume the controller is already mounting, + // and pulling the image out from under a loop device leaves it + // holding an unlinked inode rather than releasing it. Letting + // the teardown path unmount and detach first is the only order + // that ends with nothing held. _, err := r.eac.Patch(cctx, []entity.Attr{ entity.Ref(entity.DBId, diskEntityId), entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), @@ -169,48 +161,57 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt return fmt.Errorf("transitioning disk to deleting during cleanup: %w", err) } - // Reported only once the authoritative step is done, so a - // failure to reclaim the image never costs us the disk - // rollback — a leftover image is disk space, a stuck - // RESTORING disk blocks every same-name retry. - if imageErr != nil { + // Whatever the teardown did not claim. Usually this is the whole + // job, since most cleanups run before Finalize and so before any + // volume exists. Tolerate "not present": the restore may never + // have renamed the image into place, and a volume teardown may + // have moved the directory out from under it. + if imageErr := os.Remove(imagePath); imageErr != nil && !os.IsNotExist(imageErr) { + // Reported only once the authoritative step is done, so a + // failure to reclaim the image never costs us the disk + // rollback — a leftover image is disk space, a stuck + // RESTORING disk blocks every same-name retry. return fmt.Errorf("removing restored image during cleanup: %w", imageErr) } return nil }, Finalize: func(fctx context.Context) error { vol := &storage_v1alpha.DiskVolume{ - Name: name, - DiskId: diskEntityId, - VolumeId: volId, - SizeGb: sizeGb, - Filesystem: filesystem, - VolumeMode: DetectVolumeMode(), - DesiredState: storage_v1alpha.DV_PRESENT, - ActualState: storage_v1alpha.DV_READY, - ImagePath: imagePath, - NodeId: nodeId, - } + Name: name, + DiskId: diskEntityId, + VolumeId: volId, + SizeGb: sizeGb, + Filesystem: filesystem, + VolumeMode: DetectVolumeMode(), - // Transition the disk to PROVISIONED before creating the - // disk_volume. These are two independent, non-transactional - // writes, so the order matters: if the disk_volume were - // created first and the Patch then failed, the deferred - // Cleanup would orphan the committed disk_volume by deleting - // its parent disk out from under it. Patching first means a - // Create failure leaves no disk_volume behind, and a surviving - // PROVISIONED disk drives the existing DELETING-based cleanup - // and self-healing paths. - _, err := r.eac.Patch(fctx, []entity.Attr{ - entity.Ref(entity.DBId, diskEntityId), - entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisionedId), - entity.String(storage_v1alpha.DiskVolumeIdId, volId), - }, 0) - if err != nil { - return fmt.Errorf("updating disk to provisioned: %w", err) + DesiredState: storage_v1alpha.DV_PRESENT, + // Start PENDING, not READY, and let the DiskVolumeController + // drive it the rest of the way. The image is already written + // and formatted, so it mounts what was restored rather than + // reimaging. + ActualState: storage_v1alpha.DV_PENDING, + ImagePath: imagePath, + NodeId: nodeId, } - _, err = r.eac.Create(fctx, entity.New( + // The volume goes in first, and the disk moves to PROVISIONING + // rather than PROVISIONED. Both halves of that matter, and the + // reason is a window rather than a preference. + // + // DiskController ignores a RESTORING disk, so nothing watches + // this disk until the patch below. Announcing PROVISIONED first + // would end that truce while there was still no disk_volume to + // find, and handleProvisioned answers a provisioned disk with no + // volume by provisioning a blank one. The restore's own volume + // then lands beside it: two DV_READY volumes on one disk, both + // mounted, and the next backup fails on the ambiguity. + // + // This way the volume is already there when the disk becomes + // visible, and the controller promotes it to PROVISIONED once the + // volume reports ready. It is the same handshake a recovered disk + // uses, so there is one path through this transition instead of + // two. + _, err := r.eac.Create(fctx, entity.New( entity.DBId, entity.Id("disk_volume/"+volId), vol.Encode, ).Attrs()) @@ -218,6 +219,18 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt return fmt.Errorf("creating disk_volume entity: %w", err) } + // A failure here leaves a RESTORING disk with a committed volume, + // which Cleanup handles: it marks the disk DELETING, and the + // controller tears the volume down from there. + _, err = r.eac.Patch(fctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusProvisioningId), + entity.String(storage_v1alpha.DiskVolumeIdId, volId), + }, 0) + if err != nil { + return fmt.Errorf("updating disk to provisioning: %w", err) + } + return nil }, }, nil @@ -291,7 +304,10 @@ func diskSizeGb(sizeBytes int64) (int64, error) { switch { case sizeBytes < 0: return 0, fmt.Errorf("snapshot reports a negative image size (%d bytes)", sizeBytes) - case sizeBytes > math.MaxInt64-gib: + case sizeBytes > math.MaxInt64-gib+1: + // The ceiling is the largest size whose rounded-up capacity still + // converts back to bytes without overflowing, which is the whole GiB + // just below the top of the range. return 0, fmt.Errorf("snapshot reports an image size too large to be real (%d bytes)", sizeBytes) } diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index 5333265e7..f5d2493b5 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -147,8 +147,8 @@ func getTestDisk(t *testing.T, ctx context.Context, eac *entityserver_v1alpha.En return d } -// TestCreateDiskAndVolume_FinalizeSuccess is the happy path: Finalize patches -// the disk to PROVISIONED and then creates the disk_volume, leaving the disk +// TestCreateDiskAndVolume_FinalizeSuccess is the happy path: Finalize creates +// the disk_volume and then moves the disk to PROVISIONING, leaving the disk // carrying its VolumeId and exactly one disk_volume referencing it. func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { ctx := t.Context() @@ -169,7 +169,10 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { require.NoError(t, target.Finalize(ctx)) disk := getTestDisk(t, ctx, es.EAC, diskID) - assert.Equal(t, storage_v1alpha.PROVISIONED, disk.Status) + // PROVISIONING, not PROVISIONED. DiskController promotes it once the volume + // reports ready, so the restore never hands over a disk whose volume the + // controller would go and invent for itself. + assert.Equal(t, storage_v1alpha.PROVISIONING, disk.Status) assert.NotEmpty(t, disk.VolumeId) vols := volumesForDisk(t, ctx, es.EAC, diskID) @@ -178,7 +181,7 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { assert.Equal(t, diskID, vol.DiskId) assert.Equal(t, disk.VolumeId, vol.VolumeId) assert.Equal(t, storage_v1alpha.DV_PRESENT, vol.DesiredState) - assert.Equal(t, storage_v1alpha.DV_READY, vol.ActualState) + assert.Equal(t, storage_v1alpha.DV_PENDING, vol.ActualState) assert.Equal(t, target.ImagePath, vol.ImagePath) // disk_volume.NodeId must match the coordinator node the resolver found. @@ -188,10 +191,10 @@ func TestCreateDiskAndVolume_FinalizeSuccess(t *testing.T) { assert.Equal(t, nodes.Values()[0].Entity().Id(), vol.NodeId) } -// TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan is the core bug -// repro, inverted: with the reordered Finalize (Patch before Create), failing -// the disk_volume Create leaves NO disk_volume committed. Under the old -// Create-first order this is exactly the window that orphaned a disk_volume. +// TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan covers the first of +// Finalize's two writes failing. The Create is what commits a volume, so its +// failure leaves nothing behind and the disk is still RESTORING, which the disk +// controller ignores. func TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan(t *testing.T) { ctx := t.Context() fault := newFaultRPC(nil, "create", 1, fmt.Errorf("simulated disk_volume create failure")) @@ -208,17 +211,15 @@ func TestCreateDiskAndVolume_FinalizeCreateFailsLeavesNoOrphan(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "creating disk_volume entity") - // No disk_volume entity was committed: the Create that would have committed - // it ran last and failed. This is the invariant the reorder restores. + // Nothing was committed: the Create is the first write now, so its failure + // leaves no volume at all. assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC), "no disk_volume must be left behind") assert.Empty(t, volumesForDisk(t, ctx, es.EAC, diskID)) - // The disk survived and reached PROVISIONED (the Patch ran first), so it can - // drive self-healing / DELETING-based cleanup instead of being deleted out - // from under a volume the coordinator may already be reconciling. + // The disk is still RESTORING, which DiskController ignores, so a restore + // that fell over here has not handed anything half-built to the controller. disk := getTestDisk(t, ctx, es.EAC, diskID) - assert.Equal(t, storage_v1alpha.PROVISIONED, disk.Status) - assert.NotEmpty(t, disk.VolumeId) + assert.Equal(t, storage_v1alpha.RESTORING, disk.Status) // Cleanup keeps the disk alive by transitioning it to DELETING, not by // deleting it outright; the disk controller's DELETING path then drives @@ -248,23 +249,57 @@ func TestCreateDiskAndVolume_FinalizePatchFailsLeavesNoOrphan(t *testing.T) { err = target.Finalize(ctx) require.Error(t, err) - assert.Contains(t, err.Error(), "updating disk to provisioned") + assert.Contains(t, err.Error(), "updating disk to provisioning") - // Patch is now the first write, so a Patch failure cannot have left a - // disk_volume behind. - assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC)) - assert.Empty(t, volumesForDisk(t, ctx, es.EAC, diskID)) + // The volume is created first now, so a Patch failure does leave one + // behind. That is not an orphan: an entity is only orphaned when nothing + // owns its teardown, and this one is still attached to a disk. + vols := volumesForDisk(t, ctx, es.EAC, diskID) + require.Len(t, vols, 1) - // Disk still exists, still RESTORING (Patch failed, Create never ran). + // The disk never advertised itself, so nothing else has acted on it. disk := getTestDisk(t, ctx, es.EAC, diskID) assert.Equal(t, storage_v1alpha.RESTORING, disk.Status) // Cleanup's Patch is the 2nd patch call; only the 1st (Finalize's) fails, - // so cleanup still transitions the disk to DELETING. + // so cleanup still transitions the disk to DELETING. That is what hands + // the volume to the controller's teardown path, which is the only writer + // of desired_state=DV_ABSENT. require.NoError(t, target.Cleanup(ctx)) disk = getTestDisk(t, ctx, es.EAC, diskID) assert.Equal(t, storage_v1alpha.DELETING, disk.Status) - assert.Empty(t, allTestDiskVolumes(t, ctx, es.EAC)) + assert.Len(t, volumesForDisk(t, ctx, es.EAC, diskID), 1, + "the volume stays attached to the deleting disk, which is what reaps it") +} + +// The restore hands the disk over already carrying its volume, and as +// PROVISIONING rather than PROVISIONED. +// +// Both halves close the same window. DiskController answers a PROVISIONED disk +// with no disk_volume by provisioning a blank one, so announcing the disk +// before its volume exists produces a second volume beside the restored one: +// two ready volumes on one disk, both mounted, and every later lookup by disk +// ambiguous. +func TestCreateDiskAndVolume_FinalizeHandsOverAVolumeItAlreadyOwns(t *testing.T) { + ctx := t.Context() + es, resolver := setupResolver(t, nil) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) + require.NoError(t, err) + require.NoError(t, target.Finalize(ctx)) + + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + + // PROVISIONING, not PROVISIONED: the controller promotes it once the + // volume reports ready, so there is one path through this transition. + assert.Equal(t, storage_v1alpha.PROVISIONING, disks[0].Status) + + vols := volumesForDisk(t, ctx, es.EAC, disks[0].ID) + require.Len(t, vols, 1, "a restore must leave exactly one volume on the disk") + assert.Equal(t, storage_v1alpha.DV_PENDING, vols[0].ActualState) + assert.Equal(t, storage_v1alpha.DV_PRESENT, vols[0].DesiredState) + assert.Equal(t, disks[0].VolumeId, vols[0].VolumeId) } // TestCreateDiskAndVolume_CleanupDoesNotHardDeleteDisk pins the key behavioral @@ -420,7 +455,7 @@ func TestDiskSizeGbRejectsSizesItCannotRepresent(t *testing.T) { {"negative", -1}, {"very negative", math.MinInt64}, {"large enough to overflow the rounding", math.MaxInt64}, - {"just past the representable ceiling", math.MaxInt64 - (1 << 30) + 1}, + {"just past the representable ceiling", math.MaxInt64 - (1 << 30) + 2}, } { t.Run(tc.name, func(t *testing.T) { _, err := diskSizeGb(tc.sizeBytes) @@ -432,7 +467,9 @@ func TestDiskSizeGbRejectsSizesItCannotRepresent(t *testing.T) { // And the largest size it does accept round-trips back to bytes without // overflowing, which is what the disk volume controller does with it. func TestDiskSizeGbCeilingRoundTrips(t *testing.T) { - sizeGb, err := diskSizeGb(math.MaxInt64 - (1 << 30)) + // The ceiling itself: exactly divisible by a GiB, so it rounds to a + // capacity that converts straight back to the same byte count. + sizeGb, err := diskSizeGb(math.MaxInt64 - (1 << 30) + 1) require.NoError(t, err) assert.Positive(t, sizeGb) assert.Positive(t, sizeGb*(1<<30)/(1<<30), "the byte conversion must not overflow") diff --git a/servers/disk/restore.go b/servers/disk/restore.go index 167f079ca..6e9bf2e46 100644 --- a/servers/disk/restore.go +++ b/servers/disk/restore.go @@ -30,11 +30,23 @@ func (s *Server) Restore(ctx context.Context, state *disk_v1alpha.DiskBackupRest return refuse("disk name is required") } - // An uploaded snapshot lives in a file keyed by transfer id, and it stays - // there until the image is installed. Hold the id for the whole handler, so - // an overlapping call cannot append into the same file or pull it out from - // under the install. A restore point needs none of this: it comes from the - // cloud and touches no transfer file. + // Two locks, because two different things are shared here. + // + // The disk name covers the entity work and the image path. A restore + // creates the disk when it does not exist, and two restores racing to the + // same new name both find it missing and both create one, or one catches + // the other's disk halfway built and fails looking for a volume that is not + // there yet. They also write the same .restore.tmp. Held for the + // whole handler, since the name is not free again until the image is + // installed. + releaseName := s.names.acquire(name) + defer releaseName() + + // The transfer id covers the uploaded snapshot, which lives in a file of + // its own and stays there until the image is installed. Different clients + // restoring to the same name bring different transfer ids, so this does not + // duplicate the lock above. A restore point needs none of it: it comes from + // the cloud and touches no transfer file. if id := args.TransferId(); id != "" && args.RestorePoint() == "" { release := s.transfers.acquire(id) defer release() From ea485dd13a7553401cca0faf0ef6f4207c956a2a Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 10 Sep 2026 13:18:42 -0700 Subject: [PATCH 17/19] Leave a restored image to the controller once a volume owns it Last round's cleanup reorder did not do what its comment claimed. Marking the disk DELETING only starts the teardown; the controller unmounts, detaches and soft-deletes the volume some time later. Removing the image on the very next line still races that, and losing the race unlinks a file whose loop device is attached, which is the stranded-inode case the reorder was supposed to avoid. Waiting for the teardown would mean polling the entity store from an error path that is already handling a failure. Instead Finalize records that it committed the volume, and cleanup removes the image only when it did not. That is the common case anyway, since most cleanups run before Finalize, and it is exactly the case where nothing ever opened the image. Also unwind a disk abandoned between its own creation and the node lookup. CreateDiskAndVolume commits the disk first, and a lookup failure returned an error with no RestoreTarget, so the caller had no cleanup to call. The disk sat in RESTORING, and the next restore of that name found it, skipped creating one, and failed looking for the volume it never got. And split the restore warning in the addon docs so the admonition carries only the refusal, with the release limitations and the recommended path as prose, per the docs guide's one-concept rule. --- docs/docs/addons.md | 12 +++-- pkg/diskresolve/resolver.go | 67 +++++++++++++++++++++++----- pkg/diskresolve/resolver_test.go | 75 ++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 19 deletions(-) diff --git a/docs/docs/addons.md b/docs/docs/addons.md index a02d2e90a..5bbbb8249 100644 --- a/docs/docs/addons.md +++ b/docs/docs/addons.md @@ -439,15 +439,13 @@ miren disk restore -s myapp-db.miren.zst --force ``` -:::warning[Restore into a new disk, not over the running one] -Restoring on top of a disk your app is using is not supported today, and the command refuses it rather than pretending. A disk in use is held open by the kernel, so writing a new image into it would leave the running database on the old data while reporting success. - -There is currently no way to release a disk while keeping the app configured for it. Disk-backed services run at fixed concurrency, `num_instances = 0` is rejected, and scaling a pool to zero by hand is reconciled straight back to one. Even in that brief window the disk stays mounted. - -So the working recovery is to restore into a **new** disk and move the app across. +:::warning[Restoring over a disk in use is refused] +Restoring on top of a disk your app is using is not supported today, and the command stops rather than pretending. A disk in use is held open by the kernel, so writing a new image into it would leave the running database on the old data while reporting success. ::: -Restore into a new disk name: +There is currently no way to release a disk while keeping the app configured for it, so there is no sequence that makes an in-place restore work. Disk-backed services run at fixed concurrency, `num_instances = 0` is rejected, and scaling a pool to zero by hand is reconciled straight back to one. Even during that brief window the disk stays mounted. + +The working recovery is to restore into a **new** disk and move the app across. Restore into a new disk name: ```miren diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index 54dc43631..f688b15ec 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -120,9 +120,24 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt nodeId, err := r.FindNodeId(ctx) if err != nil { + // The disk is already committed but the caller is about to get an + // error and no RestoreTarget, so it has no Cleanup to call. Unwind it + // here or it sits in RESTORING forever, and the next restore of the + // same name finds that disk, skips creating one, and fails looking for + // the volume it never got. + if uerr := r.abandonDisk(ctx, diskEntityId); uerr != nil { + return nil, fmt.Errorf( + "finding node: %w (the half-created disk could not be unwound either, so %s is stuck in restoring: %v)", + err, name, uerr) + } return nil, fmt.Errorf("finding node: %w", err) } + // Set once Finalize has committed the disk_volume, which is the moment the + // image stops being ours to delete. Finalize and Cleanup are called in turn + // by the restore handler on one goroutine, so a plain bool is enough. + var volumeCommitted bool + return &snapshot.RestoreTarget{ Name: name, ImagePath: imagePath, @@ -146,13 +161,6 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt // DiskVolumeController. Idempotent: patching an already-DELETING // disk is a no-op. // - // This runs before the image is touched, because Finalize - // creates the disk_volume first now: cleanup after a partial - // Finalize can find a volume the controller is already mounting, - // and pulling the image out from under a loop device leaves it - // holding an unlinked inode rather than releasing it. Letting - // the teardown path unmount and detach first is the only order - // that ends with nothing held. _, err := r.eac.Patch(cctx, []entity.Attr{ entity.Ref(entity.DBId, diskEntityId), entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), @@ -161,11 +169,27 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt return fmt.Errorf("transitioning disk to deleting during cleanup: %w", err) } - // Whatever the teardown did not claim. Usually this is the whole - // job, since most cleanups run before Finalize and so before any - // volume exists. Tolerate "not present": the restore may never - // have renamed the image into place, and a volume teardown may - // have moved the directory out from under it. + // Once the disk_volume exists, the image belongs to the teardown + // above and not to us. That teardown is asynchronous: the patch + // only marks the disk, and the controller unmounts, detaches and + // soft-deletes the volume directory some time later. Removing the + // image here would race it, and losing that race unlinks a file + // whose loop device is still attached, which leaves the kernel + // holding an inode nobody can reach instead of releasing it. + // + // Waiting for the teardown instead would mean polling the entity + // store from an error path that is already handling a failure. + // Letting the controller finish the job it already does is both + // simpler and the thing that cannot race. + if volumeCommitted { + return nil + } + + // No volume was ever created, which is the common case: most + // cleanups run before Finalize. Nothing has opened this image, so + // there is no teardown to wait for and no loop device to strand. + // Tolerate "not present" — the restore may never have renamed the + // image into place. if imageErr := os.Remove(imagePath); imageErr != nil && !os.IsNotExist(imageErr) { // Reported only once the authoritative step is done, so a // failure to reclaim the image never costs us the disk @@ -218,6 +242,9 @@ func (r *Resolver) CreateDiskAndVolume(ctx context.Context, name string, sizeByt if err != nil { return fmt.Errorf("creating disk_volume entity: %w", err) } + // From here the volume owns the image, so Cleanup must leave it to + // the controller's teardown rather than unlinking it itself. + volumeCommitted = true // A failure here leaves a RESTORING disk with a committed volume, // which Cleanup handles: it marks the disk DELETING, and the @@ -286,6 +313,22 @@ func (r *Resolver) FindLeases(ctx context.Context, diskID string) ([]snapshot.Le return leases, nil } +// abandonDisk marks a disk DELETING when creation gave up partway through. +// +// It is the same unwind Cleanup performs, for the window before there is a +// RestoreTarget to hang a Cleanup on. +func (r *Resolver) abandonDisk(ctx context.Context, diskEntityId entity.Id) error { + // The caller's context may already be cancelled, and unwinding is exactly + // what still has to happen when it is. + ctx = context.WithoutCancel(ctx) + + _, err := r.eac.Patch(ctx, []entity.Attr{ + entity.Ref(entity.DBId, diskEntityId), + entity.Ref(storage_v1alpha.DiskStatusId, storage_v1alpha.DiskStatusDeletingId), + }, 0) + return err +} + // gib is the unit disks are sized in. const gib = 1 << 30 diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index f5d2493b5..4396a00b6 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -474,3 +474,78 @@ func TestDiskSizeGbCeilingRoundTrips(t *testing.T) { assert.Positive(t, sizeGb) assert.Positive(t, sizeGb*(1<<30)/(1<<30), "the byte conversion must not overflow") } + +// Once Finalize has committed the disk_volume, the image belongs to the +// controller's teardown, not to Cleanup. +// +// Marking the disk DELETING only starts that teardown; the unmount and detach +// happen later. Removing the image here would race them, and losing the race +// unlinks a file whose loop device is still attached, leaving the kernel +// holding an inode nothing can reach. +func TestCleanupLeavesTheImageToTeardownOnceAVolumeExists(t *testing.T) { + ctx := t.Context() + es, resolver := setupResolver(t, nil) + + dataPath := t.TempDir() + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", dataPath) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(target.ImagePath), 0700)) + require.NoError(t, os.WriteFile(target.ImagePath, []byte("restored"), 0600)) + + // Finalize commits the volume, so the image is now the teardown's business. + require.NoError(t, target.Finalize(ctx)) + require.NoError(t, target.Cleanup(ctx)) + + assert.FileExists(t, target.ImagePath, + "cleanup must not unlink an image the controller is still tearing down") + + // The disk is handed to the teardown path, which is what reclaims both. + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, storage_v1alpha.DELETING, disks[0].Status) +} + +// The common case is the opposite one: most cleanups run before Finalize, so no +// volume exists, nothing ever opened the image, and there is no teardown to +// wait for. Leaving it would just waste the space. +func TestCleanupRemovesTheImageWhenNoVolumeWasCommitted(t *testing.T) { + ctx := t.Context() + _, resolver := setupResolver(t, nil) + + dataPath := t.TempDir() + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", dataPath) + require.NoError(t, err) + + require.NoError(t, os.MkdirAll(filepath.Dir(target.ImagePath), 0700)) + require.NoError(t, os.WriteFile(target.ImagePath, []byte("restored"), 0600)) + + // No Finalize, so no volume. + require.NoError(t, target.Cleanup(ctx)) + + _, statErr := os.Stat(target.ImagePath) + assert.True(t, os.IsNotExist(statErr), + "with no volume to tear down, cleanup owns the image") +} + +// A disk is committed before the node lookup, and a lookup failure returns no +// RestoreTarget, so the caller has no Cleanup to call. Left alone the disk sits +// in RESTORING and the next restore of the same name finds it, skips creating +// one, and dies looking for the volume it never got. +func TestCreateDiskAndVolumeUnwindsWhenTheNodeLookupFails(t *testing.T) { + ctx := t.Context() + // FindNodeId lists nodes, and that is the first list this path makes. + fault := newFaultRPC(nil, "list", 1, fmt.Errorf("simulated node lookup failure")) + es, resolver := setupResolver(t, fault) + + target, err := resolver.CreateDiskAndVolume(ctx, "mydisk", 2<<30, "ext4", t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "finding node") + assert.Nil(t, target, "a failed creation must not hand back a target") + + // The disk it had already committed is on its way out rather than stuck. + disks := listTestDisks(t, ctx, es.EAC) + require.Len(t, disks, 1) + assert.Equal(t, storage_v1alpha.DELETING, disks[0].Status, + "a half-created disk must not be left in restoring, which blocks every same-name retry") +} From c810790ff1056766b3de21ef9f76f1be3f192ce2 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 05:15:08 +0000 Subject: [PATCH 18/19] Reclaim untracked pending restore images during volume deletion --- components/diskio/disk_volume_controller.go | 26 +++++++- .../diskio/disk_volume_controller_test.go | 62 +++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/components/diskio/disk_volume_controller.go b/components/diskio/disk_volume_controller.go index 37fa372ab..b35036e72 100644 --- a/components/diskio/disk_volume_controller.go +++ b/components/diskio/disk_volume_controller.go @@ -421,13 +421,35 @@ func (c *DiskVolumeController) deleteVolume(ctx context.Context, volume *storage c.log.Info("deleting disk volume", "entity_id", entityId) + volState := c.state.GetVolume(entityId) + if volState == nil { + c.log.Warn("volume not found in state", "entity_id", entityId) + // A restore can put its image in place before the volume's first + // reconcile. If deletion wins that race, no local state owns the + // directory, so reclaim it here rather than orphaning the image. + if volume.ActualState == storage_v1alpha.DV_PENDING { + volumePath := c.getVolumePath(entityId) + if c.ops.VolumePathExists(volumePath) { + imagePath := filepath.Join(volumePath, "disk.img") + dev, err := c.mntOps.FindLoopByBacking(imagePath) + if err != nil { + return fmt.Errorf("checking pending volume backing: %w", err) + } + if dev != "" { + return fmt.Errorf("pending volume image %s is still attached to %s", imagePath, dev) + } + if err := c.ops.RemoveVolumeDir(volumePath); err != nil { + return fmt.Errorf("removing untracked pending volume directory %s: %w", volumePath, err) + } + } + } + } + if err := c.updateVolumeState(ctx, volume.ID, storage_v1alpha.DV_DELETING, "", ""); err != nil { c.log.Warn("failed to update volume state to deleting", "error", err) } - volState := c.state.GetVolume(entityId) if volState == nil { - c.log.Warn("volume not found in state", "entity_id", entityId) if err := c.updateVolumeState(ctx, volume.ID, storage_v1alpha.DV_DELETED, "", ""); err != nil { c.log.Warn("failed to update volume state to deleted", "error", err) } diff --git a/components/diskio/disk_volume_controller_test.go b/components/diskio/disk_volume_controller_test.go index 3e266bc9e..65e443f89 100644 --- a/components/diskio/disk_volume_controller_test.go +++ b/components/diskio/disk_volume_controller_test.go @@ -560,6 +560,68 @@ func TestDiskVolumeControllerDeleteNotInState(t *testing.T) { assert.Equal(t, storage_v1alpha.DV_DELETED, updated.ActualState) } +func TestDiskVolumeControllerDeleteUntrackedPendingImage(t *testing.T) { + for _, tc := range []struct { + name string + attached bool + removeErr bool + }{ + {name: "unattached"}, + {name: "attached", attached: true}, + {name: "remove failure", removeErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + state := NewState() + ops := newMockDiskVolumeOps() + mntOps := newMockDiskMountOps() + volumePath := filepath.Join(dataPath, "volumes", "vol-pending") + imagePath := filepath.Join(volumePath, "disk.img") + ops.existingPaths[volumePath] = true + if tc.attached { + mntOps.loopBacking = make(map[string]string) + mntOps.loopBacking[imagePath] = "/dev/loop7" + } + if tc.removeErr { + ops.removeDirErr = errors.New("disk I/O failure") + } + + vc := NewDiskVolumeController(testutils.TestLogger(t), dataPath, compute.NewNodeId("test-node-1"), state, ops, mntOps) + vc.SetEAC(es.EAC) + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-pending", + NodeId: compute.NewNodeId("test-node-1").Id(), + DesiredState: storage_v1alpha.DV_ABSENT, + ActualState: storage_v1alpha.DV_PENDING, + } + createDiskVolumeEntity(ctx, t, es, vol) + + err := vc.reconcileVolume(ctx, vol) + resp, getErr := es.EAC.Get(ctx, string(vol.ID)) + require.NoError(t, getErr) + var updated storage_v1alpha.DiskVolume + updated.Decode(resp.Entity().Entity()) + if tc.attached { + require.ErrorContains(t, err, "still attached") + assert.Empty(t, ops.removedDirs) + assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) + } else if tc.removeErr { + require.ErrorContains(t, err, "disk I/O failure") + assert.Empty(t, ops.removedDirs) + assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) + } else { + require.NoError(t, err) + assert.Equal(t, []string{volumePath}, ops.removedDirs) + assert.Equal(t, storage_v1alpha.DV_DELETED, updated.ActualState) + } + }) + } +} + func TestDiskVolumeControllerUniversalMountAtCreation(t *testing.T) { ctx := t.Context() log := testutils.TestLogger(t) From 453c0cae4ce02869156c9b430b9167fbee68c294 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 16:18:54 +0000 Subject: [PATCH 19/19] Preserve untracked pending volumes through soft deletion --- components/diskio/disk_volume_controller.go | 12 ++-- .../diskio/disk_volume_controller_test.go | 62 ++++++++++++++++--- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/components/diskio/disk_volume_controller.go b/components/diskio/disk_volume_controller.go index b35036e72..2745efa4f 100644 --- a/components/diskio/disk_volume_controller.go +++ b/components/diskio/disk_volume_controller.go @@ -424,9 +424,8 @@ func (c *DiskVolumeController) deleteVolume(ctx context.Context, volume *storage volState := c.state.GetVolume(entityId) if volState == nil { c.log.Warn("volume not found in state", "entity_id", entityId) - // A restore can put its image in place before the volume's first - // reconcile. If deletion wins that race, no local state owns the - // directory, so reclaim it here rather than orphaning the image. + // Restore and undelete can put an image in place before the volume's + // first reconcile. Keep the same recovery window as a tracked volume. if volume.ActualState == storage_v1alpha.DV_PENDING { volumePath := c.getVolumePath(entityId) if c.ops.VolumePathExists(volumePath) { @@ -438,8 +437,11 @@ func (c *DiskVolumeController) deleteVolume(ctx context.Context, volume *storage if dev != "" { return fmt.Errorf("pending volume image %s is still attached to %s", imagePath, dev) } - if err := c.ops.RemoveVolumeDir(volumePath); err != nil { - return fmt.Errorf("removing untracked pending volume directory %s: %w", volumePath, err) + pending := &VolumeState{EntityId: entityId, VolumeId: localVolumeID(volume), DiskPath: volumePath} + if err := c.softDeleteVolume(ctx, volume, pending); err != nil { + // Without local state this may be the only copy of an undeleted + // disk. Never fall back to hard deletion if the move fails. + return fmt.Errorf("soft-deleting untracked pending volume %s: %w", volumePath, err) } } } diff --git a/components/diskio/disk_volume_controller_test.go b/components/diskio/disk_volume_controller_test.go index 65e443f89..420cd4d58 100644 --- a/components/diskio/disk_volume_controller_test.go +++ b/components/diskio/disk_volume_controller_test.go @@ -562,13 +562,13 @@ func TestDiskVolumeControllerDeleteNotInState(t *testing.T) { func TestDiskVolumeControllerDeleteUntrackedPendingImage(t *testing.T) { for _, tc := range []struct { - name string - attached bool - removeErr bool + name string + attached bool + moveErr bool }{ {name: "unattached"}, {name: "attached", attached: true}, - {name: "remove failure", removeErr: true}, + {name: "move failure", moveErr: true}, } { t.Run(tc.name, func(t *testing.T) { ctx := t.Context() @@ -581,13 +581,15 @@ func TestDiskVolumeControllerDeleteUntrackedPendingImage(t *testing.T) { mntOps := newMockDiskMountOps() volumePath := filepath.Join(dataPath, "volumes", "vol-pending") imagePath := filepath.Join(volumePath, "disk.img") + require.NoError(t, os.MkdirAll(volumePath, 0755)) + require.NoError(t, os.WriteFile(imagePath, []byte("recovered data"), 0600)) ops.existingPaths[volumePath] = true if tc.attached { mntOps.loopBacking = make(map[string]string) mntOps.loopBacking[imagePath] = "/dev/loop7" } - if tc.removeErr { - ops.removeDirErr = errors.New("disk I/O failure") + if tc.moveErr { + ops.moveDirErr = errors.New("disk I/O failure") } vc := NewDiskVolumeController(testutils.TestLogger(t), dataPath, compute.NewNodeId("test-node-1"), state, ops, mntOps) @@ -608,20 +610,64 @@ func TestDiskVolumeControllerDeleteUntrackedPendingImage(t *testing.T) { if tc.attached { require.ErrorContains(t, err, "still attached") assert.Empty(t, ops.removedDirs) + assert.Empty(t, ops.movedDirs) assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) - } else if tc.removeErr { + } else if tc.moveErr { require.ErrorContains(t, err, "disk I/O failure") assert.Empty(t, ops.removedDirs) + assert.Empty(t, ops.movedDirs) assert.Equal(t, storage_v1alpha.DV_PENDING, updated.ActualState) + data, readErr := os.ReadFile(imagePath) + require.NoError(t, readErr) + assert.Equal(t, "recovered data", string(data)) } else { require.NoError(t, err) - assert.Equal(t, []string{volumePath}, ops.removedDirs) + assert.Empty(t, ops.removedDirs) + require.Len(t, ops.movedDirs, 1) + assert.Equal(t, volumePath, ops.movedDirs[0].src) + assert.Equal(t, filepath.Join(dataPath, "deleted-volumes", "vol-pending"), ops.movedDirs[0].dst) assert.Equal(t, storage_v1alpha.DV_DELETED, updated.ActualState) } }) } } +func TestDiskVolumeControllerSoftDeletesUntrackedUndelete(t *testing.T) { + ctx := t.Context() + es, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + dataPath := t.TempDir() + volumePath := filepath.Join(dataPath, "volumes", "vol-recovered") + require.NoError(t, os.MkdirAll(volumePath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(volumePath, "disk.img"), []byte("recovered data"), 0600)) + + log := testutils.TestLogger(t) + vc := NewDiskVolumeController(log, dataPath, compute.NewNodeId("test-node-1"), NewState(), NewRealDiskVolumeOps(log), newMockDiskMountOps()) + vc.SetEAC(es.EAC) + vol := &storage_v1alpha.DiskVolume{ + ID: "disk_volume/vol-recovered", + Name: "recovered", + VolumeId: "vol-recovered", + NodeId: compute.NewNodeId("test-node-1").Id(), + DesiredState: storage_v1alpha.DV_ABSENT, + ActualState: storage_v1alpha.DV_PENDING, + } + createDiskVolumeEntity(ctx, t, es, vol) + require.NoError(t, vc.reconcileVolume(ctx, vol)) + + deletedPath := filepath.Join(dataPath, "deleted-volumes", "vol-recovered") + data, err := os.ReadFile(filepath.Join(deletedPath, "disk.img")) + require.NoError(t, err) + assert.Equal(t, "recovered data", string(data)) + meta, err := LoadDeletedVolumeMetadata(deletedPath) + require.NoError(t, err) + assert.Equal(t, "recovered", meta.DiskName) + assert.Equal(t, "vol-recovered", meta.VolumeID) + _, err = os.Stat(volumePath) + assert.True(t, os.IsNotExist(err)) +} + func TestDiskVolumeControllerUniversalMountAtCreation(t *testing.T) { ctx := t.Context() log := testutils.TestLogger(t)