diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..4a2a89003 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +third_party/** linguist-vendored diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d482a0b6..bc15ab52a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,6 +82,9 @@ jobs: with: version: v2.12.2 + - name: Check the vendored lbd source matches go.mod + run: ./hack/sync-lbd-src.sh --check + - name: Check go.mod tidiness run: | # Run go mod tidy and check if there are any changes diff --git a/api/core/core_v1alpha/extra.go b/api/core/core_v1alpha/extra.go index de5c915ce..19e818523 100644 --- a/api/core/core_v1alpha/extra.go +++ b/api/core/core_v1alpha/extra.go @@ -1,6 +1,8 @@ package core_v1alpha import ( + "strings" + entity "miren.dev/runtime/pkg/entity" ) @@ -9,3 +11,22 @@ func MD(ea entity.AttrGetter) Metadata { md.Decode(ea) return md } + +// SystemArtifactPrefix marks an image miren pushes to the cluster registry for +// its own use rather than on behalf of an app. +// +// An artifact's entity name is the tag it was pushed under (see the registry's +// putManifest), so the prefix rides in the tag and needs no schema field. It +// exists because artifact GC archives everything no AppVersion references, and +// a system image belongs to no app: without a way to tell it apart from a +// genuinely orphaned artifact, it would be collected within the hour and its +// blobs deleted underneath the nodes still pulling it. +const SystemArtifactPrefix = "miren-system-" + +// IsSystemArtifact reports whether an artifact is one miren pushed for itself, +// and so must survive garbage collection even though no AppVersion points at +// it. +func IsSystemArtifact(id entity.Id) bool { + name := strings.TrimPrefix(string(id), "artifact/") + return strings.HasPrefix(name, SystemArtifactPrefix) +} diff --git a/api/nodeadmin/nodeadmin.go b/api/nodeadmin/nodeadmin.go new file mode 100644 index 000000000..936a432ae --- /dev/null +++ b/api/nodeadmin/nodeadmin.go @@ -0,0 +1,6 @@ +// Package nodeadmin carries the RPC the coordinator uses to ask one node to +// change something about itself. +package nodeadmin + +//go:generate mkdir -p nodeadmin_v1alpha +//go:generate go run ../../pkg/rpc/cmd/rpcgen -pkg nodeadmin_v1alpha -input rpc.yml -output nodeadmin_v1alpha/rpc.gen.go diff --git a/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go b/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go new file mode 100644 index 000000000..c0c6c3b1e --- /dev/null +++ b/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go @@ -0,0 +1,220 @@ +package nodeadmin_v1alpha + +import ( + "context" + "encoding/json" + + "github.com/fxamacker/cbor/v2" + rpc "miren.dev/runtime/pkg/rpc" +) + +type nodeAdminInstallDiskAcceleratorArgsData struct { + Image *string `cbor:"0,keyasint,omitempty" json:"image,omitempty"` + Force *bool `cbor:"1,keyasint,omitempty" json:"force,omitempty"` +} + +type NodeAdminInstallDiskAcceleratorArgs struct { + call rpc.Call + data nodeAdminInstallDiskAcceleratorArgsData +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) HasImage() bool { + return v.data.Image != nil +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) Image() string { + if v.data.Image == nil { + return "" + } + return *v.data.Image +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type nodeAdminInstallDiskAcceleratorResultsData struct { + KernelRelease *string `cbor:"0,keyasint,omitempty" json:"kernel_release,omitempty"` + LbdVersion *string `cbor:"1,keyasint,omitempty" json:"lbd_version,omitempty"` + Error *string `cbor:"2,keyasint,omitempty" json:"error,omitempty"` +} + +type NodeAdminInstallDiskAcceleratorResults struct { + call rpc.Call + data nodeAdminInstallDiskAcceleratorResultsData +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetKernelRelease(kernel_release string) { + v.data.KernelRelease = &kernel_release +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetLbdVersion(lbd_version string) { + v.data.LbdVersion = &lbd_version +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetError(error string) { + v.data.Error = &error +} + +func (v *NodeAdminInstallDiskAcceleratorResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type NodeAdminInstallDiskAccelerator struct { + rpc.Call + args NodeAdminInstallDiskAcceleratorArgs + results NodeAdminInstallDiskAcceleratorResults +} + +func (t *NodeAdminInstallDiskAccelerator) Args() *NodeAdminInstallDiskAcceleratorArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *NodeAdminInstallDiskAccelerator) Results() *NodeAdminInstallDiskAcceleratorResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type NodeAdmin interface { + InstallDiskAccelerator(ctx context.Context, state *NodeAdminInstallDiskAccelerator) error +} + +type reexportNodeAdmin struct { + client rpc.Client +} + +func (reexportNodeAdmin) InstallDiskAccelerator(ctx context.Context, state *NodeAdminInstallDiskAccelerator) error { + panic("not implemented") +} + +func (t reexportNodeAdmin) CapabilityClient() rpc.Client { + return t.client +} + +func AdaptNodeAdmin(t NodeAdmin) *rpc.Interface { + methods := []rpc.Method{ + { + Name: "install_disk_accelerator", + InterfaceName: "NodeAdmin", + Index: 0, + Public: false, + Params: []string{"image", "force"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.InstallDiskAccelerator(ctx, &NodeAdminInstallDiskAccelerator{Call: call}) + }, + }, + } + + return rpc.NewInterface(methods, t) +} + +type NodeAdminClient struct { + rpc.Client +} + +func NewNodeAdminClient(client rpc.Client) *NodeAdminClient { + return &NodeAdminClient{Client: client} +} + +func (c NodeAdminClient) Export() NodeAdmin { + return reexportNodeAdmin{client: c.Client} +} + +type NodeAdminClientInstallDiskAcceleratorResults struct { + client rpc.Client + data nodeAdminInstallDiskAcceleratorResultsData +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasKernelRelease() bool { + return v.data.KernelRelease != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) KernelRelease() string { + if v.data.KernelRelease == nil { + return "" + } + return *v.data.KernelRelease +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasLbdVersion() bool { + return v.data.LbdVersion != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) LbdVersion() string { + if v.data.LbdVersion == nil { + return "" + } + return *v.data.LbdVersion +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasError() bool { + return v.data.Error != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) Error() string { + if v.data.Error == nil { + return "" + } + return *v.data.Error +} + +func (v NodeAdminClient) InstallDiskAccelerator(ctx context.Context, image string, force bool) (*NodeAdminClientInstallDiskAcceleratorResults, error) { + args := NodeAdminInstallDiskAcceleratorArgs{} + args.data.Image = &image + args.data.Force = &force + + var ret nodeAdminInstallDiskAcceleratorResultsData + + err := v.Call(ctx, "install_disk_accelerator", &args, &ret) + if err != nil { + return nil, err + } + + return &NodeAdminClientInstallDiskAcceleratorResults{client: v.Client, data: ret}, nil +} diff --git a/api/nodeadmin/rpc.yml b/api/nodeadmin/rpc.yml new file mode 100644 index 000000000..ee4d98317 --- /dev/null +++ b/api/nodeadmin/rpc.yml @@ -0,0 +1,38 @@ +apiVersion: miren.dev/rpc/v1 +kind: IDL + +# Work the coordinator asks one specific node to do to itself. +# +# This is separate from the exec service because it is not about a sandbox: it +# acts on the host, and the coordinator picks the node rather than deriving it +# from a workload's placement. + +interfaces: + - name: NodeAdmin + methods: + - name: install_disk_accelerator + doc: | + Build and load the lbd kernel module on this node, so its disks can + use accelerator mode instead of loop devices. + + The module has to be compiled against the kernel actually running + here, which is why this happens on the node rather than centrally. + The toolchain image is built once by the coordinator and pulled from + the cluster registry. + parameters: + - name: image + type: string + doc: Toolchain image reference to build with + - name: force + type: bool + doc: Rebuild even when the installed module is already current + results: + - name: kernel_release + type: string + doc: Kernel the module was built for + - name: lbd_version + type: string + doc: lbd version that was installed + - name: error + type: string + doc: Error message if the install failed diff --git a/api/runner/rpc.yml b/api/runner/rpc.yml index 97474895b..57760f4f1 100644 --- a/api/runner/rpc.yml +++ b/api/runner/rpc.yml @@ -333,6 +333,36 @@ interfaces: type: string doc: Error message if issuance failed + - name: InstallDiskAccelerator + index: 13 + doc: | + Build and load the lbd kernel module on a runner, so its disks can + use accelerator mode instead of loop devices. + + The coordinator makes sure the toolchain image is in the cluster + registry, then asks the node to do the install. It runs on the node + because the module is compiled against the kernel running there. + parameters: + - name: query + type: string + doc: Runner to install on (name, ID, or short ID) + - name: force + type: bool + doc: Rebuild even when the installed module is already current + results: + - name: name + type: string + doc: Name of the runner installed on + - name: kernel_release + type: string + doc: Kernel the module was built for + - name: lbd_version + type: string + doc: lbd version that was installed + - name: error + type: string + doc: Error message if the install failed + types: - type: InviteInfo doc: Information about a runner invite diff --git a/api/runner/runner_v1alpha/rpc.gen.go b/api/runner/runner_v1alpha/rpc.gen.go index efef3153a..aff44e082 100644 --- a/api/runner/runner_v1alpha/rpc.gen.go +++ b/api/runner/runner_v1alpha/rpc.gen.go @@ -1547,6 +1547,98 @@ func (v *RunnerRegistrationIssueSystemWorkloadTokenResults) UnmarshalJSON(data [ return json.Unmarshal(data, &v.data) } +type runnerRegistrationInstallDiskAcceleratorArgsData struct { + Query *string `cbor:"0,keyasint,omitempty" json:"query,omitempty"` + Force *bool `cbor:"1,keyasint,omitempty" json:"force,omitempty"` +} + +type RunnerRegistrationInstallDiskAcceleratorArgs struct { + call rpc.Call + data runnerRegistrationInstallDiskAcceleratorArgsData +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) HasQuery() bool { + return v.data.Query != nil +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) Query() string { + if v.data.Query == nil { + return "" + } + return *v.data.Query +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type runnerRegistrationInstallDiskAcceleratorResultsData struct { + Name *string `cbor:"0,keyasint,omitempty" json:"name,omitempty"` + KernelRelease *string `cbor:"1,keyasint,omitempty" json:"kernel_release,omitempty"` + LbdVersion *string `cbor:"2,keyasint,omitempty" json:"lbd_version,omitempty"` + Error *string `cbor:"3,keyasint,omitempty" json:"error,omitempty"` +} + +type RunnerRegistrationInstallDiskAcceleratorResults struct { + call rpc.Call + data runnerRegistrationInstallDiskAcceleratorResultsData +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetName(name string) { + v.data.Name = &name +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetKernelRelease(kernel_release string) { + v.data.KernelRelease = &kernel_release +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetLbdVersion(lbd_version string) { + v.data.LbdVersion = &lbd_version +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetError(error string) { + v.data.Error = &error +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type RunnerRegistrationCreateInvite struct { rpc.Call args RunnerRegistrationCreateInviteArgs @@ -1885,6 +1977,32 @@ func (t *RunnerRegistrationIssueSystemWorkloadToken) Results() *RunnerRegistrati return results } +type RunnerRegistrationInstallDiskAccelerator struct { + rpc.Call + args RunnerRegistrationInstallDiskAcceleratorArgs + results RunnerRegistrationInstallDiskAcceleratorResults +} + +func (t *RunnerRegistrationInstallDiskAccelerator) Args() *RunnerRegistrationInstallDiskAcceleratorArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *RunnerRegistrationInstallDiskAccelerator) Results() *RunnerRegistrationInstallDiskAcceleratorResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type RunnerRegistration interface { CreateInvite(ctx context.Context, state *RunnerRegistrationCreateInvite) error Join(ctx context.Context, state *RunnerRegistrationJoin) error @@ -1899,6 +2017,7 @@ type RunnerRegistration interface { UncordonRunner(ctx context.Context, state *RunnerRegistrationUncordonRunner) error DrainRunner(ctx context.Context, state *RunnerRegistrationDrainRunner) error IssueSystemWorkloadToken(ctx context.Context, state *RunnerRegistrationIssueSystemWorkloadToken) error + InstallDiskAccelerator(ctx context.Context, state *RunnerRegistrationInstallDiskAccelerator) error } type reexportRunnerRegistration struct { @@ -1957,6 +2076,10 @@ func (reexportRunnerRegistration) IssueSystemWorkloadToken(ctx context.Context, panic("not implemented") } +func (reexportRunnerRegistration) InstallDiskAccelerator(ctx context.Context, state *RunnerRegistrationInstallDiskAccelerator) error { + panic("not implemented") +} + func (t reexportRunnerRegistration) CapabilityClient() rpc.Client { return t.client } @@ -2093,6 +2216,16 @@ func AdaptRunnerRegistration(t RunnerRegistration) *rpc.Interface { return t.IssueSystemWorkloadToken(ctx, &RunnerRegistrationIssueSystemWorkloadToken{Call: call}) }, }, + { + Name: "InstallDiskAccelerator", + InterfaceName: "RunnerRegistration", + Index: 13, + Public: false, + Params: []string{"query", "force"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.InstallDiskAccelerator(ctx, &RunnerRegistrationInstallDiskAccelerator{Call: call}) + }, + }, } return rpc.NewInterface(methods, t) @@ -2845,3 +2978,67 @@ func (v RunnerRegistrationClient) IssueSystemWorkloadToken(ctx context.Context, return &RunnerRegistrationClientIssueSystemWorkloadTokenResults{client: v.Client, data: ret}, nil } + +type RunnerRegistrationClientInstallDiskAcceleratorResults struct { + client rpc.Client + data runnerRegistrationInstallDiskAcceleratorResultsData +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasName() bool { + return v.data.Name != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) Name() string { + if v.data.Name == nil { + return "" + } + return *v.data.Name +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasKernelRelease() bool { + return v.data.KernelRelease != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) KernelRelease() string { + if v.data.KernelRelease == nil { + return "" + } + return *v.data.KernelRelease +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasLbdVersion() bool { + return v.data.LbdVersion != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) LbdVersion() string { + if v.data.LbdVersion == nil { + return "" + } + return *v.data.LbdVersion +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasError() bool { + return v.data.Error != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) Error() string { + if v.data.Error == nil { + return "" + } + return *v.data.Error +} + +func (v RunnerRegistrationClient) InstallDiskAccelerator(ctx context.Context, query string, force bool) (*RunnerRegistrationClientInstallDiskAcceleratorResults, error) { + args := RunnerRegistrationInstallDiskAcceleratorArgs{} + args.data.Query = &query + args.data.Force = &force + + var ret runnerRegistrationInstallDiskAcceleratorResultsData + + err := v.Call(ctx, "InstallDiskAccelerator", &args, &ret) + if err != nil { + return nil, err + } + + return &RunnerRegistrationClientInstallDiskAcceleratorResults{client: v.Client, data: ret}, nil +} diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 4df287530..510c4bf5e 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1402,12 +1402,41 @@ Warning: These commands are intended for advanced users and developers. They may d.Dispatch("debug entity ensure", Infer("debug entity ensure", "Ensure an entity exists", EntityEnsure)) // Disk commands - d.Dispatch("disk", Section("disk", "Disk backup and recovery", "", WithSectionGroup(GroupServer))) + d.Dispatch("disk", Section("disk", "Disk backup, recovery, and acceleration", "", WithSectionGroup(GroupServer))) d.Dispatch("disk backup", Infer("disk backup", "Backup a disk to a snapshot file", DiskBackup)) d.Dispatch("disk restore", Infer("disk restore", "Restore a disk from a snapshot file", DiskRestore)) d.Dispatch("disk undelete", Infer("disk undelete", "Restore a recently deleted disk", DiskUndelete)) d.Dispatch("disk list-deleted", Infer("disk list-deleted", "List deleted disks available for recovery", DiskListDeleted)) + // Accelerator mode. These build and load the lbd kernel module, so they + // only do anything on Linux; the non-Linux builds register stubs that say + // so rather than leaving the command missing. + d.Dispatch("disk accelerator", Section("disk accelerator", "Faster block-device disks via the lbd kernel module", "", + WithSectionGroup(GroupServer), + WithSectionDescription(acceleratorSectionDescription))) + d.Dispatch("disk accelerator status", Infer("disk accelerator status", "Show whether accelerator mode can run on this host", DiskAcceleratorStatus, + WithExample(mflags.Example{ + Name: "Check accelerator mode", + Body: "miren disk accelerator status", + }), + )) + d.Dispatch("disk accelerator install", Infer("disk accelerator install", "Build and load the lbd kernel module for this kernel", DiskAcceleratorInstall, + WithExample(mflags.Example{ + Name: "Enable accelerator mode on a runner", + Body: "miren disk accelerator install runner1", + }), + WithExample(mflags.Example{ + Name: "Rebuild after a kernel upgrade", + Body: "miren disk accelerator install runner1 --force", + }), + )) + d.Dispatch("disk accelerator uninstall", Infer("disk accelerator uninstall", "Unload and remove the lbd kernel module", DiskAcceleratorUninstall, + WithExample(mflags.Example{ + Name: "Go back to loop devices", + Body: "sudo miren disk accelerator uninstall", + }), + )) + // Debug disk commands d.Dispatch("debug disk", Section("debug disk", "Disk entity debug commands", "", WithSectionDescription(diskSectionDescription))) d.Dispatch("debug disk create", Infer("debug disk create", "Create a disk entity for testing", DebugDiskCreate, diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go new file mode 100644 index 000000000..0aedcf70e --- /dev/null +++ b/cli/commands/disk_accelerator.go @@ -0,0 +1,135 @@ +//go:build linux + +package commands + +import ( + "time" + + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/ui" +) + +// DiskAcceleratorStatus reports whether accelerator mode can run on this host. +// It only reads, so it does not need root. +func DiskAcceleratorStatus(ctx *Context, opts struct { + FormatOptions + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + status, err := lbdmod.Probe(lbdmod.HostOptions(opts.DataPath)) + if err != nil { + return err + } + + if opts.IsJSON() { + return PrintJSON(newAcceleratorStatusJSON(status)) + } + + rows := []ui.Row{ + {"Available", yesNo(status.Available())}, + {"State", status.Explain()}, + {"Kernel", status.Host.KernelRelease}, + {"Module loaded", yesNo(status.Loaded)}, + {"Control device", yesNo(status.ControlDevicePresent)}, + {"Module installed", yesNo(status.ModuleInstalled)}, + {"lbdctl", orNone(status.LbdctlPath)}, + {"Kernel headers", orNone(status.Host.HeadersDir)}, + {"Bundled lbd version", status.EmbeddedVersion}, + } + if status.Marker != nil { + rows = append(rows, + ui.Row{"Installed version", status.Marker.LbdVersion}, + ui.Row{"Built for kernel", status.Marker.KernelRelease}, + ui.Row{"Built at", status.Marker.BuiltAt.Local().Format(time.RFC3339)}, + ) + } + table := ui.NewTable(ui.WithColumns(ui.AutoSizeColumns([]string{"", ""}, rows, nil)), ui.WithRows(rows)) + ctx.Printf("%s\n", table.Render()) + + switch { + case status.Available() && !status.Stale(): + return nil + case status.Stale(): + ctx.Warn("The installed module no longer matches this host. Run: miren disk accelerator install ") + case status.Host.HeadersDir == "" && status.Host.CanFetchHeaders(): + ctx.Info("This host has no kernel headers; the builder will fetch them. Run: miren disk accelerator install ") + case status.Host.HeadersDir == "": + ctx.Warn("This host has no kernel headers, which the build needs. %s", status.Host.InstallHint()) + default: + ctx.Info("To enable accelerator mode, run: miren disk accelerator install ") + } + return nil +} + +// DiskAcceleratorUninstall unloads the module and removes what the install put +// on the host, including the record that would otherwise rebuild it after a +// kernel upgrade. +func DiskAcceleratorUninstall(ctx *Context, opts struct { + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + installer := &lbdmod.Installer{ + Log: ctx.Log, + Options: lbdmod.HostOptions(opts.DataPath), + } + + ctx.Begin("Removing the lbd kernel module") + if err := installer.Uninstall(ctx); err != nil { + return err + } + + ctx.Completed("Accelerator mode removed; disks will use loop devices") + return nil +} + +// acceleratorStatusJSON is the machine-readable shape of the status command. +type acceleratorStatusJSON struct { + Available bool `json:"available"` + State string `json:"state"` + Kernel string `json:"kernel"` + ModuleLoaded bool `json:"module_loaded"` + ControlDevicePresent bool `json:"control_device_present"` + ModuleInstalled bool `json:"module_installed"` + Stale bool `json:"stale"` + LbdctlPath string `json:"lbdctl_path"` + KernelHeaders string `json:"kernel_headers"` + HeaderPackage string `json:"header_package"` + BundledVersion string `json:"bundled_version"` + InstalledVersion string `json:"installed_version,omitempty"` + BuiltForKernel string `json:"built_for_kernel,omitempty"` + BuiltAt string `json:"built_at,omitempty"` +} + +func newAcceleratorStatusJSON(s lbdmod.Status) acceleratorStatusJSON { + out := acceleratorStatusJSON{ + Available: s.Available(), + State: s.Explain(), + Kernel: s.Host.KernelRelease, + ModuleLoaded: s.Loaded, + ControlDevicePresent: s.ControlDevicePresent, + ModuleInstalled: s.ModuleInstalled, + Stale: s.Stale(), + LbdctlPath: s.LbdctlPath, + KernelHeaders: s.Host.HeadersDir, + HeaderPackage: s.Host.HeaderPackage(), + BundledVersion: s.EmbeddedVersion, + } + if s.Marker != nil { + out.InstalledVersion = s.Marker.LbdVersion + out.BuiltForKernel = s.Marker.KernelRelease + out.BuiltAt = s.Marker.BuiltAt.UTC().Format(time.RFC3339) + } + return out +} + +func yesNo(b bool) string { + if b { + return "yes" + } + return "no" +} + +func orNone(s string) string { + if s == "" { + return "not found" + } + return s +} diff --git a/cli/commands/disk_accelerator_doc.go b/cli/commands/disk_accelerator_doc.go new file mode 100644 index 000000000..3fa8d086d --- /dev/null +++ b/cli/commands/disk_accelerator_doc.go @@ -0,0 +1,48 @@ +package commands + +const acceleratorSectionDescription = `Miren serves block-device disks in one of two modes. + +**Universal mode** is the default and works everywhere. It backs each disk with a +loop device, which the Linux kernel provides out of the box. + +**Accelerator mode** uses ` + "`lbd`" + `, a Miren kernel module that puts a +write-ahead log in front of the disk. It is faster, and it is what continuous +backup to Miren Cloud is built on. + +` + "`lbd`" + ` is not part of the Linux kernel, so it has to be compiled for the +exact kernel each node is running. ` + "`miren disk accelerator install`" + ` does +that: your cluster builds the toolchain image with the BuildKit and registry it +already runs, the named node pulls it from there, and the module is compiled and +loaded on that node. Nothing is downloaded from us, and there is no published +image to keep up to date. + +## Getting started + +` + "```" + `bash +miren disk accelerator install runner1 # build and load it on runner1 +sudo systemctl restart miren # on runner1, to pick up the mode +` + "```" + ` + +` + "`install`" + ` names the node to install on, but ` + "`status`" + ` and +` + "`uninstall`" + ` read and change the host you run them on. To check a node +before or after installing, run ` + "`status`" + ` on that node. + +## Requirements + +- The kernel headers for your running kernel. On Debian and Ubuntu the builder + fetches them itself if the host has none. Everywhere else you install them + first, and ` + "`status`" + ` names the package -- ` + "`kernel-devel-$(uname -r)`" + ` + on Fedora and RHEL. +- Secure Boot disabled. A self-built module is unsigned, and firmware with Secure + Boot enforcing will refuse to load it. +- A kernel built with GCC. Clang-built kernels are not supported. + +## After a kernel upgrade + +A module only loads on the kernel it was built for. Once a host has installed the +module, Miren notices on startup that the running kernel has changed and rebuilds +it. You can also do it by hand with +` + "`miren disk accelerator install --force`" + `. + +Until the module is back, disks fall back to universal mode. Nothing breaks; they +are just slower.` diff --git a/cli/commands/disk_accelerator_install.go b/cli/commands/disk_accelerator_install.go new file mode 100644 index 000000000..6cfdeef13 --- /dev/null +++ b/cli/commands/disk_accelerator_install.go @@ -0,0 +1,44 @@ +package commands + +import ( + "fmt" + + "miren.dev/runtime/api/runner/runner_v1alpha" + "miren.dev/runtime/pkg/rpc" +) + +// DiskAcceleratorInstall asks the cluster to build and load the lbd kernel +// module on a node, so its disks use accelerator mode instead of loop devices. +// +// This runs through the server rather than locally because the toolchain image +// lives in the cluster registry, and reaching it needs an identity the CLI does +// not hold. The coordinator builds the image if it is missing, then hands the +// work to the node, which is where the module has to be compiled anyway. +func DiskAcceleratorInstall(ctx *Context, opts struct { + ConfigCentric + + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` +}) error { + client, err := ctx.RPCClient(rpc.ServiceRunner) + if err != nil { + return err + } + defer client.Close() + + rc := runner_v1alpha.NewRunnerRegistrationClient(client) + + ctx.Begin("Installing the lbd kernel module on %s", opts.Node) + + res, err := rc.InstallDiskAccelerator(ctx, opts.Node, opts.Force) + if err != nil { + return err + } + if res.Error() != "" { + return fmt.Errorf("%s", res.Error()) + } + + ctx.Completed("Accelerator mode is ready on %s, kernel %s", res.Name(), res.KernelRelease()) + ctx.Info("Restart that node's miren service to pick it up") + return nil +} diff --git a/cli/commands/disk_accelerator_other.go b/cli/commands/disk_accelerator_other.go new file mode 100644 index 000000000..a1989c3bc --- /dev/null +++ b/cli/commands/disk_accelerator_other.go @@ -0,0 +1,20 @@ +//go:build !linux + +package commands + +import "fmt" + +// DiskAcceleratorStatus is not supported on non-Linux platforms +func DiskAcceleratorStatus(ctx *Context, opts struct { + FormatOptions + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + return fmt.Errorf("disk accelerator status is only available on Linux") +} + +// DiskAcceleratorUninstall is not supported on non-Linux platforms +func DiskAcceleratorUninstall(ctx *Context, opts struct { + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + return fmt.Errorf("disk accelerator uninstall is only available on Linux") +} diff --git a/components/coordinate/foundation.go b/components/coordinate/foundation.go index e62644b6f..dc5b02496 100644 --- a/components/coordinate/foundation.go +++ b/components/coordinate/foundation.go @@ -269,7 +269,7 @@ regen: c.Log.Info("generating new API cert", "path", cert) cc, err := c.authority.IssueCertificate(caauth.Options{ - CommonName: "miren-api", + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: 1 * year, IPs: ips, diff --git a/components/coordinate/runner_endpoints.go b/components/coordinate/runner_endpoints.go index b8467e9b2..8bae2d9bc 100644 --- a/components/coordinate/runner_endpoints.go +++ b/components/coordinate/runner_endpoints.go @@ -13,6 +13,7 @@ import ( "miren.dev/runtime/api/telemetry/telemetry_v1alpha" "miren.dev/runtime/components/diskio" "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/servers/build" disksrv "miren.dev/runtime/servers/disk" runnerserver "miren.dev/runtime/servers/runner" sqlitebackupsrv "miren.dev/runtime/servers/sqlitebackup" @@ -84,6 +85,14 @@ func (c *RunnerEndpoints) Start(context.Context) error { VictoriametricsAddress: c.VictoriametricsAddress, VictorialogsAddress: c.VictorialogsAddress, WorkloadIssuer: c.WorkloadIssuer, + LbdBuilder: &build.LbdToolchain{ + Log: c.Log, + BuildKit: c.BuildKit, + Issuer: c.WorkloadIssuer, + EC: aes.NewClient(c.Log, c.eac), + TempDir: c.TempDir, + }, + RPC: c.state, }) server.ExposeValue(rpc.ServiceRunner, runner_v1alpha.AdaptRunnerRegistration(runnerReg)) server.ExposeValue("dev.miren.runtime/telemetry", telemetry_v1alpha.AdaptTelemetry(telemetrysrv.NewServer(c.Log))) diff --git a/components/diskio/disk_ops_darwin.go b/components/diskio/disk_ops_darwin.go index 1b50ea28b..f0185f2af 100644 --- a/components/diskio/disk_ops_darwin.go +++ b/components/diskio/disk_ops_darwin.go @@ -108,7 +108,7 @@ func EnsureLoopDevices(_ *slog.Logger) error { return fmt.Errorf("loop devices not supported on darwin") } -func EnsureLbdDevices(_ *slog.Logger) error { +func EnsureLbdDevices(_ context.Context, _ *slog.Logger) error { return fmt.Errorf("lbd not supported on darwin") } diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 72637179c..0dac8b7ca 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -17,8 +17,12 @@ import ( "unsafe" "golang.org/x/sys/unix" + "miren.dev/runtime/pkg/lbdmod" ) +// Bound modprobe on the runner startup path. +const modprobeTimeout = 30 * time.Second + const ( loopCtlGetFree = 0x4C82 loopClrFd = 0x4C01 @@ -668,19 +672,37 @@ func ensureLoopDeviceNode(log *slog.Logger, index int) error { return nil } -// EnsureLbdDevices checks if the lbd kernel module and lbdctl are available. -func EnsureLbdDevices(log *slog.Logger) error { - // Try modprobe lbd - if out, err := exec.Command("modprobe", "lbd").CombinedOutput(); err != nil { - log.Warn("modprobe lbd failed", "error", err, "output", string(out)) +// EnsureLbdDevices loads the lbd kernel module and proves it is usable. +// +// modprobe's exit code is not the test: a module can be absent, or present but +// wedged, and lbdctl can be installed on a host whose module never loaded. +// What settles it is the same probe accelerator mode itself relies on, so a +// node never selects accelerator mode it cannot serve. +func EnsureLbdDevices(ctx context.Context, log *slog.Logger) error { + ctx, cancel := context.WithTimeout(ctx, modprobeTimeout) + defer cancel() + if out, err := exec.CommandContext(ctx, "modprobe", lbdmod.ModuleName).CombinedOutput(); err != nil { + log.Debug("modprobe lbd failed", "error", err, "output", strings.TrimSpace(string(out))) + } + + status, err := lbdmod.Probe(lbdmod.HostOptions("")) + if err != nil { + return err } - - // Check that lbdctl is in PATH - if _, err := exec.LookPath("lbdctl"); err != nil { - return fmt.Errorf("lbdctl not found in PATH: %w", err) + if status.Loaded && !status.ControlDevicePresent { + if err := lbdmod.EnsureControlDevice(); err != nil { + return err + } + status, err = lbdmod.Probe(lbdmod.HostOptions("")) + if err != nil { + return err + } + } + if !status.Available() { + return errors.New(status.Explain()) } - log.Info("lbd devices available") + log.Info("lbd devices available", "kernel", status.Host.KernelRelease, "lbdctl", status.LbdctlPath) return nil } diff --git a/components/distributedrunner/boot_node_storage.go b/components/distributedrunner/boot_node_storage.go index 82c1ef0f2..950e002aa 100644 --- a/components/distributedrunner/boot_node_storage.go +++ b/components/distributedrunner/boot_node_storage.go @@ -15,17 +15,19 @@ type nodeStorageBoot struct { output boot.Output[*runner.NodeStorage] } -func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], telemetry boot.Output[telemetryBootOutput]) *nodeStorageBoot { +func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], telemetry boot.Output[telemetryBootOutput], containerd boot.Output[containerdBootOutput], networkDeps boot.Output[runner.RunnerDeps]) *nodeStorageBoot { b := &nodeStorageBoot{} - b.component, b.output = boot.Provide2("node-storage", access, telemetry, b.start, + b.component, b.output = boot.Provide4("node-storage", access, telemetry, containerd, networkDeps, b.start, boot.WithStop(b.stop, 0)) return b } -func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, telemetry telemetryBootOutput) (*runner.NodeStorage, error) { +func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, telemetry telemetryBootOutput, containerd containerdBootOutput, networkDeps runner.RunnerDeps) (*runner.NodeStorage, error) { var err error b.value, err = runner.NewNodeStorage(access.access, runner.RunnerDeps{ MetricsWriter: telemetry.metricsWriter, + CC: containerd.Client, + Resolver: networkDeps.Resolver, }, access.config) if err != nil { return nil, err diff --git a/components/distributedrunner/boot_sandbox_host.go b/components/distributedrunner/boot_sandbox_host.go index 48cf43b29..4e2b7786b 100644 --- a/components/distributedrunner/boot_sandbox_host.go +++ b/components/distributedrunner/boot_sandbox_host.go @@ -38,6 +38,35 @@ type sandboxHostBoot struct { output boot.Output[*runner.SandboxHost] } +// networkDepsBoot maps the cluster registry before storage can rebuild lbd, +// and shares the same resolver with the later sandbox host. +type networkDepsBoot struct { + component *boot.Component + inputs sandboxHostBootInputs + output boot.Output[runner.RunnerDeps] +} + +func newNetworkDepsBoot(inputs sandboxHostBootInputs, access boot.Output[clusterAccessBootOutput]) *networkDepsBoot { + b := &networkDepsBoot{inputs: inputs} + b.component, b.output = boot.Provide1("runner-network-deps", access, b.start) + return b +} + +func (b *networkDepsBoot) start(ctx context.Context, access clusterAccessBootOutput) (runner.RunnerDeps, error) { + var deps runner.RunnerDeps + if err := b.inputs.prepareNetworkDeps(&deps, access.access.CoordinatorInternalIP()); err != nil { + return runner.RunnerDeps{}, err + } + deps.EtcdEndpoints = append([]string(nil), b.inputs.etcdEndpoints...) + deps.EtcdPrefix = b.inputs.etcdPrefix + if len(deps.EtcdEndpoints) > 0 { + if err := runner.InitializeDistributedNetwork(ctx, b.inputs.log, b.inputs.dataPath, &deps, b.inputs.group); err != nil { + return runner.RunnerDeps{}, fmt.Errorf("initializing network before node storage: %w", err) + } + } + return deps, nil +} + func sandboxHostInputs(options StartOptions) sandboxHostBootInputs { return sandboxHostBootInputs{ log: options.Log, @@ -59,10 +88,11 @@ func newSandboxHostBoot( storage boot.Output[*runner.NodeStorage], containerd boot.Output[containerdBootOutput], telemetry boot.Output[telemetryBootOutput], + networkDeps boot.Output[runner.RunnerDeps], ) *sandboxHostBoot { b := &sandboxHostBoot{inputs: inputs} - b.component, b.output = boot.Provide4( - "sandbox-host", access, storage, containerd, telemetry, b.start, + b.component, b.output = boot.Provide5( + "sandbox-host", access, storage, containerd, telemetry, networkDeps, b.start, boot.WithStop(b.stop, 0), ) return b @@ -74,27 +104,19 @@ func (b *sandboxHostBoot) start( storage *runner.NodeStorage, containerd containerdBootOutput, telemetry telemetryBootOutput, + dependencies runner.RunnerDeps, ) (*runner.SandboxHost, error) { - dependencies := runner.RunnerDeps{ - CC: containerd.Client, - Namespace: containerd.Namespace, - Bridge: "rt0", - Tempdir: os.TempDir(), - - DisableLocalNet: true, - LogsMaintainer: observability.NewLogsMaintainer(), - LogWriter: telemetry.logWriter, - StatusMon: observability.NewStatusMonitor(b.inputs.log), - SandboxMetrics: telemetry.sandboxMetrics, - MetricsWriter: telemetry.metricsWriter, - ServicePrefixes: b.inputs.servicePrefixes, - - EtcdEndpoints: append([]string(nil), b.inputs.etcdEndpoints...), - EtcdPrefix: b.inputs.etcdPrefix, - } - if err := b.prepareNetworkDeps(&dependencies, access.access.CoordinatorInternalIP()); err != nil { - return nil, err - } + dependencies.CC = containerd.Client + dependencies.Namespace = containerd.Namespace + dependencies.Bridge = "rt0" + dependencies.Tempdir = os.TempDir() + dependencies.DisableLocalNet = true + dependencies.LogsMaintainer = observability.NewLogsMaintainer() + dependencies.LogWriter = telemetry.logWriter + dependencies.StatusMon = observability.NewStatusMonitor(b.inputs.log) + dependencies.SandboxMetrics = telemetry.sandboxMetrics + dependencies.MetricsWriter = telemetry.metricsWriter + dependencies.ServicePrefixes = b.inputs.servicePrefixes var err error b.value, err = runner.NewSandboxHost(access.access, storage, dependencies, access.config) @@ -107,54 +129,54 @@ func (b *sandboxHostBoot) start( return b.value, nil } -func (b *sandboxHostBoot) prepareNetworkDeps(deps *runner.RunnerDeps, coordinatorInternalIP netip.Addr) error { +func (i sandboxHostBootInputs) prepareNetworkDeps(deps *runner.RunnerDeps, coordinatorInternalIP netip.Addr) error { resolver, hostMapper := netresolve.NewLocalResolver() deps.Resolver = resolver if coordinatorInternalIP.Is4() { if err := hostMapper.SetHost("cluster.local", coordinatorInternalIP); err != nil { return fmt.Errorf("mapping cluster registry: %w", err) } - b.inputs.log.Info("mapped cluster.local to coordinator WireGuard gateway", "addr", coordinatorInternalIP) + i.log.Info("mapped cluster.local to coordinator WireGuard gateway", "addr", coordinatorInternalIP) } - coordinatorHost, coordinatorPort, splitErr := net.SplitHostPort(b.inputs.coordinator) + coordinatorHost, coordinatorPort, splitErr := net.SplitHostPort(i.coordinator) if splitErr != nil { - b.inputs.log.Warn("in-cluster API access disabled: coordinator address has no usable host and port", - "coordinator", b.inputs.coordinator, "error", splitErr) + i.log.Warn("in-cluster API access disabled: coordinator address has no usable host and port", + "coordinator", i.coordinator, "error", splitErr) } else if coordinatorAddr, err := resolveHost(coordinatorHost); err != nil { - b.inputs.log.Warn("could not resolve coordinator address", "host", coordinatorHost, "error", err) + i.log.Warn("could not resolve coordinator address", "host", coordinatorHost, "error", err) } else { // Sandboxes reach the API on the coordinator rather than the local bridge // router. This must be an IP because sandbox DNS resolves app.miren names // and nothing else, so the coordinator hostname would not resolve there. deps.ApiAddress = net.JoinHostPort(coordinatorAddr.String(), coordinatorPort) - deps.CACert = []byte(b.inputs.caCert) - b.inputs.log.Info("sandboxes will reach the cluster API at", "address", deps.ApiAddress) + deps.CACert = []byte(i.caCert) + i.log.Info("sandboxes will reach the cluster API at", "address", deps.ApiAddress) if !coordinatorInternalIP.IsValid() { // Older coordinators serve the registry on the same address as the API. if err := hostMapper.SetHost("cluster.local", coordinatorAddr); err != nil { return fmt.Errorf("mapping legacy cluster registry: %w", err) } - b.inputs.log.Warn("coordinator did not advertise an internal address; using its API address for registry pulls", "addr", coordinatorAddr) + i.log.Warn("coordinator did not advertise an internal address; using its API address for registry pulls", "addr", coordinatorAddr) } } - if b.inputs.clientCert == "" || b.inputs.clientKey == "" || b.inputs.caCert == "" { + if i.clientCert == "" || i.clientKey == "" || i.caCert == "" { return nil } - etcdCertsDir := filepath.Join(b.inputs.dataPath, "etcd-certs") + etcdCertsDir := filepath.Join(i.dataPath, "etcd-certs") if err := os.MkdirAll(etcdCertsDir, 0700); err != nil { return fmt.Errorf("creating etcd certs directory: %w", err) } deps.EtcdTLSCertFile = filepath.Join(etcdCertsDir, "client.crt") deps.EtcdTLSKeyFile = filepath.Join(etcdCertsDir, "client.key") deps.EtcdTLSCAFile = filepath.Join(etcdCertsDir, "ca.crt") - if err := os.WriteFile(deps.EtcdTLSCertFile, []byte(b.inputs.clientCert), 0644); err != nil { + if err := os.WriteFile(deps.EtcdTLSCertFile, []byte(i.clientCert), 0644); err != nil { return fmt.Errorf("writing etcd client cert: %w", err) } - if err := os.WriteFile(deps.EtcdTLSKeyFile, []byte(b.inputs.clientKey), 0600); err != nil { + if err := os.WriteFile(deps.EtcdTLSKeyFile, []byte(i.clientKey), 0600); err != nil { return fmt.Errorf("writing etcd client key: %w", err) } - if err := os.WriteFile(deps.EtcdTLSCAFile, []byte(b.inputs.caCert), 0644); err != nil { + if err := os.WriteFile(deps.EtcdTLSCAFile, []byte(i.caCert), 0644); err != nil { return fmt.Errorf("writing etcd CA cert: %w", err) } return nil diff --git a/components/distributedrunner/boot_sandbox_host_test.go b/components/distributedrunner/boot_sandbox_host_test.go index 70d49b551..0d048c7b8 100644 --- a/components/distributedrunner/boot_sandbox_host_test.go +++ b/components/distributedrunner/boot_sandbox_host_test.go @@ -8,14 +8,28 @@ import ( "github.com/stretchr/testify/require" "miren.dev/runtime/components/runner" + "miren.dev/runtime/pkg/boot" ) +func TestNetworkDepsReadyBeforeNodeStorage(t *testing.T) { + access := &runner.ClusterAccess{} + boot := newNetworkDepsBoot(sandboxHostBootInputs{ + log: testLogger(), coordinator: "198.51.100.9:8443", + }, boot.ResolvedOutput(clusterAccessBootOutput{access: access})) + deps, err := boot.start(t.Context(), clusterAccessBootOutput{access: access}) + require.NoError(t, err) + require.NotNil(t, deps.Resolver) + addr, err := deps.Resolver.LookupHost("cluster.local") + require.NoError(t, err) + require.Equal(t, netip.MustParseAddr("198.51.100.9"), addr) +} + func TestRegistryResolvesOverWireGuardNotPublicCoordinator(t *testing.T) { - boot := &sandboxHostBoot{inputs: sandboxHostBootInputs{ + inputs := sandboxHostBootInputs{ log: testLogger(), coordinator: "198.51.100.9:8443", - }} + } var deps runner.RunnerDeps - require.NoError(t, boot.prepareNetworkDeps(&deps, netip.MustParseAddr("10.8.42.1"))) + require.NoError(t, inputs.prepareNetworkDeps(&deps, netip.MustParseAddr("10.8.42.1"))) addr, err := deps.Resolver.LookupHost("cluster.local") require.NoError(t, err) require.Equal(t, netip.MustParseAddr("10.8.42.1"), addr) @@ -23,11 +37,11 @@ func TestRegistryResolvesOverWireGuardNotPublicCoordinator(t *testing.T) { } func TestLegacyCoordinatorRegistryUsesAPIAddress(t *testing.T) { - boot := &sandboxHostBoot{inputs: sandboxHostBootInputs{ + inputs := sandboxHostBootInputs{ log: testLogger(), coordinator: "198.51.100.9:8443", - }} + } var deps runner.RunnerDeps - require.NoError(t, boot.prepareNetworkDeps(&deps, netip.Addr{})) + require.NoError(t, inputs.prepareNetworkDeps(&deps, netip.Addr{})) addr, err := deps.Resolver.LookupHost("cluster.local") require.NoError(t, err) require.Equal(t, netip.MustParseAddr("198.51.100.9"), addr) diff --git a/components/distributedrunner/startup.go b/components/distributedrunner/startup.go index f12e6ed4e..9967a77ef 100644 --- a/components/distributedrunner/startup.go +++ b/components/distributedrunner/startup.go @@ -16,6 +16,7 @@ type startup struct { runtime *Runtime containerd *containerdcomp.Boot clusterAccess *clusterAccessBoot + networkDeps *networkDepsBoot lifecycle *lifecycleBoot nodeStorage *nodeStorageBoot telemetry *telemetryBoot @@ -28,15 +29,17 @@ type startup struct { func newStartup(runtime *Runtime, options StartOptions) *startup { containerd := containerdcomp.NewBoot("containerd", containerdBootConfig(options)) clusterAccess := newClusterAccessBoot(clusterAccessInputs(options)) + networkDeps := newNetworkDepsBoot(sandboxHostInputs(options), clusterAccess.output) lifecycle := newLifecycleBoot(options.Log, runtime.instance, clusterAccess.output) telemetry := newTelemetryBoot(telemetryInputs(options), clusterAccess.output) - nodeStorage := newNodeStorageBoot(clusterAccess.output, telemetry.output) + nodeStorage := newNodeStorageBoot(clusterAccess.output, telemetry.output, containerd.Output, networkDeps.output) sandboxHost := newSandboxHostBoot( sandboxHostInputs(options), clusterAccess.output, nodeStorage.output, containerd.Output, telemetry.output, + networkDeps.output, ) storageAgent := runnercomp.NewStorageAgentBoot(nodeStorage.output, sandboxHost.component, 0) sandboxAgent := runnercomp.NewSandboxAgentBoot(sandboxHost.output, 0) @@ -46,6 +49,7 @@ func newStartup(runtime *Runtime, options StartOptions) *startup { runtime: runtime, containerd: containerd, clusterAccess: clusterAccess, + networkDeps: networkDeps, lifecycle: lifecycle, nodeStorage: nodeStorage, telemetry: telemetry, @@ -60,6 +64,7 @@ func (s *startup) addComponents() error { components := []*boot.Component{ s.containerd.Component, s.clusterAccess.component, + s.networkDeps.component, s.lifecycle.component, s.nodeStorage.component, s.telemetry.component, diff --git a/components/distributedrunner/startup_test.go b/components/distributedrunner/startup_test.go index c08ef5948..9d692f371 100644 --- a/components/distributedrunner/startup_test.go +++ b/components/distributedrunner/startup_test.go @@ -43,6 +43,7 @@ func TestSandboxHostDoesNotStartAfterContainerdFailure(t *testing.T) { boot.ResolvedOutput((*runner.NodeStorage)(nil)), containerdOutput, boot.ResolvedOutput(telemetryBootOutput{}), + boot.ResolvedOutput(runner.RunnerDeps{}), ) graph := boot.NewGraph() diff --git a/components/runner/lbd.go b/components/runner/lbd.go new file mode 100644 index 000000000..aa93b8480 --- /dev/null +++ b/components/runner/lbd.go @@ -0,0 +1,129 @@ +package runner + +import ( + "context" + "errors" + "log/slog" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/components/netresolve" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/lbdmod/ctrbuild" + "miren.dev/runtime/pkg/workloadidentity" +) + +// lbdDeps is what bringing accelerator mode up needs from the runner: a +// containerd to run the build in, the cluster address mapping and identity to +// pull the toolchain image with, and the data path holding the install record. +type lbdDeps struct { + CC *containerd.Client + Resolver netresolve.Resolver + WorkloadIssuer workloadidentity.TokenIssuer + DataPath string +} + +// rebuildTimeout bounds the unattended rebuild at startup. +// +// A rebuild pulls the builder image, may fetch kernel headers, and compiles; +// on a real node that is tens of seconds, and this leaves generous room for a +// slow link. What it must not do is wait forever: without a bound, a builder +// that wedges -- a stalled fetch, a hung pull -- holds up the runner's whole +// startup with no way out, which is a far worse outcome than the slower disks +// we get by giving up. +const rebuildTimeout = 10 * time.Minute + +// setupLbd brings accelerator mode up, rebuilding the lbd kernel module if a +// kernel upgrade left the installed one unusable. +// +// A module only loads on the kernel it was built for, so an operator who +// enabled accelerator mode and then took a kernel update would otherwise find +// their disks silently back on loop devices. Rebuilding is limited to hosts +// that already installed the module: a host that never opted in should not pay +// for an unattended compile at startup. +// +// This blocks rather than running in the background because the disk +// controller picks universal or accelerator mode once, at startup, from +// whether lbd is usable. Deciding that before the module is ready would pin +// the node to loop devices until the next restart. +// +// Neither failing nor timing out is fatal. Universal mode works everywhere, so +// the worst case is slower disks, not a runner that will not start. +func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { + ready := diskio.EnsureLbdDevices(ctx, log) == nil + if deps.CC == nil || deps.Resolver == nil { + if !ready { + log.Warn("cannot rebuild the lbd kernel module without containerd and a cluster registry resolver; disks will use loop devices") + } + return + } + + // The toolchain image lives in the cluster registry and nowhere public, so + // the pull needs the cluster's own address mapping and a registry token. + // A distributed runner holds both: its issuer proxies to the coordinator, + // which is what mints the token on its behalf. + registry := &ctrbuild.ClusterRegistry{ + Resolver: deps.Resolver, + Issuer: deps.WorkloadIssuer, + } + + // dataPath has to be the runner's own, not the package default: the + // install record lives under it, and reading it from the wrong place + // would make a host that installed lbd look like one that never did, so + // the rebuild after a kernel upgrade would never fire. + installer := &lbdmod.Installer{ + Log: log, + Builder: ctrbuild.New(deps.CC, log, ctrbuild.WithClusterRegistry(registry)), + Options: lbdmod.HostOptions(deps.DataPath), + } + + ctx, cancel := context.WithTimeout(ctx, rebuildTimeout) + defer cancel() + + rebuilt, err := installer.EnsureCurrent(ctx) + if err != nil { + // A failed swap can unload the old module; the pre-build result is not + // evidence that accelerator mode is still available. + status, probeErr := lbdmod.Probe(lbdmod.HostOptions(deps.DataPath)) + usable := probeErr == nil && status.Available() + if errors.Is(err, context.DeadlineExceeded) { + log.Warn("gave up rebuilding the lbd kernel module", + "timeout", rebuildTimeout, "module_usable", usable, + "retry_with", "miren disk accelerator install") + } else { + log.Warn("could not rebuild the lbd kernel module", "error", err, "module_usable", usable) + } + return + } + if !rebuilt { + if !ready { + log.Info("accelerator mode is not enabled on this host, disks will use loop devices", + "enable_with", "miren disk accelerator install") + } + return + } + + if err := diskio.EnsureLbdDevices(ctx, log); err != nil { + log.Warn("rebuilt the lbd kernel module but it is still not usable", "error", err) + } +} + +// lbdDeps gathers what accelerator mode needs from the runner. +func (r *SandboxHost) lbdDeps() lbdDeps { + return lbdDeps{ + CC: r.deps.CC, + Resolver: r.deps.Resolver, + WorkloadIssuer: r.deps.WorkloadIssuer, + DataPath: r.DataPath, + } +} + +func (s *NodeStorage) lbdDeps() lbdDeps { + return lbdDeps{ + CC: s.deps.CC, + Resolver: s.deps.Resolver, + WorkloadIssuer: s.access.deps.WorkloadIssuer, + DataPath: s.config.DataPath, + } +} diff --git a/components/runner/lbd_test.go b/components/runner/lbd_test.go new file mode 100644 index 000000000..ad0f6a998 --- /dev/null +++ b/components/runner/lbd_test.go @@ -0,0 +1,24 @@ +package runner + +import ( + "testing" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/netresolve" +) + +func TestNodeStorageLbdDeps(t *testing.T) { + client := &containerd.Client{} + resolver, _ := netresolve.NewLocalResolver() + dataPath := t.TempDir() + storage, err := NewNodeStorage(&ClusterAccess{}, RunnerDeps{ + CC: client, Resolver: resolver, + }, RunnerConfig{DataPath: dataPath}) + require.NoError(t, err) + + deps := storage.lbdDeps() + require.Same(t, client, deps.CC) + require.Same(t, resolver, deps.Resolver) + require.Equal(t, dataPath, deps.DataPath) +} diff --git a/components/runner/nodeadmin.go b/components/runner/nodeadmin.go new file mode 100644 index 000000000..407519072 --- /dev/null +++ b/components/runner/nodeadmin.go @@ -0,0 +1,99 @@ +package runner + +import ( + "context" + "fmt" + "log/slog" + + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/lbdmod/ctrbuild" + "miren.dev/runtime/pkg/rpc" +) + +// nodeAdminServer handles work the coordinator asks this node to do to itself. +// +// It lives on the runner rather than the coordinator because the work is about +// this host: a kernel module has to be compiled against the kernel actually +// running here, and loaded into it. +type nodeAdminServer struct { + log *slog.Logger + deps lbdDeps +} + +// InstallDiskAccelerator builds and loads the lbd kernel module on this node. +// +// Failures come back in the result rather than as an RPC error, so the +// operator sees why the install did not happen instead of a transport-level +// message. The call itself only fails when the node could not be reached. +func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodeadmin_v1alpha.NodeAdminInstallDiskAccelerator) error { + res := req.Results() + + if err := requireCoordinator(ctx); err != nil { + s.log.Warn("rejected a disk accelerator install", "error", err) + res.SetError(err.Error()) + return nil + } + + // Pinned to the cluster's toolchain tag family. The image's entrypoint + // runs here and its output is loaded into this kernel as root, so a + // reference pointing at any other tag is not something to act on even from a + // caller that got past the check above. + image := req.Args().Image() + if !lbdmod.IsBuilderImage(image) { + s.log.Warn("rejected a disk accelerator install naming a foreign image", "image", image) + res.SetError(fmt.Sprintf("%q is not this cluster's lbd toolchain image", image)) + return nil + } + + installer := &lbdmod.Installer{ + Log: s.log, + Builder: ctrbuild.New(s.deps.CC, s.log, ctrbuild.WithClusterRegistry(&ctrbuild.ClusterRegistry{ + Resolver: s.deps.Resolver, + Issuer: s.deps.WorkloadIssuer, + })), + Options: lbdmod.HostOptions(s.deps.DataPath), + Image: image, + } + + status, err := installer.Install(ctx, req.Args().Force()) + if err != nil { + s.log.Warn("installing the lbd kernel module failed", "error", err) + res.SetError(err.Error()) + return nil + } + + // The disk controller reads the mode once at startup, so a node that just + // gained accelerator mode keeps serving loop devices until it restarts. + // Say so rather than letting the operator discover it from a disk that + // came up the old way. + if err := diskio.EnsureLbdDevices(ctx, s.log); err != nil { + s.log.Warn("lbd installed but is not usable yet", "error", err) + res.SetError(fmt.Sprintf("lbd installed but is not usable: %v", err)) + return nil + } + + res.SetKernelRelease(status.Host.KernelRelease) + res.SetLbdVersion(lbdmod.SourceVersion()) + return nil +} + +// requireCoordinator refuses anyone but the coordinator. +// +// The listener verifies client certificates against the cluster CA. The +// coordinator dials with its API certificate, whose subject separates it from +// other cluster members that can also reach this port. +func requireCoordinator(ctx context.Context) error { + identity := rpc.IdentityFromContext(ctx) + if identity == nil || identity.Method == rpc.AuthMethodAnonymous { + return fmt.Errorf("installing a kernel module requires the coordinator's certificate, and this caller presented none") + } + if identity.Method != rpc.AuthMethodCert { + return fmt.Errorf("installing a kernel module requires a certificate, got %q", identity.Method) + } + if identity.Subject != rpc.CoordinatorCertSubject { + return fmt.Errorf("only the coordinator may install a kernel module, not %q", identity.Subject) + } + return nil +} diff --git a/components/runner/nodeadmin_test.go b/components/runner/nodeadmin_test.go new file mode 100644 index 000000000..68a13a834 --- /dev/null +++ b/components/runner/nodeadmin_test.go @@ -0,0 +1,144 @@ +package runner + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/clientconfig" + "miren.dev/runtime/pkg/caauth" + "miren.dev/runtime/pkg/rpc" +) + +func TestRequireCoordinatorRejectsAnAnonymousCaller(t *testing.T) { + err := requireCoordinator(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "presented none") + + err = requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodAnonymous})) + require.Error(t, err) +} + +func TestRequireCoordinatorRejectsAnotherCertHolder(t *testing.T) { + // A registered runner holds a valid cluster certificate. Holding one is + // not the same as being the coordinator. + err := requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodCert, Subject: "runner-abc123"})) + require.Error(t, err) + assert.Contains(t, err.Error(), "only the coordinator") +} + +func TestRequireCoordinatorRejectsANonCertMethod(t *testing.T) { + // A bearer token or JWT carrying the right subject is still not the + // coordinator's certificate. + err := requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodJWT, Subject: rpc.CoordinatorCertSubject})) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a certificate") +} + +func TestRequireCoordinatorAcceptsTheCoordinator(t *testing.T) { + require.NoError(t, requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodCert, Subject: rpc.CoordinatorCertSubject}))) +} + +func TestRunnerListenerNeedsAClusterCA(t *testing.T) { + access, err := NewClusterAccess(slog.Default(), RunnerDeps{}, RunnerConfig{ + Id: "runner", DataPath: t.TempDir(), ListenAddress: "localhost:0", + }) + require.NoError(t, err) + _, err = access.newRPCState(t.Context()) + require.ErrorContains(t, err, "cluster config is required") + + cfg := clientconfig.NewConfig() + cfg.SetCluster("cluster", &clientconfig.ClusterConfig{Hostname: "localhost:0"}) + require.NoError(t, cfg.SetActiveCluster("cluster")) + access.Config = cfg + _, err = access.newRPCState(t.Context()) + require.ErrorContains(t, err, "cluster CA is required") +} + +func TestNodeAdminAuthenticatesCoordinatorOverWire(t *testing.T) { + ca, err := caauth.New(caauth.Options{CommonName: "cluster-ca", Organization: "miren", ValidFor: time.Hour}) + require.NoError(t, err) + runnerCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: "runner", Organization: "miren", ValidFor: time.Hour, DNSNames: []string{"localhost"}, + }) + require.NoError(t, err) + coordinatorCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + otherCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: "another-runner", Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + foreignCA, err := caauth.New(caauth.Options{CommonName: "cluster-ca", Organization: "miren", ValidFor: time.Hour}) + require.NoError(t, err) + forgedCert, err := foreignCA.IssueCertificate(caauth.Options{ + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + + for _, insecure := range []bool{false, true} { + name := "verified-outbound" + if insecure { + name = "insecure-outbound" + } + t.Run(name, func(t *testing.T) { + cfg := clientconfig.NewConfig() + cfg.SetCluster("cluster", &clientconfig.ClusterConfig{ + Hostname: "localhost:0", CACert: string(ca.GetCACertificate()), + ClientCert: string(runnerCert.CertPEM), ClientKey: string(runnerCert.KeyPEM), Insecure: insecure, + }) + require.NoError(t, cfg.SetActiveCluster("cluster")) + access, err := NewClusterAccess(slog.Default(), RunnerDeps{}, RunnerConfig{ + Id: "runner", DataPath: t.TempDir(), ListenAddress: "localhost:0", Config: cfg, + }) + require.NoError(t, err) + server, err := access.newRPCState(t.Context()) + require.NoError(t, err) + defer server.Close() + server.Server().ExposeValue(rpc.ServiceNodeAdmin, nodeadmin_v1alpha.AdaptNodeAdmin(&nodeAdminServer{log: slog.Default()})) + + call := func(cert, key []byte) (*nodeadmin_v1alpha.NodeAdminClientInstallDiskAcceleratorResults, error) { + opts := []rpc.StateOption{rpc.WithSkipVerify} + if cert != nil { + opts = append(opts, rpc.WithCertPEMs(cert, key)) + } + client, err := rpc.NewState(t.Context(), opts...) + if err != nil { + return nil, err + } + defer client.Close() + cl, err := client.Connect(server.ListenAddr(), string(rpc.ServiceNodeAdmin)) + if err != nil { + return nil, err + } + defer cl.Close() + return nodeadmin_v1alpha.NewNodeAdminClient(cl).InstallDiskAccelerator(t.Context(), "foreign/image:tag", false) + } + + // The foreign image stops authorized callers before any host work. + result, err := call(coordinatorCert.CertPEM, coordinatorCert.KeyPEM) + require.NoError(t, err) + assert.Contains(t, result.Error(), "not this cluster's lbd toolchain image") + + result, err = call(otherCert.CertPEM, otherCert.KeyPEM) + require.NoError(t, err) + assert.Contains(t, result.Error(), "only the coordinator") + + _, err = call(forgedCert.CertPEM, forgedCert.KeyPEM) + require.Error(t, err) + + _, err = call(nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "401") + }) + } +} diff --git a/components/runner/runner.go b/components/runner/runner.go index 139decfd2..12b44fae8 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -20,6 +20,7 @@ import ( "miren.dev/runtime/api/exec/exec_v1alpha" "miren.dev/runtime/api/metric/metric_v1alpha" "miren.dev/runtime/api/network/network_v1alpha" + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/api/secret/secret_v1alpha" "miren.dev/runtime/api/sqlitebackup/sqlitebackup_v1alpha" @@ -54,9 +55,8 @@ type RunnerConfig struct { Workers int `json:"workers" cbor:"workers" yaml:"workers"` DataPath string `json:"data_path" cbor:"data_path" yaml:"data_path"` - // Optional RPC configuration for advanced setups - // If not provided, a default insecure connection will be used - // to connect to the server address. + // RPC configuration supplies the cluster CA used to authenticate callers + // on the runner listener, as well as its outbound coordinator connection. Config *clientconfig.Config `json:"config" cbor:"config" yaml:"config"` // Optional cloud authentication configuration for disk replication @@ -443,12 +443,8 @@ func (r *ClusterAccess) Start(ctx context.Context) (retErr error) { client *rpc.NetworkClient ) - r.Log.Info("establishing cluster access", "listen", r.ListenAddress, "distributed", r.Config != nil) - if r.Config == nil { - rs, err = rpc.NewState(ctx, rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress), rpc.WithSkipVerify) - } else { - rs, err = r.Config.State(ctx, rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress)) - } + r.Log.Info("establishing cluster access", "listen", r.ListenAddress) + rs, err = r.newRPCState(ctx) if err != nil { return err } @@ -463,11 +459,7 @@ func (r *ClusterAccess) Start(ctx context.Context) (retErr error) { r.state = nil } }() - if r.Config == nil { - client, err = rs.Connect("", "entities") - } else { - client, err = rs.Client("entities") - } + client, err = rs.Client("entities") if err != nil { return err } @@ -495,6 +487,29 @@ func (r *ClusterAccess) Server() *rpc.Server { return r.state.Server() } +func (r *ClusterAccess) newRPCState(ctx context.Context) (*rpc.State, error) { + opts := []rpc.StateOption{ + rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress), + rpc.WithAuthenticator(&rpc.LocalOnlyAuthenticator{}), + } + if r.Config == nil { + return nil, fmt.Errorf("runner cluster config is required to authenticate coordinator requests") + } + + cluster, err := r.Config.GetActiveCluster() + if err != nil { + return nil, fmt.Errorf("runner cluster CA: %w", err) + } + if cluster.CACert == "" { + return nil, fmt.Errorf("runner cluster CA is required to authenticate coordinator requests") + } + // Config.State supplies the runner's own certificate for outbound calls. + // Apply the CA last so even an insecure outbound configuration cannot turn + // off verification of certificates presented to this listener. + opts = append(opts, rpc.WithCertificateVerification([]byte(cluster.CACert))) + return r.Config.State(ctx, opts...) +} + func (r *ClusterAccess) Close() error { var errs []error for _, closer := range r.closers { @@ -514,8 +529,8 @@ func (r *SandboxHost) Start(ctx context.Context, eg ...*errgroup.Group) error { r.Log.Info("starting sandbox host", "id", r.Id) // Initialize Flannel/WireGuard network if distributed runner configuration is provided - if len(r.deps.EtcdEndpoints) > 0 { - if err := r.initializeNetwork(ctx, eg...); err != nil { + if len(r.deps.EtcdEndpoints) > 0 && r.deps.Subnet == nil { + if err := InitializeDistributedNetwork(ctx, r.Log, r.DataPath, &r.deps, eg...); err != nil { return fmt.Errorf("failed to initialize network: %w", err) } } @@ -541,6 +556,12 @@ func (r *SandboxHost) Start(ctx context.Context, eg ...*errgroup.Group) error { r.Log.Info("Registered exec server") + r.access.state.Server().ExposeValue(rpc.ServiceNodeAdmin, nodeadmin_v1alpha.AdaptNodeAdmin(&nodeAdminServer{ + log: r.Log.With("module", "nodeadmin"), + deps: r.lbdDeps(), + })) + r.Log.Info("Registered node admin server") + return nil } @@ -733,28 +754,28 @@ func (r *ClusterAccess) setupRemoteSecrets(rs *rpc.State) error { return nil } -// initializeNetwork sets up the Flannel network for distributed runners. -// This is only called when EtcdEndpoints are configured (distributed runner mode). -func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Group) error { - r.Log.Info("Initializing distributed runner network", - "etcd_endpoints", r.deps.EtcdEndpoints, - "etcd_prefix", r.deps.EtcdPrefix) +// InitializeDistributedNetwork joins the mesh before storage tries to pull +// the lbd builder image from the coordinator's WireGuard-routed registry. +func InitializeDistributedNetwork(ctx context.Context, log *slog.Logger, dataPath string, deps *RunnerDeps, eg ...*errgroup.Group) error { + log.Info("Initializing distributed runner network", + "etcd_endpoints", deps.EtcdEndpoints, + "etcd_prefix", deps.EtcdPrefix) grungeOpts := grunge.NetworkOptions{ - EtcdEndpoints: r.deps.EtcdEndpoints, - EtcdPrefix: r.deps.EtcdPrefix, - PrevIPv4: r.deps.IPv4Routable, + EtcdEndpoints: deps.EtcdEndpoints, + EtcdPrefix: deps.EtcdPrefix, + PrevIPv4: deps.IPv4Routable, } // Add TLS config if provided - if r.deps.EtcdTLSCertFile != "" && r.deps.EtcdTLSKeyFile != "" && r.deps.EtcdTLSCAFile != "" { - r.Log.Info("Using etcd TLS", "cert", r.deps.EtcdTLSCertFile, "ca", r.deps.EtcdTLSCAFile) - grungeOpts.TLSCertFile = r.deps.EtcdTLSCertFile - grungeOpts.TLSKeyFile = r.deps.EtcdTLSKeyFile - grungeOpts.TLSCAFile = r.deps.EtcdTLSCAFile + if deps.EtcdTLSCertFile != "" && deps.EtcdTLSKeyFile != "" && deps.EtcdTLSCAFile != "" { + log.Info("Using etcd TLS", "cert", deps.EtcdTLSCertFile, "ca", deps.EtcdTLSCAFile) + grungeOpts.TLSCertFile = deps.EtcdTLSCertFile + grungeOpts.TLSKeyFile = deps.EtcdTLSKeyFile + grungeOpts.TLSCAFile = deps.EtcdTLSCAFile } - gn, err := grunge.NewNetwork(r.Log, grungeOpts) + gn, err := grunge.NewNetwork(log, grungeOpts) if err != nil { return fmt.Errorf("failed to create grunge network: %w", err) } @@ -778,18 +799,18 @@ func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Gro if localGroup { go func() { if err := runGroup.Wait(); err != nil { - r.Log.Error("network errgroup failed", "error", err) + log.Error("network errgroup failed", "error", err) } }() } // Update deps with the leased IP and subnet lease := gn.Lease() - r.deps.IPv4Routable = lease.IPv4() + deps.IPv4Routable = lease.IPv4() // Initialize netdb subnet from the flannel lease so the sandbox // controller can allocate IPs within this runner's subnet. - ndb, err := netdb.New(filepath.Join(r.DataPath, "net.db")) + ndb, err := netdb.New(filepath.Join(dataPath, "net.db")) if err != nil { return fmt.Errorf("failed to open netdb: %w", err) } @@ -797,9 +818,9 @@ func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Gro if err != nil { return fmt.Errorf("failed to create subnet from lease: %w", err) } - r.deps.Subnet = subnet + deps.Subnet = subnet - r.Log.Info("Joined Flannel network", "ipv4", lease.IPv4().String()) + log.Info("Joined Flannel network", "ipv4", lease.IPv4().String()) return nil } diff --git a/components/runner/storage.go b/components/runner/storage.go index a7e0e362a..1a1e743fc 100644 --- a/components/runner/storage.go +++ b/components/runner/storage.go @@ -71,9 +71,7 @@ func (s *NodeStorage) Start(ctx context.Context) error { if err := diskio.EnsureLoopDevices(log); err != nil { log.Warn("Loop devices not available, disk mounts will fail", "error", err) } - if err := diskio.EnsureLbdDevices(log); err != nil { - log.Warn("lbd devices not available, accelerator mode will not work", "error", err) - } + setupLbd(ctx, s.lbdDeps(), log) diskioState, err := diskio.LoadState(dataPath) if err != nil { diff --git a/components/server/boot_node_storage.go b/components/server/boot_node_storage.go index 443695d3f..9293ebe23 100644 --- a/components/server/boot_node_storage.go +++ b/components/server/boot_node_storage.go @@ -6,6 +6,7 @@ import ( "context" "miren.dev/runtime/components/coordinate" + "miren.dev/runtime/components/netresolve" "miren.dev/runtime/components/runner" "miren.dev/runtime/pkg/boot" ) @@ -14,18 +15,20 @@ type nodeStorageBoot struct { component *boot.Component value *runner.NodeStorage output boot.Output[*runner.NodeStorage] + resolver netresolve.Resolver } -func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], registration boot.Output[registrationBootOutput], observability boot.Output[observabilityBootOutput]) *nodeStorageBoot { - b := &nodeStorageBoot{} - b.component, b.output = boot.Provide3( - "node-storage", access, registration, observability, b.start, +func newNodeStorageBoot(resolver netresolve.Resolver, mapping *boot.Component, access boot.Output[clusterAccessBootOutput], registration boot.Output[registrationBootOutput], observability boot.Output[observabilityBootOutput], containerd boot.Output[containerdBootOutput]) *nodeStorageBoot { + b := &nodeStorageBoot{resolver: resolver} + b.component, b.output = boot.Provide4( + "node-storage", access, registration, observability, containerd, b.start, + boot.DependsOn(mapping), boot.WithStop(b.stop, runnerComponentStopTimeout), ) return b } -func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, registration registrationBootOutput, observability observabilityBootOutput) (*runner.NodeStorage, error) { +func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, registration registrationBootOutput, observability observabilityBootOutput, containerd containerdBootOutput) (*runner.NodeStorage, error) { config := access.config cloudAuth := registration.cloudAuth if cloudAuth.Enabled { @@ -37,6 +40,8 @@ func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOut b.value, err = runner.NewNodeStorage(access.access, runner.RunnerDeps{ IsCoordinator: true, MetricsWriter: observability.operationalMetrics, + CC: containerd.Client, + Resolver: b.resolver, }, config) if err != nil { return nil, err diff --git a/components/server/startup.go b/components/server/startup.go index 311522d2d..c7ee3d678 100644 --- a/components/server/startup.go +++ b/components/server/startup.go @@ -140,7 +140,7 @@ func newStartup(runtime *Runtime, options StartOptions) *startup { observability.output, runnerEndpoints.component, ) - nodeStorage := newNodeStorageBoot(clusterAccess.output, registration.output, observability.output) + nodeStorage := newNodeStorageBoot(resolver, registryHostMapping.component, clusterAccess.output, registration.output, observability.output, containerd.Output) sandboxHost := newSandboxHostBoot( sandboxHostInputs(options, resolver, serverPort(options.Log, address)), clusterAccess.output, diff --git a/controllers/artifact/gc.go b/controllers/artifact/gc.go index 338e16c12..2f6fa9abb 100644 --- a/controllers/artifact/gc.go +++ b/controllers/artifact/gc.go @@ -153,7 +153,11 @@ func (c *GCController) RunGC(ctx context.Context) (*GCResult, error) { var art core_v1alpha.Artifact art.Decode(e.Entity()) - if referenced[art.ID] { + // System artifacts belong to no app by design -- the toolchain image + // miren builds for itself is the case -- so the AppVersion set will + // never name them. Collecting them would delete blobs out from under + // nodes still pulling the image. + if referenced[art.ID] || core_v1alpha.IsSystemArtifact(art.ID) { result.RetainedArtifacts++ continue } diff --git a/controllers/artifact/gc_test.go b/controllers/artifact/gc_test.go index 975a4eadf..a28a0f649 100644 --- a/controllers/artifact/gc_test.go +++ b/controllers/artifact/gc_test.go @@ -124,3 +124,33 @@ func TestGCController_SkipsArchived(t *testing.T) { require.Equal(t, 0, result.TotalArtifacts, "archived artifacts are not evaluated") require.Equal(t, 0, len(result.ArchivedArtifacts)) } + +func TestGCController_RetainsSystemArtifacts(t *testing.T) { + // The toolchain image miren builds for itself belongs to no app, so the + // AppVersion set will never name it. Collecting it would delete blobs out + // from under nodes still pulling the image. + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + sysID, err := inmem.Client.Create(ctx, + core_v1alpha.SystemArtifactPrefix+"lbd-builder-21c0e11624c12f31", + &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + // A genuine orphan alongside it, to prove the exemption is narrow and not + // just "artifacts with no app survive". + orphanID, err := inmem.Client.Create(ctx, "orphan", &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + gc := &GCController{Log: log, EAC: inmem.EAC, Config: GCConfig{CheckInterval: time.Hour}} + + result, err := gc.RunGC(ctx) + require.NoError(t, err) + + require.Equal(t, 1, result.RetainedArtifacts) + require.Equal(t, []entity.Id{orphanID}, result.ArchivedArtifacts) + require.Equal(t, core_v1alpha.ACTIVE, artifactStatus(t, inmem.EAC, sysID)) + require.Equal(t, core_v1alpha.ARCHIVED, artifactStatus(t, inmem.EAC, orphanID)) +} diff --git a/controllers/disk/disk_controller.go b/controllers/disk/disk_controller.go index e062decdd..b7f1220f1 100644 --- a/controllers/disk/disk_controller.go +++ b/controllers/disk/disk_controller.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "os/exec" "strings" compute "miren.dev/runtime/api/compute/compute_v1alpha" @@ -13,6 +12,7 @@ import ( "miren.dev/runtime/pkg/controller" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/lbdmod" ) // detectDiskMode determines which disk I/O mode to use. @@ -26,8 +26,11 @@ func detectDiskMode(configured string) storage_v1alpha.DiskMode { return storage_v1alpha.ACCELERATOR } - // Auto-detect: use accelerator mode if lbd is available - if _, err := exec.LookPath("lbdctl"); err == nil { + // Auto-detect: use accelerator mode only if the lbd module is actually + // loaded and drivable. lbdctl on PATH is not enough -- miren installs it + // alongside the module, so its presence says nothing about whether the + // module loaded. + if lbdmod.Available(lbdmod.HostOptions("")) { return storage_v1alpha.ACCELERATOR } diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index 313334630..cd9ae683c 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -192,6 +192,10 @@ "id": "command/disk" }, "items": [ + "command/disk-accelerator", + "command/disk-accelerator-install", + "command/disk-accelerator-status", + "command/disk-accelerator-uninstall", "command/disk-backup", "command/disk-list-deleted", "command/disk-restore", diff --git a/docs/docs/command/disk-accelerator-install.md b/docs/docs/command/disk-accelerator-install.md new file mode 100644 index 000000000..6887c11ac --- /dev/null +++ b/docs/docs/command/disk-accelerator-install.md @@ -0,0 +1,49 @@ +--- +title: "miren disk accelerator install" +sidebar_label: "disk accelerator install" +description: "Build and load the lbd kernel module for this kernel" +--- + +# miren disk accelerator install + +Build and load the lbd kernel module for this kernel + +## Usage + +```bash +miren disk accelerator install [flags] +``` + +## Arguments + +- `node` — Runner to install on (name, ID, or short ID) + +## Flags + +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file +- `--force, -f` — Rebuild even when the module is already current + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Enable accelerator mode on a runner:** + +```bash +miren disk accelerator install runner1 +``` + +**Rebuild after a kernel upgrade:** + +```bash +miren disk accelerator install runner1 --force +``` + +## See also + +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator-status.md b/docs/docs/command/disk-accelerator-status.md new file mode 100644 index 000000000..82cb1d270 --- /dev/null +++ b/docs/docs/command/disk-accelerator-status.md @@ -0,0 +1,39 @@ +--- +title: "miren disk accelerator status" +sidebar_label: "disk accelerator status" +description: "Show whether accelerator mode can run on this host" +--- + +# miren disk accelerator status + +Show whether accelerator mode can run on this host + +## Usage + +```bash +miren disk accelerator status [flags] +``` + +## Flags + +- `--data-path` — Path to miren data (default: `/var/lib/miren`) +- `--format` — Output format (text, json) (default: `text`) +- `--json` — Shorthand for --format json + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Check accelerator mode:** + +```bash +miren disk accelerator status +``` + +## See also + +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator-uninstall.md b/docs/docs/command/disk-accelerator-uninstall.md new file mode 100644 index 000000000..4e3655ebd --- /dev/null +++ b/docs/docs/command/disk-accelerator-uninstall.md @@ -0,0 +1,37 @@ +--- +title: "miren disk accelerator uninstall" +sidebar_label: "disk accelerator uninstall" +description: "Unload and remove the lbd kernel module" +--- + +# miren disk accelerator uninstall + +Unload and remove the lbd kernel module + +## Usage + +```bash +miren disk accelerator uninstall [flags] +``` + +## Flags + +- `--data-path` — Path to miren data (default: `/var/lib/miren`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Go back to loop devices:** + +```bash +sudo miren disk accelerator uninstall +``` + +## See also + +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md new file mode 100644 index 000000000..8b18f05be --- /dev/null +++ b/docs/docs/command/disk-accelerator.md @@ -0,0 +1,72 @@ +--- +title: "miren disk accelerator" +sidebar_label: "disk accelerator" +description: "Faster block-device disks via the lbd kernel module" +--- + +# miren disk accelerator + +Faster block-device disks via the lbd kernel module + +Miren serves block-device disks in one of two modes. + +**Universal mode** is the default and works everywhere. It backs each disk with a +loop device, which the Linux kernel provides out of the box. + +**Accelerator mode** uses `lbd`, a Miren kernel module that puts a +write-ahead log in front of the disk. It is faster, and it is what continuous +backup to Miren Cloud is built on. + +`lbd` is not part of the Linux kernel, so it has to be compiled for the +exact kernel each node is running. `miren disk accelerator install` does +that: your cluster builds the toolchain image with the BuildKit and registry it +already runs, the named node pulls it from there, and the module is compiled and +loaded on that node. Nothing is downloaded from us, and there is no published +image to keep up to date. + +## Getting started + +```bash +miren disk accelerator install runner1 # build and load it on runner1 +sudo systemctl restart miren # on runner1, to pick up the mode +``` + +`install` names the node to install on, but `status` and +`uninstall` read and change the host you run them on. To check a node +before or after installing, run `status` on that node. + +## Requirements + +- The kernel headers for your running kernel. On Debian and Ubuntu the builder + fetches them itself if the host has none. Everywhere else you install them + first, and `status` names the package -- `kernel-devel-$(uname -r)` + on Fedora and RHEL. +- Secure Boot disabled. A self-built module is unsigned, and firmware with Secure + Boot enforcing will refuse to load it. +- A kernel built with GCC. Clang-built kernels are not supported. + +## After a kernel upgrade + +A module only loads on the kernel it was built for. Once a host has installed the +module, Miren notices on startup that the running kernel has changed and rebuilds +it. You can also do it by hand with +`miren disk accelerator install --force`. + +Until the module is back, disks fall back to universal mode. Nothing breaks; they +are just slower. + +## Usage + +```bash +miren disk accelerator [flags] +``` + +## Subcommands + +- [`miren disk accelerator install`](./disk-accelerator-install.md) — Build and load the lbd kernel module for this kernel +- [`miren disk accelerator status`](./disk-accelerator-status.md) — Show whether accelerator mode can run on this host +- [`miren disk accelerator uninstall`](./disk-accelerator-uninstall.md) — Unload and remove the lbd kernel module + +## See also + +- [`miren disk`](./disk.md) diff --git a/docs/docs/command/disk.md b/docs/docs/command/disk.md index b31b81d5d..db909da97 100644 --- a/docs/docs/command/disk.md +++ b/docs/docs/command/disk.md @@ -1,12 +1,12 @@ --- title: "miren disk" sidebar_label: "disk" -description: "Disk backup and recovery" +description: "Disk backup, recovery, and acceleration" --- # miren disk -Disk backup and recovery +Disk backup, recovery, and acceleration ## Usage @@ -16,6 +16,7 @@ miren disk [flags] ## Subcommands +- [`miren disk accelerator`](./disk-accelerator.md) — Faster block-device disks via the lbd kernel module - [`miren disk backup`](./disk-backup.md) — Backup a disk to a snapshot file - [`miren disk list-deleted`](./disk-list-deleted.md) — List deleted disks available for recovery - [`miren disk restore`](./disk-restore.md) — Restore a disk from a snapshot file diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 2b3681a8d..b8578b0a4 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -114,7 +114,11 @@ Complete reference for all `miren` CLI commands. | Command | Description | |---------|-------------| -| [`miren disk`](./command/disk.md) | Disk backup and recovery | +| [`miren disk`](./command/disk.md) | Disk backup, recovery, and acceleration | +| [`miren disk accelerator`](./command/disk-accelerator.md) | Faster block-device disks via the lbd kernel module | +| [`miren disk accelerator install`](./command/disk-accelerator-install.md) | Build and load the lbd kernel module for this kernel | +| [`miren disk accelerator status`](./command/disk-accelerator-status.md) | Show whether accelerator mode can run on this host | +| [`miren disk accelerator uninstall`](./command/disk-accelerator-uninstall.md) | Unload and remove the lbd kernel module | | [`miren disk backup`](./command/disk-backup.md) | Backup a disk to a snapshot file | | [`miren disk list-deleted`](./command/disk-list-deleted.md) | List deleted disks available for recovery | | [`miren disk restore`](./command/disk-restore.md) | Restore a disk from a snapshot file | diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md new file mode 100644 index 000000000..a8bde9af1 --- /dev/null +++ b/docs/docs/disk-accelerator.md @@ -0,0 +1,151 @@ +--- +title: Disk Accelerator +description: Turn on accelerator mode for Miren Disks by building and loading the lbd kernel module against your node's running kernel. +keywords: [accelerator, lbd, kernel module, disk performance, disk mode, loop device] +--- + +# Disk Accelerator + +Miren serves [Miren Disks](./disks.md#miren-disks) in one of two modes, and picks +between them on its own. + +**Universal mode** is the default and works everywhere. It backs each disk with a +*loop device*, which is the kernel's built-in way of presenting a file as though +it were a disk. Nothing to install, no configuration. + +**Accelerator mode** uses `lbd`, a Miren kernel module that puts a write-ahead +log in front of the disk. Writes land in the log first and are folded into the +disk image behind them, which makes writes faster and gives Miren an exact, +ordered record of every change. That record is what continuous backup to Miren +Cloud is built on. + +`lbd` is not part of Linux, so it has to be compiled for the exact kernel your +node is running. One command does that. + +## Minimum working example + +```bash +miren disk accelerator install runner1 # build and load the module there +sudo systemctl restart miren # on that node, to pick up the mode +``` + +`install` names the node to install on, because the module has to be compiled +against the kernel that node is running. Your cluster builds the toolchain image +itself, using the same BuildKit and registry it already uses for your apps, and +the node pulls it from there. Nothing is downloaded from us, and there is no +image to keep up to date. + +The toolchain lives in that image, so on Debian and Ubuntu there is nothing to +install on the host first — if it has no kernel headers, the builder fetches +them for itself. + +Once the server restarts, new disks use accelerator mode. Existing disks keep +whatever mode they were created with. + +## Requirements + +| Requirement | Why | If it is missing | +|---|---|---| +| Kernel headers for the running kernel | The module is compiled against them | On Debian and Ubuntu the builder fetches them; elsewhere `status` names the package | +| Secure Boot off | A self-built module is unsigned, and enforcing firmware refuses it | `install` stops and says so | +| A GCC-built kernel | The builder ships GCC, not Clang | `install` stops and says so | +| Root | Loading a kernel module needs it | Run under `sudo` | + +### About the headers + +The builder image is Debian-based, so on a Debian or Ubuntu host it can install +`linux-headers-$(uname -r)` for itself and you need nothing on the host. That +needs a route to the distribution's package archive from the node, and it can +still come up empty for a kernel too new or too old to be in the archive — the +build says so, naming the package it could not find. + +On any other distribution, install the headers yourself first. +`miren disk accelerator status` prints the exact package name; it is +`kernel-devel-$(uname -r)` on Fedora and RHEL. + +Installing the headers on the host is always the faster path, because the +builder then borrows them read-only instead of downloading them, and needs no +network at all. + +## Checking what is going on + +`status` reads the host you run it on, not the node you last installed to. To +see how a runner is doing, run it on that runner. + +```bash +miren disk accelerator status +``` + +``` +Available yes +State lbd v0.0.0-20260824210626-be4cec661034 is loaded for kernel 6.8.0-51-generic +Kernel 6.8.0-51-generic +Module loaded yes +Control device yes +Module installed yes +lbdctl /usr/local/bin/lbdctl +Kernel headers /lib/modules/6.8.0-51-generic/build +Bundled lbd version v0.0.0-20260824210626-be4cec661034 +``` + +`--format json` gives the same thing as machine-readable fields. + +**Available** is the answer to the only question that matters: can this node +serve accelerator disks right now. It is true only when the module is loaded, +its control device exists, and `lbdctl` is installed to drive it. Any one of +those missing puts disks back on loop devices. + +## After a kernel upgrade + +A module only loads on the kernel it was built for, so a kernel upgrade leaves +the installed module unusable. + +Miren handles this. On startup it notices the running kernel no longer matches +the module it built, and rebuilds. You do not have to do anything, though you +can force it by hand: + +```bash +miren disk accelerator install runner1 --force +``` + +This only happens on hosts that installed the module in the first place. A host +that never turned accelerator mode on never pays for an unattended compile at +startup. + +When a Miren upgrade bundles a newer `lbd` but the installed module still +works, startup leaves it running. Replacing a working module automatically +could strand disks if the new one fails to load. Drain disks from the node, +then run `miren disk accelerator install ` to upgrade it deliberately. + +The startup rebuild is capped at ten minutes. If it runs over, the server logs a +warning, carries on booting, and the node stays on loop devices until you run +`install` yourself. Accelerator mode is a speed-up, so a build that will not +finish must never hold up a node coming back after a reboot. + +Until the module is back, disks fall back to universal mode. Nothing breaks; +they are just slower. + +## Choosing the mode yourself + +Auto-detection can be overridden in the server config: + +```toml title="/etc/miren/server.toml" +disk_mode = "universal" # or "accelerator", or "auto" (the default) +``` + +`universal` forces loop devices even where the module is loaded. `accelerator` +insists on `lbd`, and disks will fail to attach if it is not there — useful when +you would rather find out loudly than quietly run slower. See +[Server Configuration](./server-config.md). + +## Turning it off + +```bash +sudo miren disk accelerator uninstall +sudo systemctl restart miren +``` + +This unloads the module, removes it along with `lbdctl`, and forgets that the +host ever had it, so nothing rebuilds it later. It fails if a disk is still +attached — the kernel will not unload a module in use. Stop the apps holding +disks first. diff --git a/docs/docs/disks.md b/docs/docs/disks.md index a1b34667e..0fffbaeec 100644 --- a/docs/docs/disks.md +++ b/docs/docs/disks.md @@ -103,6 +103,8 @@ Miren Disks live on your server. Back up important data with `miren disk backup` Miren Disks provide managed persistent storage for your applications. Disks are provisioned with a specific size and filesystem, support exclusive leasing for data consistency, and persist across app restarts and redeployments. +They work out of the box on any Linux host. For faster writes, [accelerator mode](./disk-accelerator.md) builds and loads a Miren kernel module tuned for this job; it is one command, and disks fall back to the default cleanly if you skip it. + ### Why Use Disks? - **Managed lifecycle**: Miren handles disk creation, formatting, and attachment automatically diff --git a/docs/docs/system-requirements.md b/docs/docs/system-requirements.md index 6d6a8d84e..32c4f99c1 100644 --- a/docs/docs/system-requirements.md +++ b/docs/docs/system-requirements.md @@ -40,7 +40,9 @@ sudo dnf install iptables nftables `miren server install` and `miren runner install` verify these are present before installing, so a missing tool stops the install with instructions rather than surfacing later as a broken network. :::note[Extra tooling for optional features] -Some features reach for more commands, installed automatically or only when you opt in. [Block-device volumes](./managing-disk-space.md) use disk tooling (`lbdctl`, `mkfs.*`, `blkid`) when a disk is provisioned, and on SELinux-enforcing hosts the installer uses `semanage` and `restorecon` to label the binary. Both paths degrade gracefully if the tools are absent, so you only need them if you use the corresponding feature. +Some features reach for more commands, installed automatically or only when you opt in. [Block-device volumes](./managing-disk-space.md) use disk tooling (`mkfs.*`, `blkid`) when a disk is provisioned, and on SELinux-enforcing hosts the installer uses `semanage` and `restorecon` to label the binary. Both paths degrade gracefully if the tools are absent, so you only need them if you use the corresponding feature. + +[Accelerator mode](./disk-accelerator.md) compiles a kernel module for your running kernel, so it needs that kernel's headers. On Debian and Ubuntu the builder fetches them itself; elsewhere install them first (`kernel-devel-$(uname -r)` on Fedora and RHEL). Without them, disks use loop devices instead — slower, but working. ::: ## Why these numbers? diff --git a/docs/sidebars.ts b/docs/sidebars.ts index ac3278e3c..7e980cec3 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -36,6 +36,7 @@ const sidebars: SidebarsConfig = { collapsed: false, items: [ 'disks', + 'disk-accelerator', 'addons', 'managing-disk-space', ], diff --git a/go.mod b/go.mod index f9368c78d..8d945089f 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 k8s.io/klog/v2 v2.130.1 - miren.dev/lbd v0.0.0-20260224020427-8914d8db2233 + miren.dev/lbd v0.0.0-20260824210626-be4cec661034 miren.dev/mflags v0.0.0-20260913023833-b11db8e63177 modernc.org/sqlite v1.45.0 sigs.k8s.io/knftables v0.0.21 diff --git a/go.sum b/go.sum index b011fe5dd..52ac0541c 100644 --- a/go.sum +++ b/go.sum @@ -1949,8 +1949,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -miren.dev/lbd v0.0.0-20260224020427-8914d8db2233 h1:9DxH7Dhnmu7hn1OA2JC5fHpLuVgAqBySws9GLNssLl4= -miren.dev/lbd v0.0.0-20260224020427-8914d8db2233/go.mod h1:+x9fy2p45csBnGUJdqxCUmzlUTCipoVDbv6zIapTgDA= +miren.dev/lbd v0.0.0-20260824210626-be4cec661034 h1:+wGXxapZA6prIbxGLlVsgaGqu29OgBe6OMUH+P+D8EM= +miren.dev/lbd v0.0.0-20260824210626-be4cec661034/go.mod h1:+x9fy2p45csBnGUJdqxCUmzlUTCipoVDbv6zIapTgDA= miren.dev/mflags v0.0.0-20260913023833-b11db8e63177 h1:7w7JpoGLuFwv3WQHDeYQHyruWA6y+SLNuZIO4LOZggk= miren.dev/mflags v0.0.0-20260913023833-b11db8e63177/go.mod h1:G1eQ/upWVdO6BGT6dlh5Yqjt+9ncH5RUAKX6UKi1F9Q= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= diff --git a/hack/dev-distributed b/hack/dev-distributed index 2a3e48a31..ac7474bb5 100755 --- a/hack/dev-distributed +++ b/hack/dev-distributed @@ -34,7 +34,13 @@ bootstrap_peer() { # build VCS stamping fails the build. Unlike the standalone dev flow # (hack/dev.sh), which builds as the host user, this path builds as root, so # mark /src safe. Mirrors hack/package-release.sh. - peer_exec "$peer" git config --global --add safe.directory /src + # + # Run it from / rather than /src. In a git worktree /src/.git is a file + # pointing at the parent repo, which is not mounted in the peer, so git + # resolves the repo before writing global config and exits 128 on a path it + # cannot see. Writing global config needs no repo, so stepping outside the + # worktree sidesteps it. + peer_exec "$peer" bash -c "cd / && git config --global --add safe.directory /src" # Build binary peer_exec "$peer" make bin/miren # Symlinks diff --git a/hack/sync-lbd-src.sh b/hack/sync-lbd-src.sh new file mode 100755 index 000000000..b4c1a1425 --- /dev/null +++ b/hack/sync-lbd-src.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Sync the lbd kernel module source from the miren.dev/lbd module into +# third_party/lbd, so it can be embedded in the miren binary and handed to the +# builder image. +# +# The version is whatever go.mod pins -- never a hardcoded string here. Run +# without arguments to update the checked-in tree; run with --check to verify it +# matches, which is what CI does. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Only src/ is generated. third_party/lbd also holds a hand-written README.md +# and embed.go, which this script must leave alone. +DEST="third_party/lbd/src" +MODULE="miren.dev/lbd" + +check_only=0 +if [ "${1:-}" = "--check" ]; then + check_only=1 +elif [ -n "${1:-}" ]; then + echo "usage: $0 [--check]" >&2 + exit 2 +fi + +version="$(go list -m -f '{{.Version}}' "$MODULE")" +if [ -z "$version" ]; then + echo "could not resolve the $MODULE version from go.mod" >&2 + exit 1 +fi + +# Ensure the module is in the cache, then ask go where it landed. The cache is +# read-only, so everything copied out of it needs its mode fixed up. +go mod download "$MODULE" +src="$(go list -m -f '{{.Dir}}' "$MODULE")/src" +if [ ! -d "$src" ]; then + echo "no src/ directory in $MODULE $version (looked in $src)" >&2 + exit 1 +fi + +staging="$(mktemp -d)" +trap 'rm -rf "$staging"' EXIT + +# Everything the module build needs, and nothing else: the module's own C, the +# vendored LZ4, the Makefile, and dkms.conf. README.md and test_lbd.sh are +# developer files that belong in the lbd repo, not in the binary. +for f in \ + Makefile \ + dkms.conf \ + lbd.h \ + lbd_main.c \ + lbd_qcow2.c \ + lbd_qcow2.h \ + lbd_qcow2_format.h \ + lbdctl.c \ + cbor_dec.h \ + cbor_enc.h \ + lz4_kcompat.h \ + lz4/lz4.c \ + lz4/lz4.h; do + if [ ! -f "$src/$f" ]; then + echo "$MODULE $version is missing src/$f" >&2 + echo "the file list in $0 needs updating to match the module" >&2 + exit 1 + fi + mkdir -p "$staging/$(dirname "$f")" + install -m 0644 "$src/$f" "$staging/$f" +done + +printf '%s\n' "$version" >"$staging/VERSION" + +if [ "$check_only" -eq 1 ]; then + if diff -ru "$DEST" "$staging"; then + echo "$DEST is in sync with $MODULE $version" + exit 0 + fi + echo >&2 + echo "$DEST does not match $MODULE $version -- run hack/sync-lbd-src.sh" >&2 + exit 1 +fi + +rm -rf "$DEST" +mkdir -p "$(dirname "$DEST")" +cp -R "$staging" "$DEST" +echo "synced $DEST from $MODULE $version" diff --git a/pkg/diskresolve/resolver.go b/pkg/diskresolve/resolver.go index f688b15ec..1661b9bd6 100644 --- a/pkg/diskresolve/resolver.go +++ b/pkg/diskresolve/resolver.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "os" - "os/exec" "path/filepath" "strings" @@ -15,6 +14,7 @@ import ( "miren.dev/runtime/api/storage/storage_v1alpha" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/lbdmod" "miren.dev/runtime/pkg/snapshot" ) @@ -380,10 +380,13 @@ func ParseFilesystem(fs string) storage_v1alpha.DiskFilesystem { } func DetectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { - if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { + switch os.Getenv("MIREN_DISK_MODE") { + case "accelerator": return storage_v1alpha.VM_ACCELERATOR + case "universal": + return storage_v1alpha.VM_UNIVERSAL } - if _, err := exec.LookPath("lbdctl"); err == nil { + if lbdmod.Available(lbdmod.HostOptions("")) { return storage_v1alpha.VM_ACCELERATOR } return storage_v1alpha.VM_UNIVERSAL diff --git a/pkg/diskresolve/resolver_test.go b/pkg/diskresolve/resolver_test.go index 4396a00b6..7b9cb3f55 100644 --- a/pkg/diskresolve/resolver_test.go +++ b/pkg/diskresolve/resolver_test.go @@ -20,6 +20,13 @@ import ( "miren.dev/runtime/pkg/rpc" ) +func TestDetectVolumeModeExplicitOverrides(t *testing.T) { + t.Setenv("MIREN_DISK_MODE", "universal") + require.Equal(t, storage_v1alpha.VM_UNIVERSAL, DetectVolumeMode()) + t.Setenv("MIREN_DISK_MODE", "accelerator") + require.Equal(t, storage_v1alpha.VM_ACCELERATOR, DetectVolumeMode()) +} + // faultRPC wraps an rpc.Client and fails the failAt-th (1-based) call to the // named entity-server method, simulating the partial-Finalize RPC failure the // bug report describes (one of Finalize's two writes succeeding and the other diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go new file mode 100644 index 000000000..a2eae5bc5 --- /dev/null +++ b/pkg/lbdmod/build.go @@ -0,0 +1,481 @@ +package lbdmod + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "miren.dev/runtime/components/ocireg" +) + +const ( + // buildContainerName is fixed so a build killed before its own cleanup + // leaves something the next run can find and remove. + buildContainerName = "miren-lbd-build" + + // lbdctlInstallDir is where lbdctl goes. It has to be somewhere on the + // server process's PATH: the release directory is prepended to + // containerd's PATH, not miren's. + lbdctlInstallDir = "/usr/local/bin" + + // modulesLoadConf makes the kernel load lbd at boot, so a reboot does not + // depend on miren starting first. + modulesLoadConf = "/etc/modules-load.d/lbd.conf" +) + +// Installer builds and installs the lbd kernel module on this host. +type Installer struct { + // Log receives progress and the builder container's output. + Log *slog.Logger + + // Builder runs the builder image. Required for Install; Uninstall does + // not need it. pkg/lbdmod/ctrbuild provides the containerd one. + Builder Builder + + // Options say where to read host state and keep the install record. + Options Options + + // Image is the toolchain image to build in. Empty falls back to the + // cluster registry reference, which is where the coordinator publishes it. + Image string +} + +// buildDir is the scratch directory a build works in. It is keyed by kernel +// and module version so a rebuild after a kernel upgrade cannot pick up stale +// object files from the previous kernel. +func (i *Installer) buildDir(release string) string { + key := fmt.Sprintf("%s-%s", SourceVersion(), release) + return filepath.Join(i.Options.dataPath(), "lbd", "build", key) +} + +func (i *Installer) image() string { + if i.Image != "" { + return i.Image + } + return BuilderImage(ocireg.Host) +} + +// Install compiles the module against the running kernel and loads it. It is +// safe to call when the module is already installed and current: that is +// reported as a no-op unless force is set. +// +// The caller must be root. +func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { + status, err := Probe(i.Options) + if err != nil { + return status, err + } + + if !force && status.Marker != nil && status.Available() && !status.Stale() { + i.Log.Info("lbd is already installed and current", "kernel", status.Host.KernelRelease) + return status, nil + } + + if err := i.checkCanBuild(status); err != nil { + return status, err + } + + // Held for the whole build-and-load, so a concurrent install cannot clear + // the build directory or delete the builder container out from under this + // one. Taken after the cheap checks so an obviously impossible install + // still fails with the real reason rather than a lock error. + lock, err := acquireBuildLock(i.Options.dataPath()) + if err != nil { + return status, err + } + defer lock.release() + + // Another process may have finished the very build this one was about to + // start while we waited to be let in. + if !force { + if current, err := Probe(i.Options); err == nil && current.Marker != nil && current.Available() && !current.Stale() { + i.Log.Info("another process installed lbd while this one waited", + "kernel", current.Host.KernelRelease) + return current, nil + } + } + + if err := i.build(ctx, status.Host); err != nil { + return status, err + } + + if err := i.load(ctx, status.Host); err != nil { + return status, err + } + + // Prove the module is usable before recording the install. Writing the + // marker first would leave a record claiming success behind a failure, so + // `status` would show an installed version and build time for something + // that never worked. Availability does not depend on the marker, so this + // check is meaningful without it. + verified, err := Probe(i.Options) + if err != nil { + return status, err + } + if !verified.Available() { + return verified, fmt.Errorf("lbd was built and loaded but is still not usable: %s", verified.Explain()) + } + + marker := Marker{ + LbdVersion: SourceVersion(), + KernelRelease: status.Host.KernelRelease, + BuilderImage: i.image(), + ModulePath: modulePath(status.Host.KernelRelease), + LbdctlPath: filepath.Join(lbdctlInstallDir, "lbdctl"), + BuiltAt: time.Now().UTC(), + } + if err := writeMarker(i.Options.dataPath(), marker); err != nil { + return verified, err + } + + // Re-probe so the caller gets a status that includes the record just + // written, which is what `status` renders. + after, err := Probe(i.Options) + if err != nil { + return verified, err + } + + i.Log.Info("lbd installed", + "kernel", after.Host.KernelRelease, + "version", SourceVersion(), + "module", marker.ModulePath) + return after, nil +} + +// EnsureCurrent rebuilds an installed module when it is no longer usable. +// An older but working module is left alone: swapping it unattended after a +// miren upgrade could strand disks if the new module fails to load. +// +// A host with no install record is left alone: it never opted into accelerator +// mode, so it should not pay for an unattended compile at startup. It reports +// whether it rebuilt. +func (i *Installer) EnsureCurrent(ctx context.Context) (bool, error) { + status, err := Probe(i.Options) + if err != nil { + return false, err + } + + if status.Available() { + if status.Stale() { + i.Log.Warn("lbd is usable but out of date; defer the upgrade until disks can be detached", + "reason", status.Explain(), "upgrade_with", "miren disk accelerator install ") + } + return false, nil + } + + if status.Marker == nil { + return false, nil + } + + if i.Image == "" { + i.Image = status.Marker.BuilderImage + } + i.Log.Info("rebuilding the lbd kernel module", "reason", status.Explain()) + if _, err := i.Install(ctx, false); err != nil { + return false, err + } + return true, nil +} + +// checkCanBuild refuses the cases where a build would either fail confusingly +// or produce a module that cannot be loaded, and says why. +func (i *Installer) checkCanBuild(status Status) error { + if os.Geteuid() != 0 { + return fmt.Errorf("installing a kernel module requires root privileges (use sudo)") + } + + if i.Builder == nil { + return fmt.Errorf("no container runtime to run the lbd builder in") + } + + if err := checkKernelTools(); err != nil { + return err + } + + return i.checkCompilerAndHeaders(status) +} + +// checkKernelTools makes sure the commands that load a module are present. +// +// Checked before the build rather than after. The build takes minutes, and +// failing at the end on a missing binary wastes all of it and reads like the +// compile itself went wrong. Found on a container that had /lib/modules +// mounted but no kmod installed. +func checkKernelTools() error { + for _, tool := range []string{"depmod", "modprobe"} { + if _, err := exec.LookPath(tool); err != nil { + return fmt.Errorf("%s is not installed, and loading a kernel module needs it: install kmod", tool) + } + } + return nil +} + +// checkCompilerAndHeaders covers the host conditions that make a build +// pointless: firmware that will refuse the result, a toolchain we cannot +// match, or no build tree to compile against. +func (i *Installer) checkCompilerAndHeaders(status Status) error { + if secureBootEnforcing(i.Options.root()) { + return fmt.Errorf("this host has Secure Boot enabled, which refuses unsigned kernel modules. " + + "miren cannot sign the module, so accelerator mode needs Secure Boot disabled or a signed module from your distribution") + } + + if status.Host.Compiler.Name == "clang" { + return fmt.Errorf("this kernel was built with %s, which the lbd builder does not support", + status.Host.Compiler) + } + + if status.Host.HeadersDir == "" && !status.Host.CanFetchHeaders() { + return fmt.Errorf("no kernel headers for %s on this host: %s", + status.Host.KernelRelease, status.Host.InstallHint()) + } + + return nil +} + +// build runs the builder container and leaves lbd.ko and lbdctl in the build +// directory's out/ subdirectory. +func (i *Installer) build(ctx context.Context, host Host) error { + dir := i.buildDir(host.KernelRelease) + srcDir := filepath.Join(dir, "src") + outDir := filepath.Join(dir, "out") + + // Start from clean source every time. A retry after a failed build must + // not inherit half-written object files. + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("clearing the build directory %s: %w", dir, err) + } + if err := materializeSource(srcDir); err != nil { + return err + } + if err := os.MkdirAll(outDir, 0755); err != nil { + return fmt.Errorf("creating %s: %w", outDir, err) + } + + i.Log.Info("building the lbd kernel module", + "kernel", host.KernelRelease, + "headers", host.HeadersDir, + "version", SourceVersion()) + + spec := BuildSpec{ + Name: buildContainerName, + Image: i.image(), + Args: []string{"/usr/local/bin/build-lbd"}, + Env: []string{ + "KERNEL_RELEASE=" + host.KernelRelease, + "KERNEL_HEADERS=" + host.HeadersDir, + "HOST_DISTRO_ID=" + host.DistroID, + "HOST_DISTRO_LIKE=" + strings.Join(host.DistroLike, " "), + }, + Mounts: []Mount{ + {Destination: "/src", Source: srcDir}, + {Destination: "/out", Source: outDir}, + }, + } + + if host.HeadersDir != "" { + // Mounted at their real paths, not under a prefix: a kernel build + // tree is full of absolute symlinks (/lib/modules//build usually + // points into /usr/src) and they only resolve if the paths match the + // host's. + spec.Mounts = append(spec.Mounts, + Mount{Destination: "/lib/modules", Source: "/lib/modules", ReadOnly: true}, + Mount{Destination: "/usr/src", Source: "/usr/src", ReadOnly: true}, + ) + } else { + // No build tree on the host, so the builder installs one for itself. + // Those paths are left unmounted precisely so it can write to them, + // and it needs a network to reach the distro archive. + spec.Env = append(spec.Env, "FETCH_HEADERS=1") + spec.HostNetwork = true + } + + if err := i.Builder.Build(ctx, spec); err != nil { + return err + } + + for _, name := range []string{"lbd.ko", "lbdctl"} { + if _, err := os.Stat(filepath.Join(outDir, name)); err != nil { + return fmt.Errorf("the build reported success but produced no %s", name) + } + } + return nil +} + +// uninstallPaths lists the files an install left on the host, taken from what +// it recorded rather than from the current state. +// +// Both details matter. The module path has to come from the marker, because +// after a kernel upgrade the running kernel is no longer the one the module +// was built for, and deriving the path from it would miss the real artifact +// and orphan it. lbdctl has to come from the marker too, because the lbd +// repo's README tells people to install their own at the same location, and a +// path we never recorded is not ours to delete. +func uninstallPaths(m *Marker) []string { + if m == nil { + return nil + } + + var paths []string + for _, p := range []string{m.ModulePath, m.LbdctlPath} { + if p != "" { + paths = append(paths, p) + } + } + return append(paths, modulesLoadConf) +} + +// installedLbdctl reports whether a previous miren install is what put lbdctl +// at this path. +func (i *Installer) installedLbdctl(path string) bool { + marker, err := readMarker(i.Options.dataPath()) + if err != nil || marker == nil { + return false + } + return marker.LbdctlPath == path +} + +// load installs the built artifacts and brings the module up. +func (i *Installer) load(ctx context.Context, host Host) error { + outDir := filepath.Join(i.buildDir(host.KernelRelease), "out") + + // Unload before touching anything on disk. rmmod refuses while a device is + // attached, and that is the common case rather than a rare one -- a node + // with a running app that has a disk. Overwriting lbd.ko and lbdctl first + // and only then discovering the refusal would leave a userspace lbdctl + // talking to a kernel module built from different source, with no marker + // written to say so. Failing here leaves the host exactly as it was. + if isModuleLoaded(i.Options.root(), ModuleName) { + i.Log.Info("unloading the previous lbd module") + if out, err := exec.CommandContext(ctx, "rmmod", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("could not unload the running lbd module, which is usually because a disk is still attached: %w: %s", + err, strings.TrimSpace(string(out))) + } + } + + dest := modulePath(host.KernelRelease) + if err := installFile(filepath.Join(outDir, "lbd.ko"), dest, 0644); err != nil { + return err + } + + // The lbd repo's own README tells people to install lbdctl here by hand, + // so an existing binary may well be theirs rather than a previous install + // of ours. Replacing it is still the right move -- lbdctl and the module + // have to come from the same source -- but it should not happen silently. + lbdctl := filepath.Join(lbdctlInstallDir, "lbdctl") + if _, err := os.Stat(lbdctl); err == nil && !i.installedLbdctl(lbdctl) { + i.Log.Warn("replacing an lbdctl that miren did not install", "path", lbdctl) + } + if err := installFile(filepath.Join(outDir, "lbdctl"), lbdctl, 0755); err != nil { + return err + } + + // depmod rebuilds the dependency index modprobe consults; without it + // modprobe cannot find a module that was just dropped into extra/. + if out, err := exec.CommandContext(ctx, "depmod", "-a", host.KernelRelease).CombinedOutput(); err != nil { + return fmt.Errorf("depmod failed: %w: %s", err, strings.TrimSpace(string(out))) + } + + if out, err := exec.CommandContext(ctx, "modprobe", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("modprobe %s failed: %w: %s", ModuleName, err, strings.TrimSpace(string(out))) + } + if err := EnsureControlDevice(); err != nil { + return fmt.Errorf("lbd control device: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(modulesLoadConf), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(modulesLoadConf), err) + } + if err := os.WriteFile(modulesLoadConf, []byte(ModuleName+"\n"), 0644); err != nil { + return fmt.Errorf("writing %s: %w", modulesLoadConf, err) + } + + return nil +} + +// Uninstall unloads the module and removes everything the install put on the +// host, including the record that would otherwise trigger a rebuild later. +func (i *Installer) Uninstall(ctx context.Context) error { + if os.Geteuid() != 0 { + return fmt.Errorf("removing a kernel module requires root privileges (use sudo)") + } + + status, err := Probe(i.Options) + if err != nil { + return err + } + + if status.Marker == nil { + return fmt.Errorf("miren did not install lbd on this host, so there is nothing to remove") + } + + if status.Loaded { + if out, err := exec.CommandContext(ctx, "rmmod", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("could not unload lbd, which is usually because a disk is still attached: %w: %s", + err, strings.TrimSpace(string(out))) + } + } + + for _, path := range uninstallPaths(status.Marker) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing %s: %w", path, err) + } + } + + // depmod has to reindex the kernel the module was installed for, which + // after an upgrade is not the one running now. + if out, err := exec.CommandContext(ctx, "depmod", "-a", status.Marker.KernelRelease).CombinedOutput(); err != nil { + i.Log.Warn("depmod failed after removing lbd", "error", err, + "kernel", status.Marker.KernelRelease, "output", strings.TrimSpace(string(out))) + } + + if err := os.RemoveAll(filepath.Join(i.Options.dataPath(), "lbd", "build")); err != nil { + i.Log.Warn("failed to remove the lbd build directory", "error", err) + } + + return removeMarker(i.Options.dataPath()) +} + +// installFile copies src to dest, creating the destination directory. It writes +// to a temporary name and renames, so a reader never sees a half-written module. +func installFile(src, dest string, mode os.FileMode) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("reading %s: %w", src, err) + } + + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(dest), err) + } + + tmp := dest + ".tmp" + if err := os.WriteFile(tmp, data, mode); err != nil { + return fmt.Errorf("writing %s: %w", tmp, err) + } + if err := os.Rename(tmp, dest); err != nil { + os.Remove(tmp) + return fmt.Errorf("installing %s: %w", dest, err) + } + return nil +} + +// secureBootEnforcing reports whether the firmware will refuse unsigned +// modules. The efivars file carries a five-byte value whose last byte is the +// flag; anything we cannot read is treated as "not enforcing", since guessing +// yes would block hosts that are simply not using EFI. +func secureBootEnforcing(root string) bool { + matches, err := filepath.Glob(filepath.Join(root, "sys/firmware/efi/efivars/SecureBoot-*")) + if err != nil || len(matches) == 0 { + return false + } + + data, err := os.ReadFile(matches[0]) + if err != nil || len(data) == 0 { + return false + } + return data[len(data)-1] == 1 +} diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go new file mode 100644 index 000000000..c0789e804 --- /dev/null +++ b/pkg/lbdmod/build_test.go @@ -0,0 +1,535 @@ +package lbdmod + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testInstaller(t *testing.T, root, dataPath string) *Installer { + t.Helper() + return &Installer{ + Log: slog.New(slog.DiscardHandler), + Options: Options{Root: root, DataPath: dataPath}, + } +} + +func TestCheckCanBuildRefusesClangKernels(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "proc/version", + "Linux version "+testRelease+" (build@) (Android clang version 17.0.4, LLD 17.0.4) #1 SMP\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + // Root and containerd are checked first, so exercise the compiler rule + // directly rather than depending on how the test runner is invoked. + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "built with clang-17") +} + +func TestCheckCanBuildNeedsHeadersOffTheDebianFamily(t *testing.T) { + // The builder image is Debian-based, so it can only fetch headers from a + // Debian-family archive. Anywhere else the operator has to install them, + // and the message has to name the package or they have to go looking. + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.11.4-301.fc41.x86_64\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel headers") + assert.Contains(t, err.Error(), "dnf install kernel-devel-6.11.4-301.fc41.x86_64") +} + +func TestCheckCanBuildLetsDebianFamilyHostsFetchHeaders(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", testRelease+"\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + require.Empty(t, status.Host.HeadersDir) + + // Missing headers are not fatal here: the builder installs them itself. + require.NoError(t, testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status)) +} + +func TestBuildAsksTheBuilderToFetchHeadersWhenTheHostHasNone(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + + host := Host{KernelRelease: testRelease, DistroID: "ubuntu", DistroLike: []string{"debian"}} + require.NoError(t, i.build(t.Context(), host)) + + assert.Contains(t, builder.spec.Env, "FETCH_HEADERS=1") + assert.True(t, builder.spec.HostNetwork, "fetching headers needs to reach the archive") + + // /lib/modules and /usr/src are deliberately left unmounted so the + // builder can install headers into its own filesystem. + for _, m := range builder.spec.Mounts { + assert.NotEqual(t, "/lib/modules", m.Destination) + assert.NotEqual(t, "/usr/src", m.Destination) + } +} + +func TestBuildAgainstHostHeadersNeedsNoNetwork(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + + host := Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"} + require.NoError(t, i.build(t.Context(), host)) + + assert.False(t, builder.spec.HostNetwork, "a build against host headers should reach nothing") + assert.NotContains(t, builder.spec.Env, "FETCH_HEADERS=1") +} + +func TestCheckCanBuildAcceptsAGoodHost(t *testing.T) { + root := ubuntuRoot(t) + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + require.NoError(t, testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status)) +} + +func TestInstallDoesNotSkipALoadedModuleWithoutAnInstallRecord(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 0 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + status, err := Probe(i.Options) + require.NoError(t, err) + require.True(t, status.Available()) + require.Nil(t, status.Marker) + + // A load followed by a failed verification leaves no marker. It must not + // report the next install as successful without recording opt-in for rebuilds. + _, err = i.Install(t.Context(), false) + require.Error(t, err, "a missing builder must prevent completing the install") + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + _, err = i.Install(t.Context(), false) + require.NoError(t, err, "a completed install can skip the build") +} + +func TestSecureBootDetection(t *testing.T) { + // No EFI at all: not enforcing, rather than guessing yes and blocking a + // host that simply is not using EFI. + assert.False(t, secureBootEnforcing(t.TempDir())) + + off := t.TempDir() + writeFile(t, off, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 0})) + assert.False(t, secureBootEnforcing(off)) + + on := t.TempDir() + writeFile(t, on, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 1})) + assert.True(t, secureBootEnforcing(on)) +} + +func TestSecureBootBlocksTheBuild(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 1})) + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "Secure Boot") +} + +func TestBuildDirIsKeyedByKernelAndVersion(t *testing.T) { + i := testInstaller(t, "/", "/var/lib/miren") + + // A rebuild after a kernel upgrade must not reuse the old kernel's object + // files, so the two kernels get separate directories. + assert.NotEqual(t, i.buildDir("6.8.0-51-generic"), i.buildDir("6.8.0-52-generic")) + assert.Contains(t, i.buildDir("6.8.0-51-generic"), SourceVersion()) + assert.Contains(t, i.buildDir("6.8.0-51-generic"), "6.8.0-51-generic") +} + +func TestInstallFileIsAtomicAndCreatesParents(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "lbd.ko") + require.NoError(t, os.WriteFile(src, []byte("module"), 0644)) + + dest := filepath.Join(dir, "lib", "modules", testRelease, "extra", "lbd.ko") + require.NoError(t, installFile(src, dest, 0644)) + + data, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "module", string(data)) + + info, err := os.Stat(dest) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0644), info.Mode().Perm()) + + // No temporary file is left behind. + _, err = os.Stat(dest + ".tmp") + assert.True(t, os.IsNotExist(err)) + + // Overwriting an existing module works, which is the kernel-upgrade path. + require.NoError(t, os.WriteFile(src, []byte("newer"), 0644)) + require.NoError(t, installFile(src, dest, 0644)) + data, err = os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "newer", string(data)) +} + +func TestInstallFileReportsAMissingSource(t *testing.T) { + dir := t.TempDir() + err := installFile(filepath.Join(dir, "absent"), filepath.Join(dir, "dest"), 0644) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading") +} + +// fakeBuilder stands in for the container runtime. onBuild may write into the +// spec's /out mount to imitate a successful compile. +type fakeBuilder struct { + spec BuildSpec + called bool + err error + onBuild func(spec BuildSpec) error +} + +func (f *fakeBuilder) Build(_ context.Context, spec BuildSpec) error { + f.called = true + f.spec = spec + if f.err != nil { + return f.err + } + if f.onBuild != nil { + return f.onBuild(spec) + } + return nil +} + +// hostMount finds a mount by its path inside the container. +func hostMount(t *testing.T, spec BuildSpec, dest string) Mount { + t.Helper() + for _, m := range spec.Mounts { + if m.Destination == dest { + return m + } + } + t.Fatalf("no mount at %s", dest) + return Mount{} +} + +// produceArtifacts imitates a builder that compiled successfully. +func produceArtifacts(t *testing.T) func(BuildSpec) error { + t.Helper() + return func(spec BuildSpec) error { + out := hostMount(t, spec, "/out").Source + for _, name := range []string{"lbd.ko", "lbdctl"} { + if err := os.WriteFile(filepath.Join(out, name), []byte(name), 0644); err != nil { + return err + } + } + return nil + } +} + +func TestBuildHandsTheBuilderSourceAndHeaders(t *testing.T) { + dataPath := t.TempDir() + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + + i := testInstaller(t, ubuntuRoot(t), dataPath) + i.Builder = builder + + host := Host{ + KernelRelease: testRelease, + HeadersDir: "/lib/modules/" + testRelease + "/build", + DistroID: "ubuntu", + DistroLike: []string{"debian"}, + } + require.NoError(t, i.build(t.Context(), host)) + require.True(t, builder.called) + + // The source is materialized into a writable directory, because the + // kernel build writes its object files next to the source. + src := hostMount(t, builder.spec, "/src") + assert.False(t, src.ReadOnly) + _, err := os.Stat(filepath.Join(src.Source, "lbd_main.c")) + require.NoError(t, err) + + // The host's kernel tree is mounted at its real path, so the absolute + // symlinks inside it resolve, and read-only so a build cannot damage it. + modules := hostMount(t, builder.spec, "/lib/modules") + assert.Equal(t, "/lib/modules", modules.Source) + assert.True(t, modules.ReadOnly) + assert.True(t, hostMount(t, builder.spec, "/usr/src").ReadOnly) + + assert.Contains(t, builder.spec.Env, "KERNEL_RELEASE="+testRelease) + assert.Contains(t, builder.spec.Env, "KERNEL_HEADERS=/lib/modules/"+testRelease+"/build") + assert.Contains(t, builder.spec.Env, "HOST_DISTRO_ID=ubuntu") + assert.Contains(t, builder.spec.Env, "HOST_DISTRO_LIKE=debian") +} + +func TestBuildRejectsABuilderThatProducedNothing(t *testing.T) { + // The module Makefile downgrades unresolved symbols to warnings, so a + // zero exit does not prove there is a module to install. + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + i.Builder = &fakeBuilder{} + + err := i.build(t.Context(), Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "produced no lbd.ko") +} + +func TestBuildSurfacesTheBuilderError(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + i.Builder = &fakeBuilder{err: &BuildFailedError{ExitCode: 2, Output: "error: no kernel headers"}} + + err := i.build(t.Context(), Host{KernelRelease: testRelease}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel headers") +} + +func TestBuildStartsFromCleanSource(t *testing.T) { + dataPath := t.TempDir() + i := testInstaller(t, ubuntuRoot(t), dataPath) + host := Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"} + + // Leave debris from a build that failed partway through. + stale := filepath.Join(i.buildDir(testRelease), "src", "lbd_main.o") + require.NoError(t, os.MkdirAll(filepath.Dir(stale), 0755)) + require.NoError(t, os.WriteFile(stale, []byte("stale object"), 0644)) + + i.Builder = &fakeBuilder{onBuild: produceArtifacts(t)} + require.NoError(t, i.build(t.Context(), host)) + + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err), "object files from a failed build must not survive into the retry") +} + +func TestBuildUsesTheConfiguredImage(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + i.Image = "example.test/lbd-builder:local" + + require.NoError(t, i.build(t.Context(), Host{KernelRelease: testRelease})) + assert.Equal(t, "example.test/lbd-builder:local", builder.spec.Image) +} + +func TestEnsureCurrentLeavesAHostThatNeverOptedInAlone(t *testing.T) { + // No install record means accelerator mode was never enabled here, so + // startup must not pay for an unattended compile. + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{} + i.Builder = builder + + rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) + assert.False(t, rebuilt) + assert.False(t, builder.called, "a host with no install record must not be built for") +} + +func TestEnsureCurrentSkipsAHealthyHost(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + builder := &fakeBuilder{} + i.Builder = builder + + rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) + assert.False(t, rebuilt) + assert.False(t, builder.called) +} + +func TestEnsureCurrentDefersAnOlderWorkingModule(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 0 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "older-lbd", KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + builder := &fakeBuilder{} + i.Builder = builder + status, err := Probe(i.Options) + require.NoError(t, err) + require.True(t, status.Available()) + require.True(t, status.Stale()) + + // A successful compile does not guarantee the new module will load. Keep the + // working version until an operator can drain disks and upgrade explicitly. + rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) + require.False(t, rebuilt) + require.False(t, builder.called) +} + +func TestEnsureCurrentRebuildsAfterAKernelUpgrade(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + // The host installed lbd for a kernel it is no longer running. + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: "6.8.0-45-generic", + ModulePath: modulePath("6.8.0-45-generic"), + })) + + i := testInstaller(t, root, dataPath) + builder := &fakeBuilder{} + i.Builder = builder + + // It decides to act, which is the point. Whether it then gets past the + // root check depends on how the tests were invoked, and the install needs + // a real kernel and depmod either way -- so what is asserted is that it + // did not quietly do nothing. + rebuilt, err := i.EnsureCurrent(t.Context()) + assert.False(t, rebuilt && err != nil, "a rebuild cannot both succeed and fail") + assert.True(t, err != nil || builder.called, + "a stale module must trigger a rebuild attempt, not silence") +} + +func TestEnsureCurrentUsesThePublishedImageFromTheInstallRecord(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + previousImage := "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-previous" + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), KernelRelease: "6.8.0-45-generic", BuilderImage: previousImage, + })) + + i := testInstaller(t, root, dataPath) + // No builder is needed to verify image selection: the install will fail + // its preflight check before it could touch the host kernel. + _, err := i.EnsureCurrent(t.Context()) + require.Error(t, err) + assert.Equal(t, previousImage, i.image(), "a new binary's toolchain tag may not have been published") + + // An explicit image from a coordinator install must still take precedence. + i.Image = "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-new" + _, err = i.EnsureCurrent(t.Context()) + require.Error(t, err) + assert.Equal(t, "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-new", i.image()) +} + +func TestBuildFailedErrorQuotesTheOutput(t *testing.T) { + err := &BuildFailedError{ExitCode: 2, Output: "error: no kernel headers for 6.8.0-51-generic"} + assert.Contains(t, err.Error(), "exit 2") + assert.Contains(t, err.Error(), "no kernel headers") + + bare := &BuildFailedError{ExitCode: 1} + assert.Equal(t, "the lbd build failed (exit 1)", bare.Error()) + assert.False(t, strings.HasSuffix(bare.Error(), ":\n")) +} + +func TestUninstallRemovesWhatWasInstalledNotWhatIsRunning(t *testing.T) { + // The host has moved on to a newer kernel since the install. Deriving the + // module path from the running kernel would miss the real artifact and + // leave it on disk forever. + installedKernel := "6.8.0-45-generic" + m := &Marker{ + LbdVersion: SourceVersion(), + KernelRelease: installedKernel, + ModulePath: modulePath(installedKernel), + LbdctlPath: "/usr/local/bin/lbdctl", + } + + paths := uninstallPaths(m) + + assert.Contains(t, paths, modulePath(installedKernel)) + assert.NotContains(t, paths, modulePath(testRelease), + "the running kernel's path was never installed") + assert.Contains(t, paths, "/usr/local/bin/lbdctl") + assert.Contains(t, paths, modulesLoadConf) +} + +func TestUninstallLeavesAnLbdctlItDidNotInstall(t *testing.T) { + // An operator who followed the lbd repo's README has their own lbdctl at + // the same path. A marker that never recorded one must not license + // deleting it. + m := &Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + } + + paths := uninstallPaths(m) + + assert.Contains(t, paths, modulePath(testRelease)) + for _, p := range paths { + assert.NotContains(t, p, "lbdctl", "an unrecorded lbdctl is not ours to remove") + } +} + +func TestUninstallPathsWithNoMarker(t *testing.T) { + // No install record means miren put nothing on this host. + assert.Empty(t, uninstallPaths(nil)) +} + +func TestCheckKernelToolsFailsBeforeAnExpensiveBuild(t *testing.T) { + // A container can have /lib/modules mounted and still lack kmod, which is + // how this was found. Catching it up front matters because the build takes + // minutes and the failure otherwise looks like the compile went wrong. + t.Setenv("PATH", t.TempDir()) + + err := checkKernelTools() + require.Error(t, err) + assert.Contains(t, err.Error(), "depmod is not installed") + assert.Contains(t, err.Error(), "install kmod", "the error has to say what to do about it") +} + +func TestCheckKernelToolsPassesWhenBothArePresent(t *testing.T) { + dir := t.TempDir() + for _, tool := range []string{"depmod", "modprobe"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, tool), []byte("#!/bin/sh\n"), 0755)) + } + t.Setenv("PATH", dir) + + require.NoError(t, checkKernelTools()) +} + +func TestCheckKernelToolsNeedsModprobeToo(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "depmod"), []byte("#!/bin/sh\n"), 0755)) + t.Setenv("PATH", dir) + + err := checkKernelTools() + require.Error(t, err) + assert.Contains(t, err.Error(), "modprobe is not installed") +} diff --git a/pkg/lbdmod/builder.go b/pkg/lbdmod/builder.go new file mode 100644 index 000000000..3408fa272 --- /dev/null +++ b/pkg/lbdmod/builder.go @@ -0,0 +1,69 @@ +package lbdmod + +import ( + "context" + "fmt" +) + +// Mount is a bind mount into the builder. +type Mount struct { + // Source is the path on the host. + Source string + + // Destination is where it appears inside the builder. + Destination string + + // ReadOnly keeps the builder from writing through the mount. + ReadOnly bool +} + +// BuildSpec describes one run of the builder image. +type BuildSpec struct { + // Name is the container's id. It is fixed rather than random so a build + // killed before its own cleanup leaves something the next run can find. + Name string + + // Image is the builder image reference. + Image string + + // Args replaces the image's entrypoint arguments. + Args []string + + // Env is added to the image's environment as "KEY=value" pairs. + Env []string + + // Mounts are bind mounts into the container. + Mounts []Mount + + // HostNetwork gives the builder the host's network and resolver. It is + // set only when the builder has to fetch kernel headers for itself; a + // build against a host's own headers needs no network at all. + HostNetwork bool +} + +// Builder runs the builder image once and waits for it to finish. It is an +// interface so this package stays free of container runtime dependencies -- +// components/diskio and controllers/disk import it only to ask whether lbd is +// available, and should not pull containerd along with them. The containerd +// implementation is pkg/lbdmod/ctrbuild. +type Builder interface { + // Build runs the container to completion and returns nil only if it + // exited zero. A non-zero exit should be reported as a BuildFailedError + // so the build output survives. + Build(ctx context.Context, spec BuildSpec) error +} + +// BuildFailedError reports a builder container that ran but exited non-zero. +// It carries the tail of the build output, which is where the real explanation +// lives. +type BuildFailedError struct { + ExitCode uint32 + Output string +} + +func (e *BuildFailedError) Error() string { + if e.Output == "" { + return fmt.Sprintf("the lbd build failed (exit %d)", e.ExitCode) + } + return fmt.Sprintf("the lbd build failed (exit %d):\n%s", e.ExitCode, e.Output) +} diff --git a/pkg/lbdmod/builder/Dockerfile b/pkg/lbdmod/builder/Dockerfile new file mode 100644 index 000000000..4ada89fbf --- /dev/null +++ b/pkg/lbdmod/builder/Dockerfile @@ -0,0 +1,65 @@ +# Toolchain for compiling the lbd kernel module against a node's running kernel. +# +# This is embedded in the miren binary and built by the cluster's own BuildKit +# into the cluster-local registry, not published by us. That is why it is here +# rather than under docker/: nothing outside miren ever builds it, and the tag +# is a hash of this directory, so it is rebuilt exactly when it changes. +# +# It deliberately does NOT contain the module source. miren embeds the source +# and bind-mounts it at /src, so a new module version ships as a new miren +# without touching this image at all. +# +# The contract with pkg/lbdmod: +# +# /src read-write module source; object files are written here +# /out read-write where lbd.ko and lbdctl are left +# /lib/modules read-only the host's module tree, mounted at its real +# /usr/src read-only path so the kernel build tree's absolute +# symlinks still resolve +# +# KERNEL_RELEASE the kernel to build for; defaults to uname -r, which is +# already the host's kernel since containers share it +# KERNEL_HEADERS the build tree to compile against; probed when unset +# HOST_DISTRO_ID the host's os-release ID, for error messages +# HOST_DISTRO_LIKE the host's os-release ID_LIKE + +FROM ubuntu:24.04 + +# bc, flex, bison, libelf-dev and libssl-dev are what the kernel build system +# reaches for; kmod gives us modinfo to verify what we produced. +# +# One gcc, not several. A module's ABI contract is its vermagic and symbol +# CRCs, both of which come from the kernel headers rather than the compiler -- +# DKMS has always built out-of-tree modules with whatever gcc the host happens +# to have. Carrying gcc-12 and gcc-14 as well cost ~300MB to silence a warning. +# build.sh still reports a major-version mismatch, and the Clang case, which is +# the one that genuinely fails to load, is refused outright. +# +# Deliberately not build-essential: that drags in g++ and libstdc++-dev (~50MB) +# to compile C++ we never write. libc6-dev and make are what it was wanted for. +# +# The dpkg excludes also apply to the kernel headers the builder installs for +# itself on the fetch path, so they keep paying off after image build. +RUN printf 'path-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-exclude=/usr/share/info/*\npath-exclude=/usr/share/lintian/*\npath-exclude=/usr/share/locale/*\npath-include=/usr/share/locale/en*\n' \ + > /etc/dpkg/dpkg.cfg.d/01-nodoc \ + && apt-get update && apt-get install -y --no-install-recommends \ + bc \ + binutils \ + bison \ + ca-certificates \ + flex \ + gcc-13 \ + kmod \ + libc6-dev \ + libelf-dev \ + libssl-dev \ + make \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* /usr/share/doc /usr/share/man + +# The build context is this directory, materialized from the embed, so the +# script sits next to the Dockerfile rather than under a docker/ prefix. +COPY build.sh /usr/local/bin/build-lbd +RUN chmod +x /usr/local/bin/build-lbd + +ENTRYPOINT ["/usr/local/bin/build-lbd"] diff --git a/pkg/lbdmod/builder/build.sh b/pkg/lbdmod/builder/build.sh new file mode 100755 index 000000000..856b1404a --- /dev/null +++ b/pkg/lbdmod/builder/build.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Compile the lbd kernel module and lbdctl against the running kernel. +# +# Runs inside the lbd-builder image; see docker/Dockerfile.lbd-builder for the +# mounts and environment it expects. Everything it prints is shown to the +# operator, so messages here are the ones they will act on. + +set -euo pipefail + +SRC=${SRC:-/src} +OUT=${OUT:-/out} + +fail() { + echo "error: $*" >&2 + exit 1 +} + +[ -d "$SRC" ] || fail "no module source at $SRC" +[ -f "$SRC/Makefile" ] || fail "$SRC has no Makefile; the source mount looks wrong" +mkdir -p "$OUT" + +# A container shares the host's kernel, so uname and /proc/version already +# describe the machine we are building for. +KERNEL_RELEASE=${KERNEL_RELEASE:-$(uname -r)} +[ -n "$KERNEL_RELEASE" ] || fail "could not determine the kernel release" + +# --------------------------------------------------------------------------- +# Kernel headers +# --------------------------------------------------------------------------- + +find_headers() { + local candidate + for candidate in \ + "/lib/modules/$KERNEL_RELEASE/build" \ + "/usr/src/kernels/$KERNEL_RELEASE" \ + "/usr/src/linux-headers-$KERNEL_RELEASE"; do + if [ -f "$candidate/Makefile" ]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +header_package() { + local id + for id in "${HOST_DISTRO_ID:-}" ${HOST_DISTRO_LIKE:-}; do + case "$id" in + debian | ubuntu) echo "linux-headers-$KERNEL_RELEASE"; return 0 ;; + fedora | rhel | centos) echo "kernel-devel-$KERNEL_RELEASE"; return 0 ;; + arch | alpine) echo "linux-headers"; return 0 ;; + suse | opensuse* | sles) echo "kernel-devel"; return 0 ;; + esac + done + return 1 +} + +KDIR=${KERNEL_HEADERS:-} +if [ -n "$KDIR" ] && [ ! -f "$KDIR/Makefile" ]; then + echo "warning: $KDIR is not a kernel build tree; looking elsewhere" >&2 + KDIR="" +fi +if [ -z "$KDIR" ]; then + KDIR=$(find_headers || true) +fi +# No build tree came in from the host. miren leaves /lib/modules and /usr/src +# unmounted in that case, so the builder can install headers into its own +# filesystem instead. This only reaches a Debian-family archive, because that is +# what this image is built from. +if [ -z "$KDIR" ] && [ "${FETCH_HEADERS:-0}" = "1" ]; then + pkg="linux-headers-$KERNEL_RELEASE" + echo "No kernel headers on the host; fetching $pkg" + + if ! apt-get update -qq; then + fail "could not reach the package archive to fetch $pkg. Install it on the host and try again." + fi + + if ! DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$pkg"; then + fail "$pkg is not available from the package archive, which usually means this kernel is too new or too old for it. Install the headers on the host and try again." + fi + + KDIR=$(find_headers || true) + if [ -z "$KDIR" ]; then + fail "$pkg installed but left no build tree for $KERNEL_RELEASE" + fi + echo "Fetched kernel headers into $KDIR" +fi + +if [ -z "$KDIR" ]; then + pkg=$(header_package || true) + if [ -n "$pkg" ]; then + fail "no kernel headers for $KERNEL_RELEASE. Install $pkg on the host and try again." + fi + fail "no kernel headers for $KERNEL_RELEASE. Install this kernel's headers on the host and try again." +fi + +echo "Building lbd for kernel $KERNEL_RELEASE against $KDIR" + +# --------------------------------------------------------------------------- +# Compiler +# +# A module should be built by roughly the compiler that built the kernel. A +# major-version gap is usually only a modpost warning, but a Clang-built kernel +# with control-flow integrity rejects a GCC-built module outright -- better to +# say so than to hand back a module that silently will not load. +# --------------------------------------------------------------------------- + +kernel_compiler_line=$(cat /proc/version 2>/dev/null || echo "") + +if echo "$kernel_compiler_line" | grep -qi 'clang version'; then + fail "this kernel was built with Clang, which the lbd builder does not support. Install the module from your distribution instead, or build it on the host." +fi + +want_major=$(echo "$kernel_compiler_line" | grep -oE '\bgcc[^0-9]*([0-9]+)' | grep -oE '[0-9]+' | head -1 || true) + +pick_gcc() { + local want=$1 candidate best="" + if [ -n "$want" ] && command -v "gcc-$want" >/dev/null 2>&1; then + echo "gcc-$want" + return 0 + fi + # No exact match: take the newest installed major and warn. Ordering here + # is oldest to newest so the last hit wins. + for candidate in 12 13 14 15 16; do + command -v "gcc-$candidate" >/dev/null 2>&1 && best="gcc-$candidate" + done + if [ -n "$best" ]; then + echo "$best" + return 0 + fi + command -v gcc >/dev/null 2>&1 && echo gcc && return 0 + return 1 +} + +CC=$(pick_gcc "$want_major") || fail "no C compiler in the builder image" + +if [ -n "$want_major" ] && [ "$CC" != "gcc-$want_major" ]; then + echo "warning: this kernel was built with gcc-$want_major but the builder only has $CC." >&2 + echo "warning: the module should still load, but report it if modprobe refuses it." >&2 +elif [ -z "$want_major" ]; then + echo "warning: could not tell which compiler built this kernel; using $CC" >&2 +fi + +echo "Compiling with $CC ($($CC -dumpversion))" + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +# The module's Makefile passes KBUILD_MODPOST_WARN=1, which downgrades +# unresolved symbols to warnings. That means a zero exit code alone does not +# prove the module will load, so the log is checked below. +log=$(mktemp) +if ! make -C "$SRC" KDIR="$KDIR" CC="$CC" lbd.ko 2>&1 | tee "$log"; then + fail "compiling lbd.ko failed" +fi + +if grep -q 'undefined!' "$log"; then + grep 'undefined!' "$log" >&2 + fail "lbd.ko references symbols this kernel does not export, so it would fail to load" +fi + +# lbdctl is linked statically: it runs on the host, whose libc is not the +# builder image's. +if ! make -C "$SRC" CC="$CC -static" lbdctl; then + fail "compiling lbdctl failed" +fi + +# --------------------------------------------------------------------------- +# Verify and hand back +# --------------------------------------------------------------------------- + +[ -f "$SRC/lbd.ko" ] || fail "the build reported success but produced no lbd.ko" +[ -f "$SRC/lbdctl" ] || fail "the build reported success but produced no lbdctl" + +# vermagic is what the kernel checks at load time. If it disagrees with the +# running kernel, modprobe will refuse the module, so catch it here where we +# can explain why. +vermagic=$(modinfo -F vermagic "$SRC/lbd.ko" 2>/dev/null || echo "") +case "$vermagic" in +"$KERNEL_RELEASE"*) ;; +"") echo "warning: could not read vermagic from lbd.ko" >&2 ;; +*) fail "lbd.ko was built for '$vermagic' but this host runs $KERNEL_RELEASE; the headers at $KDIR do not match the running kernel" ;; +esac + +install -m 0644 "$SRC/lbd.ko" "$OUT/lbd.ko" +install -m 0755 "$SRC/lbdctl" "$OUT/lbdctl" + +echo "Built lbd.ko ($(stat -c %s "$OUT/lbd.ko") bytes) and lbdctl for $KERNEL_RELEASE" diff --git a/pkg/lbdmod/builderimage.go b/pkg/lbdmod/builderimage.go new file mode 100644 index 000000000..a99b40d0b --- /dev/null +++ b/pkg/lbdmod/builderimage.go @@ -0,0 +1,161 @@ +package lbdmod + +import ( + "crypto/sha256" + "embed" + "encoding/hex" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/components/ocireg" +) + +// builderFS holds the toolchain image definition. It is embedded rather than +// published so the cluster builds its own copy: the image is a base plus a +// handful of packages and a script, and owning a released artifact to +// distribute that costs more than rebuilding it. +// +//go:embed all:builder +var builderFS embed.FS + +// BuilderRepository is the repository half of the builder's image reference. +// The registry resolves manifests by tag alone and ignores this, so it is here +// to make the reference readable in logs and `ctr images ls`. +const BuilderRepository = "miren-system/lbd-builder" + +// BuilderDockerfile is the Dockerfile's name inside the build context, which +// BuildKit's dockerfile frontend takes as its "filename" attribute. +const BuilderDockerfile = "Dockerfile" + +// BuilderTag is the tag the image is pushed under. +// +// The tag carries SystemArtifactPrefix because an artifact's entity name is +// the tag it was pushed under, and that name is the only thing artifact GC has +// to tell a system image apart from a genuinely orphaned one. Without the +// prefix the toolchain image is archived within the hour and its blobs +// deleted. The rest is a content hash, so the tag moves exactly when the +// toolchain does. +func BuilderTag() string { + return core_v1alpha.SystemArtifactPrefix + "lbd-builder-" + BuilderVersion() +} + +// BuilderVersion is a content hash of the toolchain definition, used as the +// image tag. Hashing the content rather than tagging by hand means the image +// is rebuilt exactly when the Dockerfile or its build script changes, and +// never otherwise. +func BuilderVersion() string { + sum, err := hashFS(builderFS, "builder") + if err != nil { + // The tree is embedded at compile time, so a walk over it cannot fail + // for any reason a caller could act on. + panic(fmt.Sprintf("hashing the embedded lbd builder: %v", err)) + } + return sum +} + +// BuilderImage is the full reference the coordinator pushes to and nodes pull +// from. registryHost is normally ocireg.Host. +func BuilderImage(registryHost string) string { + return fmt.Sprintf("%s/%s:%s", registryHost, BuilderRepository, BuilderTag()) +} + +// IsBuilderImage reports whether a reference names a cluster lbd toolchain tag. +// +// This is an authorization check rather than a convenience. A node asked to +// build runs whatever the image's entrypoint says and then loads the result +// into its own kernel as root. The registry resolves manifests by tag alone, +// so checking the repository without the tag prefix would also accept an app +// image under a forged repository name. +// +// The content hash is deliberately not checked: a newer coordinator may +// publish a different version of the toolchain for this node to build with. +func IsBuilderImage(ref string) bool { + prefix := ocireg.Host + "/" + BuilderRepository + ":" + core_v1alpha.SystemArtifactPrefix + "lbd-builder-" + return strings.HasPrefix(ref, prefix) && len(ref) > len(prefix) +} + +// MaterializeBuilder writes the toolchain definition into dir, which then +// becomes the BuildKit context. It has to reach a real directory: fsutil.NewFS +// only takes a path, and the repo has no in-memory build context. +func MaterializeBuilder(dir string) error { + return materializeEmbed(builderFS, "builder", dir) +} + +// hashFS produces a stable digest over every file under root: each path and +// its bytes, in sorted order, so the result does not depend on walk order. +func hashFS(fsys fs.FS, root string) (string, error) { + var paths []string + err := fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return "", err + } + sort.Strings(paths) + + h := sha256.New() + for _, path := range paths { + data, err := fs.ReadFile(fsys, path) + if err != nil { + return "", err + } + // Length-prefix the name so a path and its content cannot be confused + // with a different split of the same bytes. + fmt.Fprintf(h, "%d:%s\n", len(path), path) + fmt.Fprintf(h, "%d:", len(data)) + h.Write(data) + } + + // Short enough to read in an image tag, long enough not to collide. + return hex.EncodeToString(h.Sum(nil))[:16], nil +} + +// materializeEmbed writes an embedded tree rooted at root into dir, stripping +// the root prefix. Executable bits are not carried by embed.FS, so anything +// that has to run is given one. +func materializeEmbed(fsys fs.FS, root, dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + + return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + target := filepath.Join(dir, rel) + + if d.IsDir() { + return os.MkdirAll(target, 0755) + } + + data, err := fs.ReadFile(fsys, path) + if err != nil { + return fmt.Errorf("reading embedded %s: %w", path, err) + } + + mode := os.FileMode(0644) + if filepath.Ext(rel) == ".sh" { + mode = 0755 + } + if err := os.WriteFile(target, data, mode); err != nil { + return fmt.Errorf("writing %s: %w", target, err) + } + return nil + }) +} diff --git a/pkg/lbdmod/builderimage_test.go b/pkg/lbdmod/builderimage_test.go new file mode 100644 index 000000000..19ba29e89 --- /dev/null +++ b/pkg/lbdmod/builderimage_test.go @@ -0,0 +1,133 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/entity" +) + +func TestBuilderVersionIsStable(t *testing.T) { + // The tag is a content hash, so it must not move between calls -- a tag + // that changed per process would rebuild and re-push on every install. + first := BuilderVersion() + assert.Equal(t, first, BuilderVersion()) + assert.Len(t, first, 16) + assert.NotEmpty(t, first) +} + +func TestBuilderImageReference(t *testing.T) { + ref := BuilderImage("cluster.local:5000") + assert.Equal(t, "cluster.local:5000/"+BuilderRepository+":"+BuilderTag(), ref) + assert.Contains(t, ref, BuilderVersion(), "the tag carries the content hash") +} + +func TestBuilderTagSurvivesArtifactGC(t *testing.T) { + // An artifact's entity name is the tag it was pushed under, and that name + // is all artifact GC has to tell a system image from an orphan. If this + // ever stops holding, the toolchain image is collected within the hour and + // its blobs deleted underneath the nodes still pulling it. + assert.True(t, core_v1alpha.IsSystemArtifact(entity.Id("artifact/"+BuilderTag()))) + assert.False(t, core_v1alpha.IsSystemArtifact(entity.Id("artifact/orphan"))) +} + +func TestHashFSChangesWithContent(t *testing.T) { + base := fstest.MapFS{ + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + "b/build.sh": {Data: []byte("echo hi\n")}, + } + changed := fstest.MapFS{ + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + "b/build.sh": {Data: []byte("echo bye\n")}, + } + + baseSum, err := hashFS(base, "b") + require.NoError(t, err) + changedSum, err := hashFS(changed, "b") + require.NoError(t, err) + + assert.NotEqual(t, baseSum, changedSum, "changing the build script must change the tag") + + // Same content hashes the same, so an unchanged toolchain is never rebuilt. + again, err := hashFS(fstest.MapFS{ + "b/build.sh": {Data: []byte("echo hi\n")}, + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + }, "b") + require.NoError(t, err) + assert.Equal(t, baseSum, again, "the hash must not depend on walk order") +} + +func TestHashFSSeparatesNamesFromContent(t *testing.T) { + // Without length-prefixing, a rename could be cancelled out by a content + // change that shifts the same bytes across the boundary. + a, err := hashFS(fstest.MapFS{"b/ab": {Data: []byte("c")}}, "b") + require.NoError(t, err) + b, err := hashFS(fstest.MapFS{"b/a": {Data: []byte("bc")}}, "b") + require.NoError(t, err) + assert.NotEqual(t, a, b) +} + +func TestMaterializeBuilderWritesABuildContext(t *testing.T) { + dir := filepath.Join(t.TempDir(), "ctx") + require.NoError(t, MaterializeBuilder(dir)) + + // BuildKit's dockerfile frontend takes the filename relative to the + // context root, so the Dockerfile has to sit at the top of it. + dockerfile, err := os.ReadFile(filepath.Join(dir, BuilderDockerfile)) + require.NoError(t, err) + assert.Contains(t, string(dockerfile), "FROM ubuntu:24.04") + assert.Contains(t, string(dockerfile), "build-lbd") + + script, err := os.Stat(filepath.Join(dir, "build.sh")) + require.NoError(t, err) + // embed.FS drops the executable bit, and the image COPYs this in and runs + // it, so materializing has to put it back. + assert.NotZero(t, script.Mode().Perm()&0100, "build.sh must be executable") +} + +func TestMaterializeBuilderIsRepeatable(t *testing.T) { + dir := filepath.Join(t.TempDir(), "ctx") + require.NoError(t, MaterializeBuilder(dir)) + require.NoError(t, os.WriteFile(filepath.Join(dir, BuilderDockerfile), []byte("clobbered"), 0644)) + + require.NoError(t, MaterializeBuilder(dir)) + data, err := os.ReadFile(filepath.Join(dir, BuilderDockerfile)) + require.NoError(t, err) + assert.NotEqual(t, "clobbered", string(data)) +} + +func TestIsBuilderImageRejectsForeignReferences(t *testing.T) { + // This gates what a node will pull, run, and load into its kernel, so it + // has to reject anything outside the cluster's own toolchain repository. + for _, ref := range []string{ + "docker.io/library/ubuntu:24.04", + "evil.example.com/miren-system/lbd-builder:v1", + "cluster.local:5000/someapp:latest", + // A prefix match on the host alone is not enough. + "cluster.local:5000/miren-system/lbd-builder-evil:v1", + // The registry ignores the repository on lookup, so an app tag can + // masquerade behind the toolchain repository name. + ocireg.Host + "/" + BuilderRepository + ":app-version-123", + ocireg.Host + "/" + BuilderRepository + ":" + core_v1alpha.SystemArtifactPrefix + "lbd-builder-", + // No tag at all. + "cluster.local:5000/" + BuilderRepository, + "cluster.local:5000/" + BuilderRepository + ":", + "", + } { + assert.False(t, IsBuilderImage(ref), "should have rejected %q", ref) + } +} + +func TestIsBuilderImageAcceptsOurOwn(t *testing.T) { + assert.True(t, IsBuilderImage(BuilderImage(ocireg.Host))) + + // A coordinator on a newer miren carries a different content hash, and + // asking a node to build with it is legitimate. + assert.True(t, IsBuilderImage(ocireg.Host+"/"+BuilderRepository+":miren-system-lbd-builder-0000000000000000")) +} diff --git a/pkg/lbdmod/ctrbuild/ctrbuild.go b/pkg/lbdmod/ctrbuild/ctrbuild.go new file mode 100644 index 000000000..db3e0cc6b --- /dev/null +++ b/pkg/lbdmod/ctrbuild/ctrbuild.go @@ -0,0 +1,292 @@ +// Package ctrbuild runs the lbd builder image on containerd. +// +// It is separate from pkg/lbdmod so that package can stay free of container +// runtime dependencies: components/diskio and controllers/disk import lbdmod +// only to ask whether lbd is usable, and should not pull containerd in behind +// that question. +package ctrbuild + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync" + "syscall" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/pkg/cio" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "miren.dev/runtime/pkg/lbdmod" +) + +// DefaultNamespace is the containerd namespace miren's own containers live in. +const DefaultNamespace = "miren" + +// cleanupTimeout bounds the teardown of a container after the work is done, +// including the case where the caller's context has already been cancelled. +const cleanupTimeout = 30 * time.Second + +// outputTailLines is how much of the build output a failure quotes back. +const outputTailLines = 40 + +// Builder runs a build container on containerd. +// +// Nothing in components/ does this: every component there is a supervised +// daemon with a restart policy. A build is the opposite -- it runs to +// completion, its exit code is the answer, and it must leave nothing behind. +type Builder struct { + cc *containerd.Client + log *slog.Logger + + // registry, when set, teaches the pull about the cluster-local registry. + // The toolchain image lives there and nowhere public, so a builder without + // it can only run an image already in the node's containerd. + registry *ClusterRegistry +} + +// New returns a Builder that runs containers on cc. +func New(cc *containerd.Client, log *slog.Logger, opts ...Option) *Builder { + b := &Builder{cc: cc, log: log} + for _, o := range opts { + o(b) + } + return b +} + +// Option configures a Builder. +type Option func(*Builder) + +// WithClusterRegistry lets the builder pull from the cluster-local registry, +// which is where the toolchain image is published. +func WithClusterRegistry(r *ClusterRegistry) Option { + return func(b *Builder) { b.registry = r } +} + +// Build pulls the image, runs the container to completion, and tears +// everything down. It returns an error unless the container exited zero. +func (b *Builder) Build(ctx context.Context, spec lbdmod.BuildSpec) error { + ctx = namespaces.WithNamespace(ctx, DefaultNamespace) + + image, err := b.resolveImage(ctx, spec.Image) + if err != nil { + return err + } + + // A previous run that died before its own cleanup leaves the container + // behind and its name taken. + if existing, err := b.cc.LoadContainer(ctx, spec.Name); err == nil { + b.log.Warn("removing a container left by an earlier build", "container", spec.Name) + b.removeContainer(ctx, existing) + } + + // Deliberately not privileged. Compiling C against read-only bind mounts + // needs no extra capabilities -- runc's default masked and read-only + // paths cover /proc and /sys, not /lib/modules or /usr/src -- and this + // container runs on every node that turns accelerator mode on. Verified + // against a real containerd for both the host-headers and header-fetch + // paths; if that ever stops holding, add the one capability that is + // missing rather than all of them. + opts := []oci.SpecOpts{ + oci.WithImageConfig(image), + oci.WithProcessArgs(spec.Args...), + oci.WithEnv(spec.Env), + oci.WithMounts(ociMounts(spec.Mounts)), + } + if spec.HostNetwork { + opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostResolvconf) + } + + container, err := b.cc.NewContainer(ctx, spec.Name, + containerd.WithImage(image), + containerd.WithNewSnapshot(spec.Name+"-snapshot", image), + containerd.WithNewSpec(opts...), + ) + if err != nil { + return fmt.Errorf("creating the build container: %w", err) + } + defer b.removeContainer(context.WithoutCancel(ctx), container) + + output := newTailWriter(b.log, spec.Name, outputTailLines) + task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, output, output))) + if err != nil { + return fmt.Errorf("creating the build task: %w", err) + } + defer func() { + output.flush() + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + if _, err := task.Delete(cleanupCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete the build task", "error", err) + } + }() + + // Establish the exit channel before starting, so the exit event cannot be + // missed by a build that finishes immediately. + exitCh, err := task.Wait(ctx) + if err != nil { + return fmt.Errorf("waiting on the build task: %w", err) + } + + if err := task.Start(ctx); err != nil { + return fmt.Errorf("starting the build: %w", err) + } + + select { + case status := <-exitCh: + output.flush() + if err := status.Error(); err != nil { + return fmt.Errorf("the build did not report a result: %w", err) + } + if code := status.ExitCode(); code != 0 { + return &lbdmod.BuildFailedError{ExitCode: code, Output: output.Tail()} + } + return nil + + case <-ctx.Done(): + // Kill rather than leaving a compile running against a directory we + // are about to delete. + if err := task.Kill(context.WithoutCancel(ctx), syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to kill the build task", "error", err) + } + return ctx.Err() + } +} + +// resolveImage returns the builder image, pulling it only if it is not already +// in the local store. +// +// Preferring the local copy is what lets a node that has already built once +// rebuild without a registry, and what makes a side-loaded image usable at all +// -- `ctr images import` plus --image is the only way to run a builder on a +// node with no reachable registry. It is safe here because the builder +// reference is pinned to a tag that never moves. +func (b *Builder) resolveImage(ctx context.Context, ref string) (containerd.Image, error) { + if img, err := b.cc.GetImage(ctx, ref); err == nil { + b.log.Info("using the lbd builder image already on this node", "image", ref) + return img, nil + } + + b.log.Info("pulling the lbd builder image", "image", ref) + + pullOpts := []containerd.RemoteOpt{containerd.WithPullUnpack} + if b.registry != nil { + pullOpts = append(pullOpts, containerd.WithResolver(b.registry.resolver())) + } + + img, err := b.cc.Pull(ctx, ref, pullOpts...) + if err != nil { + return nil, fmt.Errorf("pulling %s: %w", ref, err) + } + return img, nil +} + +// removeContainer deletes a container and its snapshot, tolerating a container +// that is already gone. +func (b *Builder) removeContainer(ctx context.Context, container containerd.Container) { + ctx, cancel := context.WithTimeout(namespaces.WithNamespace(ctx, DefaultNamespace), cleanupTimeout) + defer cancel() + + if task, err := container.Task(ctx, nil); err == nil { + if _, err := task.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete a leftover build task", "error", err) + } + } + + if err := container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete the build container", "error", err, "container", container.ID()) + } +} + +// ociMounts converts the runtime-agnostic mounts into OCI bind mounts. +func ociMounts(mounts []lbdmod.Mount) []specs.Mount { + out := make([]specs.Mount, 0, len(mounts)) + for _, m := range mounts { + access := "rw" + if m.ReadOnly { + access = "ro" + } + out = append(out, specs.Mount{ + Destination: m.Destination, + Type: "bind", + Source: m.Source, + Options: []string{"rbind", access}, + }) + } + return out +} + +// tailWriter forwards container output to a logger a line at a time and keeps +// the last few lines, so a failure can quote what actually went wrong instead +// of just its exit code. +type tailWriter struct { + log *slog.Logger + name string + limit int + + mu sync.Mutex + partial []byte + lines []string +} + +func newTailWriter(log *slog.Logger, name string, limit int) *tailWriter { + return &tailWriter{log: log, name: name, limit: limit} +} + +func (w *tailWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.partial = append(w.partial, p...) + for { + idx := strings.IndexByte(string(w.partial), '\n') + if idx < 0 { + break + } + line := strings.TrimRight(string(w.partial[:idx]), "\r") + w.partial = w.partial[idx+1:] + w.record(line) + } + return len(p), nil +} + +// flush emits whatever the container left without a trailing newline. +func (w *tailWriter) flush() { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.partial) > 0 { + w.record(strings.TrimRight(string(w.partial), "\r")) + w.partial = nil + } +} + +// record must be called with w.mu held. +// +// Build output goes to Debug, not Info. A compile is thirty-odd lines of make +// output that would be identical the next thousand times it runs, and the +// automatic rebuild after a kernel upgrade emits them into the daemon log with +// nobody watching. The outcome is logged at Info by the installer, and a +// failure carries the tail in BuildFailedError regardless of level -- so the +// case that actually needs this output never depended on it being Info. +// Operators watching a build can see it with -v. +func (w *tailWriter) record(line string) { + if strings.TrimSpace(line) == "" { + return + } + w.log.Debug(line, "source", w.name) + w.lines = append(w.lines, line) + if len(w.lines) > w.limit { + w.lines = w.lines[len(w.lines)-w.limit:] + } +} + +// Tail returns the retained lines as a single block. +func (w *tailWriter) Tail() string { + w.mu.Lock() + defer w.mu.Unlock() + return strings.Join(w.lines, "\n") +} diff --git a/pkg/lbdmod/ctrbuild/ctrbuild_test.go b/pkg/lbdmod/ctrbuild/ctrbuild_test.go new file mode 100644 index 000000000..9d0d440ac --- /dev/null +++ b/pkg/lbdmod/ctrbuild/ctrbuild_test.go @@ -0,0 +1,76 @@ +package ctrbuild + +import ( + "bytes" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/lbdmod" +) + +func TestTailWriterLogsLinesAndKeepsTheTail(t *testing.T) { + var logged bytes.Buffer + log := slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{Level: slog.LevelDebug})) + w := newTailWriter(log, "build", 3) + + // Split across writes, as a pipe delivers it. + _, err := w.Write([]byte("first\nsec")) + require.NoError(t, err) + _, err = w.Write([]byte("ond\nthird\nfourth\n")) + require.NoError(t, err) + + assert.Equal(t, "second\nthird\nfourth", w.Tail(), "only the last few lines are kept") + assert.Contains(t, logged.String(), "first", "every line still reaches the log") + assert.Contains(t, logged.String(), "fourth") +} + +func TestTailWriterKeepsBuildOutputOffInfo(t *testing.T) { + // The automatic rebuild after a kernel upgrade runs unattended, so a whole + // compile at Info would be noise in the daemon log. The tail is kept + // regardless of level, which is what a failure reports. + var atInfo bytes.Buffer + log := slog.New(slog.NewTextHandler(&atInfo, &slog.HandlerOptions{Level: slog.LevelInfo})) + w := newTailWriter(log, "build", 10) + + _, err := w.Write([]byte(" CC [M] /src/lbd_main.o\n")) + require.NoError(t, err) + + assert.Empty(t, atInfo.String(), "compile output must not reach an Info-level log") + assert.Contains(t, w.Tail(), "lbd_main.o", "the tail is still captured for failure reporting") +} + +func TestTailWriterFlushesAnUnterminatedLine(t *testing.T) { + w := newTailWriter(slog.New(slog.DiscardHandler), "build", 10) + _, err := w.Write([]byte("no trailing newline")) + require.NoError(t, err) + + assert.Empty(t, w.Tail(), "an unterminated line is not a line yet") + w.flush() + assert.Equal(t, "no trailing newline", w.Tail()) +} + +func TestTailWriterIgnoresBlankLines(t *testing.T) { + w := newTailWriter(slog.New(slog.DiscardHandler), "build", 10) + _, err := w.Write([]byte("\n \nreal\n\n")) + require.NoError(t, err) + assert.Equal(t, "real", w.Tail()) +} + +func TestOCIMountsCarryReadOnlyThrough(t *testing.T) { + mounts := ociMounts([]lbdmod.Mount{ + {Source: "/build/src", Destination: "/src"}, + {Source: "/lib/modules", Destination: "/lib/modules", ReadOnly: true}, + }) + + require.Len(t, mounts, 2) + + assert.Equal(t, "/src", mounts[0].Destination) + assert.Equal(t, "/build/src", mounts[0].Source) + assert.Equal(t, "bind", mounts[0].Type) + assert.Equal(t, []string{"rbind", "rw"}, mounts[0].Options) + + // The host's module tree must never be writable from the builder. + assert.Equal(t, []string{"rbind", "ro"}, mounts[1].Options) +} diff --git a/pkg/lbdmod/ctrbuild/registry.go b/pkg/lbdmod/ctrbuild/registry.go new file mode 100644 index 000000000..3a48a08fe --- /dev/null +++ b/pkg/lbdmod/ctrbuild/registry.go @@ -0,0 +1,88 @@ +package ctrbuild + +import ( + "fmt" + "net/http" + + "github.com/containerd/containerd/v2/core/remotes" + "github.com/containerd/containerd/v2/core/remotes/docker" + "miren.dev/runtime/components/netresolve" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/containerdx" + "miren.dev/runtime/pkg/workloadidentity" +) + +// ClusterRegistry resolves images from the cluster-local registry, which is +// where the toolchain image lives. Without it a node can only reach public +// registries, and the builder image is deliberately not published to one. +// +// This is the same path app image pulls already take on a runner: the address +// comes from an in-process host map rather than DNS, the hop is plain HTTP on +// the cluster's own network, and the bearer token is minted by the coordinator +// on a distributed runner's behalf. +type ClusterRegistry struct { + // Resolver maps cluster.local to an address. On a runner that is the + // coordinator's IP; on the coordinator it is the local router. + Resolver netresolve.Resolver + + // Issuer mints the registry token. A distributed runner holds a remote + // issuer that proxies to the coordinator, since it has no signing key of + // its own. + Issuer workloadidentity.TokenIssuer +} + +// resolver returns a containerd resolver that knows the cluster registry and +// falls back to the normal public behavior for every other host. +func (c *ClusterRegistry) resolver() remotes.Resolver { + return docker.NewResolver(docker.ResolverOptions{ + Hosts: func(host string) ([]docker.RegistryHost, error) { + switch host { + case "cluster.local", ocireg.Host: + h, err := c.host() + if err != nil { + return nil, err + } + return []docker.RegistryHost{h}, nil + default: + return []docker.RegistryHost{containerdx.DefaultRegistryHost(host)}, nil + } + }, + }) +} + +func (c *ClusterRegistry) host() (docker.RegistryHost, error) { + addr, err := c.Resolver.LookupHost("cluster.local") + if err != nil { + return docker.RegistryHost{}, fmt.Errorf("resolving cluster.local: %w", err) + } + + h := docker.RegistryHost{ + Client: http.DefaultClient, + Host: addr.String() + ":5000", + Scheme: "http", + Path: "/v2", + // Pull only. The registry refuses a push under this identity anyway, + // and nothing here ever needs one. + Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + + if c.Issuer == nil { + return h, nil + } + + // Reuses the sandbox controller's identity rather than minting a new one: + // it is the identity that means "this node pulling an image from the + // cluster registry", it is already the only non-BuildKit workload the + // registry grants GET and HEAD to, and it is already in the set a runner + // is allowed to ask the coordinator for. + token, err := c.Issuer.IssueSystemWorkloadToken( + workloadidentity.SystemWorkloadSandboxController, + workloadidentity.TokenOptions{Audience: []string{ocireg.Audience}}, + ) + if err != nil { + return docker.RegistryHost{}, fmt.Errorf("issuing a registry token: %w", err) + } + h.Header = http.Header{"Authorization": []string{"Bearer " + token}} + + return h, nil +} diff --git a/pkg/lbdmod/ctrbuild/registry_test.go b/pkg/lbdmod/ctrbuild/registry_test.go new file mode 100644 index 000000000..25a0f7440 --- /dev/null +++ b/pkg/lbdmod/ctrbuild/registry_test.go @@ -0,0 +1,89 @@ +package ctrbuild + +import ( + "net/netip" + "testing" + + "github.com/containerd/containerd/v2/core/remotes/docker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/workloadidentity" +) + +type fakeResolver struct { + addr netip.Addr + err error +} + +func (f fakeResolver) LookupHost(string) (netip.Addr, error) { return f.addr, f.err } + +// fakeIssuer records what identity and audience the pull asked for. +type fakeIssuer struct { + workloadidentity.TokenIssuer + + workload workloadidentity.SystemWorkload + audience []string +} + +func (f *fakeIssuer) IssueSystemWorkloadToken(w workloadidentity.SystemWorkload, opts workloadidentity.TokenOptions) (string, error) { + f.workload = w + f.audience = opts.Audience + return "test-token", nil +} + +func TestClusterRegistryHostTargetsTheClusterRegistry(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + + h, err := r.host() + require.NoError(t, err) + + // The registry is reached over the cluster's own network, so plain HTTP on + // the coordinator's address rather than a public registry over TLS. + assert.Equal(t, "10.1.2.3:5000", h.Host) + assert.Equal(t, "http", h.Scheme) + assert.Equal(t, "/v2", h.Path) + // Pull only: the registry refuses a push under this identity anyway. + assert.Zero(t, h.Capabilities&docker.HostCapabilityPush) + assert.NotZero(t, h.Capabilities&docker.HostCapabilityPull) +} + +func TestClusterRegistryWithoutAnIssuerSendsNoToken(t *testing.T) { + // A coordinator with no issuer configured still resolves; the registry + // only enforces when it has an issuer of its own. + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + + h, err := r.host() + require.NoError(t, err) + assert.Empty(t, h.Header.Get("Authorization")) +} + +func TestClusterRegistryReportsAResolveFailure(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{err: assert.AnError}} + + _, err := r.host() + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving cluster.local") +} + +func TestClusterRegistryAsksForAPullIdentity(t *testing.T) { + issuer := &fakeIssuer{} + r := &ClusterRegistry{ + Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}, + Issuer: issuer, + } + + h, err := r.host() + require.NoError(t, err) + + assert.Equal(t, "Bearer test-token", h.Header.Get("Authorization")) + // The registry grants GET and HEAD to exactly this identity; anything else + // is a 403 on every pull. + assert.Equal(t, workloadidentity.SystemWorkloadSandboxController, issuer.workload) + assert.Equal(t, []string{ocireg.Audience}, issuer.audience) +} + +func TestResolverIsBuilt(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + assert.NotNil(t, r.resolver()) +} diff --git a/pkg/lbdmod/kernel.go b/pkg/lbdmod/kernel.go new file mode 100644 index 000000000..6e73581a5 --- /dev/null +++ b/pkg/lbdmod/kernel.go @@ -0,0 +1,245 @@ +package lbdmod + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// Host describes the machine we are about to build a module for. +type Host struct { + // KernelRelease is `uname -r`, e.g. "6.8.0-51-generic". A module built + // here only loads on this exact release. + KernelRelease string + + // HeadersDir is the kernel build tree the module must compile against, + // or empty if the host does not have one installed. + HeadersDir string + + // Compiler is the toolchain the running kernel was built with, as + // reported by /proc/version. + Compiler Compiler + + // DistroID and DistroLike come from /etc/os-release: ID is the specific + // distribution ("ubuntu"), and DistroLike is its family ("debian"), which + // is what decides how to name a header package. Either may be empty. + DistroID string + DistroLike []string +} + +// Compiler identifies what built the running kernel. +type Compiler struct { + // Name is "gcc" or "clang", or empty when /proc/version says something + // we do not recognize. + Name string + + // Major is the compiler's major version, or 0 if it could not be read. + Major int +} + +// String renders the compiler for logs and error messages. +func (c Compiler) String() string { + switch { + case c.Name == "": + return "unknown" + case c.Major == 0: + return c.Name + default: + return fmt.Sprintf("%s-%d", c.Name, c.Major) + } +} + +// HeaderPackage names the distro package that provides the build tree for this +// kernel, so an error message can tell the operator exactly what to install. +// Returns an empty string when the distribution is not one we recognize. +func (h Host) HeaderPackage() string { + for _, id := range append([]string{h.DistroID}, h.DistroLike...) { + switch id { + case "debian", "ubuntu": + return "linux-headers-" + h.KernelRelease + case "fedora", "rhel", "centos": + return "kernel-devel-" + h.KernelRelease + case "arch": + return "linux-headers" + case "alpine": + return "linux-headers" + case "suse", "opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles": + return "kernel-devel" + } + } + return "" +} + +// CanFetchHeaders reports whether the builder can install kernel headers for +// itself rather than borrowing the host's. +// +// The builder image is Debian-based, so it can only reach a Debian-family +// archive. On any other distribution the operator has to install the headers, +// which is what InstallHint tells them to do. Even on a Debian-family host the +// fetch can still come up empty -- a kernel that has aged out of the archive, +// or a Debian host whose package is not in the builder's Ubuntu sources -- and +// the builder reports that itself, naming the package it could not find. +func (h Host) CanFetchHeaders() bool { + return h.hasFamily("debian") || h.hasFamily("ubuntu") +} + +// InstallHint is the sentence to show an operator whose host has no kernel +// build tree. +func (h Host) InstallHint() string { + pkg := h.HeaderPackage() + if pkg == "" { + return fmt.Sprintf("install the kernel headers for %s and try again", h.KernelRelease) + } + switch { + case h.hasFamily("debian"): + return "run: apt-get install " + pkg + case h.hasFamily("fedora"), h.hasFamily("rhel"), h.hasFamily("centos"): + return "run: dnf install " + pkg + case h.hasFamily("arch"): + return "run: pacman -S " + pkg + case h.hasFamily("alpine"): + return "run: apk add " + pkg + case h.hasFamily("suse"), h.hasFamily("sles"): + return "run: zypper install " + pkg + } + return "install " + pkg + " and try again" +} + +func (h Host) hasFamily(id string) bool { + if h.DistroID == id { + return true + } + for _, like := range h.DistroLike { + if like == id { + return true + } + } + return false +} + +// headerCandidates lists where a kernel build tree may live, in the order to +// try. Debian and Ubuntu populate /lib/modules//build; Fedora ships the +// tree under /usr/src/kernels and does not always leave that symlink behind. +func headerCandidates(release string) []string { + return []string{ + filepath.Join("/lib/modules", release, "build"), + filepath.Join("/usr/src/kernels", release), + filepath.Join("/usr/src", "linux-headers-"+release), + } +} + +// findHeaders returns the first candidate that looks like a usable kernel build +// tree, or an empty string. Makefile is the file the module's own build invokes, +// so its absence means the tree is unusable however complete it otherwise looks. +func findHeaders(root, release string) string { + for _, dir := range headerCandidates(release) { + if _, err := os.Stat(filepath.Join(root, dir, "Makefile")); err == nil { + return dir + } + } + return "" +} + +// compilerPattern matches the toolchain stanza /proc/version carries, e.g. +// "(gcc-13 (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0, ...)" or "(clang version 18.1.3". +var compilerPattern = regexp.MustCompile(`\b(gcc|clang)\b[^0-9]*([0-9]+)`) + +// parseCompiler pulls the building toolchain out of a /proc/version line. +func parseCompiler(procVersion string) Compiler { + m := compilerPattern.FindStringSubmatch(procVersion) + if m == nil { + return Compiler{} + } + major, err := strconv.Atoi(m[2]) + if err != nil { + return Compiler{Name: m[1]} + } + return Compiler{Name: m[1], Major: major} +} + +// parseOSRelease reads the ID and ID_LIKE fields of an os-release file. Values +// may be quoted, and ID_LIKE is a space-separated list ordered most-specific +// first. +func parseOSRelease(content string) (id string, like []string) { + for _, line := range strings.Split(content, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + value = strings.Trim(value, `"'`) + switch key { + case "ID": + id = value + case "ID_LIKE": + like = strings.Fields(value) + } + } + return id, like +} + +// kernelRelease reports `uname -r`. It reads the procfs file rather than +// shelling out, and falls back to uname(1) only for a real host, since a +// fixture root has no process to ask. +func kernelRelease(root string) (string, error) { + path := filepath.Join(root, "proc/sys/kernel/osrelease") + data, readErr := os.ReadFile(path) + if readErr == nil { + if release := strings.TrimSpace(string(data)); release != "" { + return release, nil + } + // The file is there but says nothing. Name that, so the error below + // never wraps a nil. + readErr = fmt.Errorf("%s is empty", path) + } + + if root != "/" { + return "", fmt.Errorf("no kernel release under %s: %w", root, readErr) + } + + out, unameErr := exec.Command("uname", "-r").Output() + if unameErr != nil { + return "", fmt.Errorf("could not determine the kernel release: %w", unameErr) + } + release := strings.TrimSpace(string(out)) + if release == "" { + return "", errors.New("could not determine the kernel release: uname -r said nothing") + } + return release, nil +} + +// DetectHost inspects the machine miren is running on. root is the filesystem +// to read from, normally "/"; tests pass a fixture directory. +// +// Missing kernel headers are not an error here: whether they can be fetched +// instead is the caller's decision. +func DetectHost(root string) (Host, error) { + release, err := kernelRelease(root) + if err != nil { + return Host{}, err + } + + h := Host{ + KernelRelease: release, + HeadersDir: findHeaders(root, release), + } + + if data, err := os.ReadFile(filepath.Join(root, "proc/version")); err == nil { + h.Compiler = parseCompiler(string(data)) + } + + for _, path := range []string{"etc/os-release", "usr/lib/os-release"} { + data, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + continue + } + h.DistroID, h.DistroLike = parseOSRelease(string(data)) + break + } + + return h, nil +} diff --git a/pkg/lbdmod/kernel_test.go b/pkg/lbdmod/kernel_test.go new file mode 100644 index 000000000..d2fc91467 --- /dev/null +++ b/pkg/lbdmod/kernel_test.go @@ -0,0 +1,177 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeFile creates path under root, making its parents. +func writeFile(t *testing.T, root, path, content string) { + t.Helper() + full := filepath.Join(root, path) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0755)) + require.NoError(t, os.WriteFile(full, []byte(content), 0644)) +} + +func TestParseCompiler(t *testing.T) { + tests := []struct { + name string + input string + want Compiler + }{ + { + name: "ubuntu gcc", + input: "Linux version 6.8.0-51-generic (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0, GNU ld (GNU Binutils) 2.42) #52-Ubuntu SMP\n", + want: Compiler{Name: "gcc", Major: 13}, + }, + { + name: "fedora gcc", + input: "Linux version 6.11.4-301.fc41.x86_64 (mockbuild@) (gcc (GCC) 14.2.1 20240912, GNU ld version 2.43.1) #1 SMP\n", + want: Compiler{Name: "gcc", Major: 14}, + }, + { + name: "clang built", + input: "Linux version 6.6.30-android14 (build@) (Android (11368139) clang version 17.0.4, LLD 17.0.4) #1 SMP\n", + want: Compiler{Name: "clang", Major: 17}, + }, + { + name: "unrecognized", + input: "Linux version 5.10.0 (someone@somewhere) #1 SMP\n", + want: Compiler{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, parseCompiler(tt.input)) + }) + } +} + +func TestCompilerString(t *testing.T) { + assert.Equal(t, "gcc-13", Compiler{Name: "gcc", Major: 13}.String()) + assert.Equal(t, "clang", Compiler{Name: "clang"}.String()) + assert.Equal(t, "unknown", Compiler{}.String()) +} + +func TestParseOSRelease(t *testing.T) { + id, like := parseOSRelease(`NAME="Ubuntu" +ID=ubuntu +ID_LIKE=debian +VERSION_ID="24.04" +`) + assert.Equal(t, "ubuntu", id) + assert.Equal(t, []string{"debian"}, like) + + id, like = parseOSRelease("ID=fedora\n") + assert.Equal(t, "fedora", id) + assert.Empty(t, like) + + id, like = parseOSRelease(`ID="rocky" +ID_LIKE="rhel centos fedora" +`) + assert.Equal(t, "rocky", id) + assert.Equal(t, []string{"rhel", "centos", "fedora"}, like) +} + +func TestDetectHostReadsFixtureRoot(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.8.0-51-generic\n") + writeFile(t, root, "proc/version", + "Linux version 6.8.0-51-generic (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0) 13.3.0) #52-Ubuntu SMP\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + writeFile(t, root, "lib/modules/6.8.0-51-generic/build/Makefile", "# kernel build tree\n") + + host, err := DetectHost(root) + require.NoError(t, err) + + assert.Equal(t, "6.8.0-51-generic", host.KernelRelease) + assert.Equal(t, "/lib/modules/6.8.0-51-generic/build", host.HeadersDir) + assert.Equal(t, Compiler{Name: "gcc", Major: 13}, host.Compiler) + assert.Equal(t, "ubuntu", host.DistroID) + assert.Equal(t, []string{"debian"}, host.DistroLike) +} + +func TestDetectHostWithoutHeaders(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.11.4-301.fc41.x86_64\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + + host, err := DetectHost(root) + require.NoError(t, err) + + // Missing headers are reported, not an error: the builder may be able to + // fetch them instead. + assert.Empty(t, host.HeadersDir) + assert.Equal(t, "kernel-devel-6.11.4-301.fc41.x86_64", host.HeaderPackage()) + assert.Equal(t, "run: dnf install kernel-devel-6.11.4-301.fc41.x86_64", host.InstallHint()) +} + +func TestDetectHostFindsFedoraStyleHeaders(t *testing.T) { + root := t.TempDir() + release := "6.11.4-301.fc41.x86_64" + writeFile(t, root, "proc/sys/kernel/osrelease", release+"\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + // Fedora ships the build tree here and does not always leave the + // /lib/modules//build symlink behind. + writeFile(t, root, "usr/src/kernels/"+release+"/Makefile", "# kernel build tree\n") + + host, err := DetectHost(root) + require.NoError(t, err) + assert.Equal(t, "/usr/src/kernels/"+release, host.HeadersDir) +} + +func TestDetectHostFallsBackToUsrLibOSRelease(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.8.0-51-generic\n") + writeFile(t, root, "usr/lib/os-release", "ID=debian\n") + + host, err := DetectHost(root) + require.NoError(t, err) + assert.Equal(t, "debian", host.DistroID) +} + +func TestDetectHostNeedsAKernelRelease(t *testing.T) { + _, err := DetectHost(t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel release") +} + +func TestHeaderPackageUsesTheDistroFamily(t *testing.T) { + // A derivative distro we do not name explicitly still gets the right + // package via ID_LIKE. + h := Host{KernelRelease: "6.8.0-51", DistroID: "pop", DistroLike: []string{"ubuntu", "debian"}} + assert.Equal(t, "linux-headers-6.8.0-51", h.HeaderPackage()) + assert.Equal(t, "run: apt-get install linux-headers-6.8.0-51", h.InstallHint()) +} + +func TestHeaderPackageUnknownDistro(t *testing.T) { + h := Host{KernelRelease: "6.8.0-51", DistroID: "somethingelse"} + assert.Empty(t, h.HeaderPackage()) + assert.Equal(t, "install the kernel headers for 6.8.0-51 and try again", h.InstallHint()) +} + +func TestKernelReleaseErrorNamesTheRealCause(t *testing.T) { + // A fixture root with no procfs file at all: the error has to carry the + // read failure, not a nil wrapped by %w. + _, err := kernelRelease(t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel release") + assert.NotContains(t, err.Error(), "%!w", "an error was wrapped that was nil") +} + +func TestKernelReleaseRejectsAnEmptyOsrelease(t *testing.T) { + // The file exists but says nothing. Before, err was nil here and %w + // rendered as %!w(), telling the operator nothing. + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "\n") + + _, err := kernelRelease(root) + require.Error(t, err) + assert.Contains(t, err.Error(), "is empty") + assert.NotContains(t, err.Error(), "%!w") +} diff --git a/pkg/lbdmod/lock.go b/pkg/lbdmod/lock.go new file mode 100644 index 000000000..8520fa11f --- /dev/null +++ b/pkg/lbdmod/lock.go @@ -0,0 +1,67 @@ +package lbdmod + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// ErrBuildInProgress is returned when another process is already building the +// module on this host. +var ErrBuildInProgress = errors.New("an lbd build is already running on this host") + +// buildLock serializes installs across processes. +// +// Two can genuinely overlap now that the server rebuilds unattended: an +// operator running `miren disk accelerator install` while a kernel upgrade has +// the server rebuilding in the background. Both would use the same build +// directory, which build() clears with RemoveAll, and the same fixed container +// name, which the builder deletes before creating its own -- so each would +// destroy the other's work and report a baffling failure. +// +// The lock is an flock rather than a lockfile whose existence is the signal, +// so a process killed mid-build releases it instead of wedging the host until +// someone deletes a stale file. +type buildLock struct { + f *os.File +} + +// acquireBuildLock takes the host-wide build lock without waiting. It returns +// ErrBuildInProgress if another process holds it. +func acquireBuildLock(dataPath string) (*buildLock, error) { + dir := filepath.Join(dataPath, "lbd") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("creating %s: %w", dir, err) + } + + path := filepath.Join(dir, "build.lock") + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", path, err) + } + + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + f.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, ErrBuildInProgress + } + return nil, fmt.Errorf("locking %s: %w", path, err) + } + + return &buildLock{f: f}, nil +} + +// release drops the lock. The file is left behind on purpose: removing it +// would let a second process create and lock a new file at the same path while +// a third still holds the old one. +func (l *buildLock) release() { + if l == nil || l.f == nil { + return + } + unix.Flock(int(l.f.Fd()), unix.LOCK_UN) + l.f.Close() + l.f = nil +} diff --git a/pkg/lbdmod/lock_test.go b/pkg/lbdmod/lock_test.go new file mode 100644 index 000000000..3d0ed956a --- /dev/null +++ b/pkg/lbdmod/lock_test.go @@ -0,0 +1,75 @@ +package lbdmod + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildLockExcludesASecondHolder(t *testing.T) { + dataPath := t.TempDir() + + first, err := acquireBuildLock(dataPath) + require.NoError(t, err) + require.NotNil(t, first) + + // The case this guards: an operator running `install` while the server is + // already rebuilding after a kernel upgrade. Without the lock both would + // clear the same build directory and delete each other's container. + second, err := acquireBuildLock(dataPath) + require.ErrorIs(t, err, ErrBuildInProgress) + assert.Nil(t, second) + + first.release() + + // Once the first is done the lock is available again. + third, err := acquireBuildLock(dataPath) + require.NoError(t, err) + third.release() +} + +func TestBuildLockCreatesItsDirectory(t *testing.T) { + // A host installing for the first time has no /var/lib/miren/lbd yet. + dataPath := filepath.Join(t.TempDir(), "fresh") + + lock, err := acquireBuildLock(dataPath) + require.NoError(t, err) + defer lock.release() + + _, err = os.Stat(filepath.Join(dataPath, "lbd", "build.lock")) + require.NoError(t, err) +} + +func TestReleasingTwiceIsSafe(t *testing.T) { + lock, err := acquireBuildLock(t.TempDir()) + require.NoError(t, err) + + lock.release() + lock.release() + + var nilLock *buildLock + nilLock.release() +} + +func TestBuildLockIsNotHeldByAStaleFile(t *testing.T) { + // An flock dies with the process that held it, so a build killed partway + // through must not wedge the host behind a leftover file. + dataPath := t.TempDir() + + lock, err := acquireBuildLock(dataPath) + require.NoError(t, err) + lock.release() + + // The file survives on purpose; only the lock is gone. + _, err = os.Stat(filepath.Join(dataPath, "lbd", "build.lock")) + require.NoError(t, err) + + again, err := acquireBuildLock(dataPath) + require.NoError(t, err, "a leftover lock file must not block a later build") + assert.False(t, errors.Is(err, ErrBuildInProgress)) + again.release() +} diff --git a/pkg/lbdmod/probe.go b/pkg/lbdmod/probe.go new file mode 100644 index 000000000..29709a096 --- /dev/null +++ b/pkg/lbdmod/probe.go @@ -0,0 +1,208 @@ +package lbdmod + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" + + lbdsrc "miren.dev/runtime/third_party/lbd" +) + +// DefaultDataPath is where miren keeps the lbd install record and build +// scratch space. +const DefaultDataPath = "/var/lib/miren" + +// Options say where to look. The zero value probes the real host with miren's +// default data directory. +type Options struct { + // Root is the filesystem to read, normally "" for "/". Tests point this + // at a fixture directory. + Root string + + // DataPath is miren's data directory, holding the install record. + // Defaults to DefaultDataPath. + DataPath string + + // SearchPath holds extra directories to look for lbdctl in, ahead of + // PATH. The server's release directory belongs here: it is prepended to + // containerd's PATH but not to miren's own. + SearchPath []string +} + +// systemReleasePath is searched for lbdctl by default. It is prepended to +// containerd's PATH but not to miren's own, so without this every caller would +// have to remember to add it. +const systemReleasePath = "/var/lib/miren/release" + +// EnsureControlDevice creates the misc device node when /dev is a private +// tmpfs without udev. The module must already be loaded so sysfs can supply +// its assigned device number. +func EnsureControlDevice() error { + return ensureControlDevice("/sys/class/misc/lbd-control/dev", ControlDevice) +} + +func ensureControlDevice(sysDev, devicePath string) error { + data, err := os.ReadFile(sysDev) + if err != nil { + return fmt.Errorf("read %s: %w", sysDev, err) + } + majorStr, minorStr, ok := strings.Cut(strings.TrimSpace(string(data)), ":") + if !ok { + return fmt.Errorf("invalid device number in %s: %q", sysDev, strings.TrimSpace(string(data))) + } + major, err := strconv.ParseUint(majorStr, 10, 32) + if err != nil { + return fmt.Errorf("invalid major number in %s: %w", sysDev, err) + } + minor, err := strconv.ParseUint(minorStr, 10, 32) + if err != nil { + return fmt.Errorf("invalid minor number in %s: %w", sysDev, err) + } + want := unix.Mkdev(uint32(major), uint32(minor)) + verify := func() error { + fi, err := os.Stat(devicePath) + if err != nil { + return err + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok || fi.Mode()&os.ModeCharDevice == 0 || uint64(st.Rdev) != want { + return fmt.Errorf("%s is not the expected character device %d:%d", devicePath, major, minor) + } + return nil + } + if err := verify(); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := unix.Mknod(devicePath, unix.S_IFCHR|0660, int(want)); err != nil && !errors.Is(err, unix.EEXIST) { + return fmt.Errorf("mknod %s (%d:%d): %w", devicePath, major, minor, err) + } + return verify() +} + +// HostOptions builds the options for inspecting this host. dataPath is where +// miren keeps its data; empty means DefaultDataPath. +// +// Everything that decides whether a disk gets accelerator mode goes through +// here, because they all have to reach the same answer or a node picks a mode +// it cannot serve. They disagreed before: the CLI searched the release +// directory it resolved through $HOME while the disk controller searched only +// the system one, so a host with lbdctl under ~/.miren/release would have the +// CLI choose accelerator and the controller choose universal. +// +// The rule is now the system release directory and PATH, for every caller. +// Nothing resolves a per-user location, since the CLI and the server run as +// different users and would resolve it differently. +func HostOptions(dataPath string) Options { + return Options{DataPath: dataPath} +} + +func (o Options) root() string { + if o.Root == "" { + return "/" + } + return o.Root +} + +// searchPath returns the directories to look for lbdctl in. Callers that name +// none still get the release directory, so every caller agrees on whether +// lbdctl is present. +// +// This builds a new slice rather than appending to o.SearchPath, which would +// write into the caller's backing array whenever it has spare capacity. +func (o Options) searchPath() []string { + return slices.Concat(o.SearchPath, []string{systemReleasePath}) +} + +func (o Options) dataPath() string { + if o.DataPath == "" { + return DefaultDataPath + } + return o.DataPath +} + +// findLbdctl locates the lbdctl binary, checking the caller's directories +// before falling back to PATH. +func findLbdctl(root string, searchPath []string) string { + for _, dir := range searchPath { + if dir == "" { + continue + } + path := filepath.Join(dir, "lbdctl") + if info, err := os.Stat(filepath.Join(root, path)); err == nil && !info.IsDir() { + return path + } + } + + // PATH only makes sense against the real filesystem. + if root != "/" { + return "" + } + if path, err := exec.LookPath("lbdctl"); err == nil { + return path + } + return "" +} + +// Probe reports what miren knows about lbd on this host. It never modifies +// anything, so it is safe to call without root. +func Probe(opts Options) (Status, error) { + root := opts.root() + + host, err := DetectHost(root) + if err != nil { + return Status{}, err + } + + marker, err := readMarker(opts.dataPath()) + if err != nil { + return Status{}, err + } + + installed := false + if _, err := os.Stat(filepath.Join(root, modulePath(host.KernelRelease))); err == nil { + installed = true + } + + _, ctlErr := os.Stat(filepath.Join(root, ControlDevice)) + + return Status{ + Host: host, + Loaded: isModuleLoaded(root, ModuleName), + ControlDevicePresent: ctlErr == nil, + ModuleInstalled: installed, + LbdctlPath: findLbdctl(root, opts.searchPath()), + Marker: marker, + EmbeddedVersion: lbdsrc.Version(), + }, nil +} + +// Available reports whether accelerator mode can run right now: the module is +// loaded, its control device exists, and lbdctl is there to drive it. +// +// This is the check that decides a disk's mode, so it deliberately does not +// read the install record and cannot fail -- unlike Probe, which is for +// explaining the situation to a person. lbdctl being on PATH is not enough on +// its own: miren installs lbdctl alongside the module, so a host that has the +// binary but no loaded module would otherwise be sent down the accelerator +// path and fail at attach time. +func Available(opts Options) bool { + root := opts.root() + + if !isModuleLoaded(root, ModuleName) { + return false + } + if _, err := os.Stat(filepath.Join(root, ControlDevice)); err != nil { + return false + } + return findLbdctl(root, opts.searchPath()) != "" +} diff --git a/pkg/lbdmod/source.go b/pkg/lbdmod/source.go new file mode 100644 index 000000000..22ffd702e --- /dev/null +++ b/pkg/lbdmod/source.go @@ -0,0 +1,18 @@ +package lbdmod + +import ( + lbdsrc "miren.dev/runtime/third_party/lbd" +) + +// SourceVersion reports the miren.dev/lbd version this binary carries. +func SourceVersion() string { + return lbdsrc.Version() +} + +// materializeSource writes the embedded module source into dir, which the +// builder container then mounts. The kernel build system writes its object +// files next to the source, so this has to be a real writable directory rather +// than a read-only mount of something we already have. +func materializeSource(dir string) error { + return materializeEmbed(lbdsrc.FS, "src", dir) +} diff --git a/pkg/lbdmod/source_test.go b/pkg/lbdmod/source_test.go new file mode 100644 index 000000000..d002ee771 --- /dev/null +++ b/pkg/lbdmod/source_test.go @@ -0,0 +1,64 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSourceVersionMatchesTheCheckedInTree(t *testing.T) { + // hack/sync-lbd-src.sh writes VERSION beside the source it copied, and CI + // checks the tree against go.mod. If this is empty the embed is broken. + version := SourceVersion() + require.NotEmpty(t, version) + assert.NotEqual(t, "unknown", version) +} + +func TestMaterializeSourceWritesABuildableTree(t *testing.T) { + dir := filepath.Join(t.TempDir(), "src") + require.NoError(t, materializeSource(dir)) + + // Everything the kernel build and lbdctl need, including the vendored LZ4 + // in its subdirectory. + for _, name := range []string{ + "Makefile", + "dkms.conf", + "lbd_main.c", + "lbd_qcow2.c", + "lbdctl.c", + "lbd.h", + "lz4_kcompat.h", + "lz4/lz4.c", + "lz4/lz4.h", + } { + info, err := os.Stat(filepath.Join(dir, name)) + require.NoError(t, err, "missing %s", name) + assert.Positive(t, info.Size(), "%s is empty", name) + } + + // The build system writes object files next to the source, so the tree has + // to be writable -- it cannot be a read-only mount of the embed. + require.NoError(t, os.WriteFile(filepath.Join(dir, "lbd.o"), []byte("x"), 0644)) + + // The Makefile must still carry the kernel-version probe that lets the + // module build against recent kernels. + makefile, err := os.ReadFile(filepath.Join(dir, "Makefile")) + require.NoError(t, err) + assert.Contains(t, string(makefile), "LBD_RENAME_PARENT") + assert.Contains(t, string(makefile), "obj-m := lbd.o") +} + +func TestMaterializeSourceIsRepeatable(t *testing.T) { + dir := filepath.Join(t.TempDir(), "src") + require.NoError(t, materializeSource(dir)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "lbd_main.c"), []byte("clobbered"), 0644)) + + // A retry after a failed build has to restore the source it overwrote. + require.NoError(t, materializeSource(dir)) + data, err := os.ReadFile(filepath.Join(dir, "lbd_main.c")) + require.NoError(t, err) + assert.NotEqual(t, "clobbered", string(data)) +} diff --git a/pkg/lbdmod/status.go b/pkg/lbdmod/status.go new file mode 100644 index 000000000..8b60ab58a --- /dev/null +++ b/pkg/lbdmod/status.go @@ -0,0 +1,230 @@ +package lbdmod + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + // ModuleName is what the module is called once loaded, and the name + // modprobe takes. + ModuleName = "lbd" + + // ControlDevice is the misc device the module registers. Its presence is + // the only trustworthy proof that the module is loaded and working -- + // lbdctl being on PATH proves nothing. + ControlDevice = "/dev/lbd-control" + + // markerName records what was built and for which kernel, so a later boot + // can tell a stale module from a missing one. + markerName = "installed.json" +) + +// Marker is the record left behind by a successful install. Its presence means +// this host has opted into accelerator mode, which is what licenses miren to +// rebuild the module unattended after a kernel upgrade. +type Marker struct { + // LbdVersion is the miren.dev/lbd version the source came from. + LbdVersion string `json:"lbd_version"` + + // KernelRelease is the kernel the module was built for. A module built + // for one release will not load on another. + KernelRelease string `json:"kernel_release"` + + // BuilderImage is the cluster image used for this install. Retaining the + // reference lets startup rebuild without asking the coordinator to publish + // the current binary's potentially newer toolchain tag first. + BuilderImage string `json:"builder_image,omitempty"` + + // ModulePath is where the built module was installed. + ModulePath string `json:"module_path"` + + // LbdctlPath is where the built lbdctl was installed. + LbdctlPath string `json:"lbdctl_path"` + + // BuiltAt is when the build finished. + BuiltAt time.Time `json:"built_at"` +} + +// Status is what miren knows about lbd on this host. +type Status struct { + // Host is the machine as detected. + Host Host + + // Loaded is true when the module is in /proc/modules. + Loaded bool + + // ControlDevicePresent is true when /dev/lbd-control exists. Together + // with Loaded this is the real availability test. + ControlDevicePresent bool + + // ModuleInstalled is true when a built module exists for the running + // kernel, whether or not it is currently loaded. + ModuleInstalled bool + + // LbdctlPath is where lbdctl was found, or empty. + LbdctlPath string + + // Marker is the record of the last successful install, or nil if this + // host has never installed the module. + Marker *Marker + + // EmbeddedVersion is the miren.dev/lbd version this binary carries. + EmbeddedVersion string +} + +// Available reports whether accelerator mode can actually run right now: the +// module is loaded, its control device is there, and lbdctl exists to drive it. +func (s Status) Available() bool { + return s.Loaded && s.ControlDevicePresent && s.LbdctlPath != "" +} + +// Stale reports whether this host installed the module before but what is on +// disk no longer fits -- almost always because the kernel was upgraded, but +// also when miren itself now carries a newer lbd. A usable old module is +// reported as stale but is not swapped out unattended. +func (s Status) Stale() bool { + return s.staleReason() != "" +} + +// staleReason names what stopped fitting, phrased to follow "lbd is loaded +// but ...". It is empty when nothing is stale. +func (s Status) staleReason() string { + if s.Marker == nil { + return "" + } + switch { + case s.Marker.KernelRelease != s.Host.KernelRelease: + return fmt.Sprintf("it was built for kernel %s, not the %s this host is running", + s.Marker.KernelRelease, s.Host.KernelRelease) + case s.EmbeddedVersion != "" && s.Marker.LbdVersion != s.EmbeddedVersion: + return fmt.Sprintf("miren now bundles lbd %s and the installed module is %s", + s.EmbeddedVersion, s.Marker.LbdVersion) + case !s.ModuleInstalled: + return fmt.Sprintf("its module file %s is gone", s.Marker.ModulePath) + } + return "" +} + +// Explain renders the status as a sentence for logs and CLI output. +func (s Status) Explain() string { + // Concrete faults come first. A module can be loaded without miren having + // installed it -- by hand, or by a distro package -- and in that case the + // specific problem is more useful than "not installed". + stale := s.staleReason() + + switch { + // A loaded module can still be the wrong one -- most often after miren + // was upgraded to a build carrying a newer lbd. Saying only that it is + // loaded would hide that an operator should arrange an upgrade. + case s.Available() && stale != "": + return "lbd is loaded but " + stale + case s.Available(): + return fmt.Sprintf("lbd %s is loaded for kernel %s", s.markerVersion(), s.Host.KernelRelease) + case s.Loaded && !s.ControlDevicePresent: + return fmt.Sprintf("lbd is loaded but %s is missing", ControlDevice) + case s.Loaded && s.LbdctlPath == "": + return "lbd is loaded but lbdctl is missing" + case s.Marker == nil: + return "lbd is not installed" + case stale != "": + return "lbd is not usable: " + stale + case !s.Loaded: + return "lbd is installed but not loaded" + default: + return "lbd is installed but lbdctl is missing" + } +} + +func (s Status) markerVersion() string { + if s.Marker != nil && s.Marker.LbdVersion != "" { + return s.Marker.LbdVersion + } + return s.EmbeddedVersion +} + +// isModuleLoaded reports whether the named module appears in /proc/modules. +func isModuleLoaded(root, name string) bool { + content, err := os.ReadFile(filepath.Join(root, "proc/modules")) + if err != nil { + return false + } + for line := range strings.SplitSeq(string(content), "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && fields[0] == name { + return true + } + } + return false +} + +// modulePath is where a built module for the given kernel lives. "extra" is the +// conventional home for out-of-tree modules and is on depmod's search path. +func modulePath(release string) string { + return filepath.Join("/lib/modules", release, "extra", ModuleName+".ko") +} + +// markerPath is where the install record lives, under miren's data directory. +func markerPath(dataPath string) string { + return filepath.Join(dataPath, "lbd", markerName) +} + +// readMarker loads the install record, returning nil when the host has never +// installed the module. A corrupt marker is reported as an error rather than +// silently treated as absent, since discarding it would strand a module that is +// actually installed. +func readMarker(dataPath string) (*Marker, error) { + data, err := os.ReadFile(markerPath(dataPath)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading the lbd install record: %w", err) + } + + var m Marker + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("the lbd install record at %s is corrupt: %w", markerPath(dataPath), err) + } + return &m, nil +} + +// writeMarker records a successful install. +func writeMarker(dataPath string, m Marker) error { + path := markerPath(dataPath) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("encoding the lbd install record: %w", err) + } + + // Written through a rename so a crash mid-write cannot leave a truncated + // record. readMarker reports a corrupt one as an error rather than + // treating it as absent, which would otherwise wedge every later probe. + tmp := path + ".tmp" + if err := os.WriteFile(tmp, append(data, '\n'), 0644); err != nil { + return fmt.Errorf("writing %s: %w", tmp, err) + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return fmt.Errorf("installing %s: %w", path, err) + } + return nil +} + +// removeMarker forgets that lbd was ever installed, so later boots stop +// rebuilding it. +func removeMarker(dataPath string) error { + err := os.Remove(markerPath(dataPath)) + if os.IsNotExist(err) { + return nil + } + return err +} diff --git a/pkg/lbdmod/status_test.go b/pkg/lbdmod/status_test.go new file mode 100644 index 000000000..2f8453a45 --- /dev/null +++ b/pkg/lbdmod/status_test.go @@ -0,0 +1,319 @@ +package lbdmod + +import ( + "errors" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +const testRelease = "6.8.0-51-generic" + +// ubuntuRoot builds a fixture filesystem for a plausible Ubuntu host with +// kernel headers installed and no lbd anywhere. +func ubuntuRoot(t *testing.T) string { + t.Helper() + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", testRelease+"\n") + writeFile(t, root, "proc/version", + "Linux version "+testRelease+" (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0) 13.3.0) #52-Ubuntu SMP\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + writeFile(t, root, "lib/modules/"+testRelease+"/build/Makefile", "# kernel build tree\n") + writeFile(t, root, "proc/modules", "loop 69632 0 - Live 0x0000000000000000\n") + return root +} + +func TestIsModuleLoaded(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/modules", `nf_tables 356352 0 - Live 0x0000000000000000 +lbd 65536 1 - Live 0x0000000000000000 +loop 69632 0 - Live 0x0000000000000000 +`) + assert.True(t, isModuleLoaded(root, "lbd")) + assert.True(t, isModuleLoaded(root, "loop")) + assert.False(t, isModuleLoaded(root, "lbdctl")) + assert.False(t, isModuleLoaded(t.TempDir(), "lbd")) +} + +func TestProbeOnAHostWithoutLbd(t *testing.T) { + root := ubuntuRoot(t) + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + assert.False(t, status.Available()) + assert.False(t, status.Loaded) + assert.False(t, status.ModuleInstalled) + assert.Nil(t, status.Marker) + assert.False(t, status.Stale(), "a host that never installed lbd has nothing to rebuild") + assert.Equal(t, "lbd is not installed", status.Explain()) + assert.NotEmpty(t, status.EmbeddedVersion, "the binary should know which lbd it carries") +} + +func TestProbeOnAHealthyHost(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{ + Root: root, + DataPath: dataPath, + SearchPath: []string{"/usr/local/bin"}, + }) + require.NoError(t, err) + + assert.True(t, status.Available()) + assert.False(t, status.Stale()) + assert.Equal(t, "/usr/local/bin/lbdctl", status.LbdctlPath) + assert.Contains(t, status.Explain(), "is loaded for kernel "+testRelease) +} + +func TestLbdctlPresenceAloneIsNotAvailability(t *testing.T) { + // The bug this guards: shipping lbdctl in the release bundle used to be + // enough to select accelerator mode, even with no module loaded. + root := ubuntuRoot(t) + writeFile(t, root, "usr/local/bin/lbdctl", "") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir(), SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + assert.NotEmpty(t, status.LbdctlPath) + assert.False(t, status.Available()) +} + +func TestLoadedWithoutControlDeviceIsNotAvailable(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, "usr/local/bin/lbdctl", "") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir(), SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + assert.True(t, status.Loaded) + assert.False(t, status.ControlDevicePresent) + assert.False(t, status.Available()) + assert.Contains(t, status.Explain(), ControlDevice+" is missing") +} + +func TestEnsureControlDevice(t *testing.T) { + dir := t.TempDir() + sysDev := filepath.Join(dir, "sysdev") + device := filepath.Join(dir, "lbd-control") + require.NoError(t, os.WriteFile(sysDev, []byte("10:263\n"), 0644)) + + // Wrong existing nodes must never be overwritten, even without mknod privileges. + require.NoError(t, os.WriteFile(device, []byte("not a device"), 0644)) + require.ErrorContains(t, ensureControlDevice(sysDev, device), "not the expected character device") + require.NoError(t, os.Remove(device)) + require.NoError(t, os.WriteFile(sysDev, []byte("bad-number"), 0644)) + require.ErrorContains(t, ensureControlDevice(sysDev, device), "invalid device number") + require.NoError(t, os.WriteFile(sysDev, []byte("10:263\n"), 0644)) + + err := ensureControlDevice(sysDev, device) + if errors.Is(err, os.ErrPermission) { + t.Skip("mknod requires CAP_MKNOD") + } + require.NoError(t, err) + info, err := os.Stat(device) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeCharDevice) + require.Equal(t, unix.Mkdev(10, 263), uint64(info.Sys().(*syscall.Stat_t).Rdev)) + require.NoError(t, ensureControlDevice(sysDev, device), "an existing matching node is retained") + + require.NoError(t, os.WriteFile(sysDev, []byte("10:264\n"), 0644)) + err = ensureControlDevice(sysDev, device) + require.ErrorContains(t, err, "not the expected character device") +} + +func TestStaleAfterAKernelUpgrade(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + // The marker remembers the kernel the module was built for; the host is + // now running a different one. + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: "6.8.0-45-generic", + ModulePath: modulePath("6.8.0-45-generic"), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + + assert.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "built for kernel 6.8.0-45-generic") + assert.Contains(t, status.Explain(), testRelease+" this host is running") +} + +func TestALoadedButStaleModuleDoesNotReadAsHealthy(t *testing.T) { + // Upgrading miren to a build carrying a newer lbd leaves the old module + // loaded and working. Reporting only "is loaded" would read as healthy + // while a rebuild is pending. + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "v0.0.0-20250101000000-000000000000", + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath, SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + require.True(t, status.Available()) + require.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "miren now bundles lbd") + assert.Contains(t, status.Explain(), "v0.0.0-20250101000000-000000000000") +} + +func TestStaleWhenMirenCarriesANewerLbd(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, modulePath(testRelease), "") + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "v0.0.0-20250101000000-000000000000", + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + assert.True(t, status.Stale()) +} + +func TestStaleWhenTheModuleFileWentAway(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + assert.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "is gone") +} + +func TestMarkerRoundTrip(t *testing.T) { + dataPath := t.TempDir() + + marker, err := readMarker(dataPath) + require.NoError(t, err) + assert.Nil(t, marker, "no record means the host never installed lbd") + + want := Marker{ + LbdVersion: "v0.0.0-20260824210626-be4cec661034", + KernelRelease: testRelease, + BuilderImage: "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-oldhash", + ModulePath: modulePath(testRelease), + LbdctlPath: "/usr/local/bin/lbdctl", + BuiltAt: time.Now().UTC().Truncate(time.Second), + } + require.NoError(t, writeMarker(dataPath, want)) + + got, err := readMarker(dataPath) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, want, *got) + + require.NoError(t, removeMarker(dataPath)) + got, err = readMarker(dataPath) + require.NoError(t, err) + assert.Nil(t, got) + + // Removing an absent record is not an error. + require.NoError(t, removeMarker(dataPath)) +} + +func TestCorruptMarkerIsAnError(t *testing.T) { + dataPath := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dataPath, "lbd"), 0755)) + require.NoError(t, os.WriteFile(markerPath(dataPath), []byte("{not json"), 0644)) + + // Reported rather than treated as absent: silently discarding it would + // strand a module that really is installed. + _, err := readMarker(dataPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "corrupt") + + _, err = Probe(Options{Root: ubuntuRoot(t), DataPath: dataPath}) + require.Error(t, err) +} + +func TestFindLbdctlPrefersTheSearchPath(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "var/lib/miren/release/lbdctl", "") + + assert.Equal(t, "/var/lib/miren/release/lbdctl", + findLbdctl(root, []string{"", "/nowhere", "/var/lib/miren/release"})) + assert.Empty(t, findLbdctl(root, []string{"/nowhere"})) +} + +func TestSearchPathDoesNotWriteIntoTheCallersSlice(t *testing.T) { + // A slice with spare capacity is what makes append dangerous: it writes + // into the caller's backing array instead of allocating. + caller := make([]string, 1, 4) + caller[0] = "/opt/bin" + + opts := Options{SearchPath: caller} + got := opts.searchPath() + + assert.Equal(t, []string{"/opt/bin", systemReleasePath}, got) + assert.Equal(t, []string{"/opt/bin"}, caller, "the caller's slice must be untouched") + + // Extending the caller's slice writes into its spare capacity, which is + // the array searchPath would have appended into. What it returned must not + // move. + extended := append(caller, "/clobbered") + require.Equal(t, "/clobbered", extended[1]) + assert.Equal(t, []string{"/opt/bin", systemReleasePath}, got, + "the returned slice must not alias the caller's array") +} + +func TestMarkerSurvivesATruncatedWrite(t *testing.T) { + // The record is written through a rename, so a crash mid-write cannot + // leave a half-file behind. readMarker treats a corrupt record as an + // error rather than as absent, so a torn write would wedge every probe. + dataPath := t.TempDir() + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + // No temporary file is left behind. + _, err := os.Stat(markerPath(dataPath) + ".tmp") + assert.True(t, os.IsNotExist(err)) + + got, err := readMarker(dataPath) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, testRelease, got.KernelRelease) +} diff --git a/pkg/rpc/authenticator.go b/pkg/rpc/authenticator.go index a37a44735..c23152d4a 100644 --- a/pkg/rpc/authenticator.go +++ b/pkg/rpc/authenticator.go @@ -45,6 +45,12 @@ const ( AuthMethodSigned AuthMethod = "signed" // ed25519-signed request over a message transport ) +// CoordinatorCertSubject is the common name on the certificate the coordinator +// dials with. A runner uses it to tell the coordinator apart from anything else +// that can reach its API. The listener accepts certless connections but +// authenticates non-public calls with a verified client certificate. +const CoordinatorCertSubject = "miren-api" + // Identity represents an authenticated caller type Identity struct { // Subject is the primary identifier (cert CN, JWT subject, etc.) diff --git a/pkg/rpc/service.go b/pkg/rpc/service.go index 937c5b4e4..a399dca01 100644 --- a/pkg/rpc/service.go +++ b/pkg/rpc/service.go @@ -13,6 +13,10 @@ type ServiceID = string const ( ServiceRunner ServiceID = "dev.miren.runtime/runner" + // ServiceNodeAdmin is served by each runner, for work the coordinator asks + // one specific node to do to itself. + ServiceNodeAdmin ServiceID = "dev.miren.runtime/nodeadmin" + // ServiceSqliteBackup stores LTX transaction files replicated from // SQLite-provider disks on runners. ServiceSqliteBackup ServiceID = "dev.miren.runtime/sqlite-backup" diff --git a/servers/build/lbd_builder.go b/servers/build/lbd_builder.go new file mode 100644 index 000000000..401ff5b96 --- /dev/null +++ b/servers/build/lbd_builder.go @@ -0,0 +1,123 @@ +package build + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/tonistiigi/fsutil" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/workloadidentity" +) + +// lbdBuilderLock serializes the toolchain build within one coordinator. Two +// concurrent builds would both succeed -- the registry dedupes by manifest +// digest -- but they would each spend a full image build to get there. +// +// It is a channel rather than a sync.Mutex so a caller can give up: an image +// build takes minutes, and the RPC handler waiting behind one has to stay +// cancellable. +var lbdBuilderLock = make(chan struct{}, 1) + +// LbdToolchain builds the image before workload control starts; it cannot +// depend on the app builder, which is initialized after runner endpoints. +type LbdToolchain struct { + Log *slog.Logger + BuildKit BuildKitProvider + Issuer *workloadidentity.Issuer + EC *entityserver.Client + TempDir string +} + +// EnsureLbdBuilderImage makes sure the lbd toolchain image is in the cluster +// registry and returns the reference nodes should pull. +// +// The image is a base plus a handful of build packages; it carries no lbd +// source. Building it here rather than publishing one means there is no +// released artifact to version and no public registry to depend on, and adding +// a builder for another distribution later costs only a Dockerfile. +// +// It is keyed by a content hash of the embedded Dockerfile and build script, so +// this is a no-op on every call after the first until that content changes. +func (b *LbdToolchain) EnsureLbdBuilderImage(ctx context.Context) (string, error) { + ref := lbdmod.BuilderImage(ocireg.Host) + + if b.present(ctx) { + return ref, nil + } + + select { + case lbdBuilderLock <- struct{}{}: + case <-ctx.Done(): + return "", fmt.Errorf("waiting for another lbd toolchain build to finish: %w", ctx.Err()) + } + defer func() { <-lbdBuilderLock }() + + // Another call may have finished the build while this one waited. + if b.present(ctx) { + return ref, nil + } + + if b.BuildKit == nil { + return "", fmt.Errorf("no buildkit available to build the lbd toolchain image") + } + + dir, err := os.MkdirTemp(b.TempDir, "lbd-builder-") + if err != nil { + return "", fmt.Errorf("creating a build context directory: %w", err) + } + defer os.RemoveAll(dir) + + if err := lbdmod.MaterializeBuilder(dir); err != nil { + return "", err + } + + // The dockerfile frontend takes its context from a real directory, and + // resolves "filename" relative to that root, so the Dockerfile has to sit + // inside the context we just wrote. + dfs, err := fsutil.NewFS(dir) + if err != nil { + return "", fmt.Errorf("opening the build context %s: %w", dir, err) + } + + bkc, err := b.BuildKit.Client(ctx) + if err != nil { + return "", fmt.Errorf("connecting to buildkit: %w", err) + } + defer bkc.Close() + + b.Log.Info("building the lbd toolchain image", "image", ref) + + bk := &Buildkit{Client: bkc, Log: b.Log, WorkloadIssuer: b.Issuer} + res, err := bk.BuildImage(ctx, dfs, BuildStack{ + Stack: "dockerfile", + Input: lbdmod.BuilderDockerfile, + }, lbdmod.BuilderRepository, ref) + if err != nil { + return "", fmt.Errorf("building the lbd toolchain image: %w", err) + } + + b.Log.Info("built the lbd toolchain image", "image", ref, "digest", res.ManifestDigest) + return ref, nil +} + +// present reports whether the toolchain image for this content hash is already +// in the registry. An artifact's entity name is the tag it was pushed under, +// so the tag is the lookup key. +func (b *LbdToolchain) present(ctx context.Context) bool { + if b.EC == nil { + return false + } + + var artifact core_v1alpha.Artifact + if err := b.EC.Get(ctx, lbdmod.BuilderTag(), &artifact); err != nil { + return false + } + // An archived artifact has had, or is about to have, its blobs collected, + // so it cannot be pulled and has to be rebuilt. + return artifact.Status == core_v1alpha.ACTIVE +} diff --git a/servers/build/lbd_builder_test.go b/servers/build/lbd_builder_test.go new file mode 100644 index 000000000..ef580402c --- /dev/null +++ b/servers/build/lbd_builder_test.go @@ -0,0 +1,97 @@ +package build + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/lbdmod" +) + +func TestEnsureLbdBuilderImageSkipsAnExistingImage(t *testing.T) { + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + // The artifact's entity name is the tag it was pushed under, so a prior + // build shows up under exactly this name. + _, err := inmem.Client.Create(ctx, lbdmod.BuilderTag(), + &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + // BuildKit is deliberately nil: finding the image must short-circuit + // before anything tries to build, or every install would rebuild. + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} + + ref, err := b.EnsureLbdBuilderImage(ctx) + require.NoError(t, err) + assert.Equal(t, lbdmod.BuilderImage(ocireg.Host), ref) +} + +func TestEnsureLbdBuilderImageRebuildsAnArchivedImage(t *testing.T) { + // An archived artifact has had, or is about to have, its blobs collected, + // so it cannot be pulled. Treating it as present would hand nodes a + // reference that fails at pull time. + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + _, err := inmem.Client.Create(ctx, lbdmod.BuilderTag(), + &core_v1alpha.Artifact{Status: core_v1alpha.ARCHIVED}) + require.NoError(t, err) + + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} + + _, err = b.EnsureLbdBuilderImage(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no buildkit", "it should have tried to rebuild") +} + +func TestEnsureLbdBuilderImageGivesUpWhenTheCallerDoes(t *testing.T) { + // A toolchain build takes minutes, so a second caller has to be able to + // walk away rather than pinning an RPC handler until the first one lands. + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + lbdBuilderLock <- struct{}{} + defer func() { <-lbdBuilderLock }() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + log := testutils.TestLogger(t) + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} + + done := make(chan error, 1) + go func() { + _, err := b.EnsureLbdBuilderImage(ctx) + done <- err + }() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("it blocked on the build lock instead of honouring the cancelled context") + } +} + +func TestEnsureLbdBuilderImageNeedsBuildkit(t *testing.T) { + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + b := &LbdToolchain{Log: testutils.TestLogger(t), EC: entityserver.NewClient(testutils.TestLogger(t), inmem.EAC)} + + _, err := b.EnsureLbdBuilderImage(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no buildkit available") +} diff --git a/servers/runner/disk_accelerator.go b/servers/runner/disk_accelerator.go new file mode 100644 index 000000000..29e241fa1 --- /dev/null +++ b/servers/runner/disk_accelerator.go @@ -0,0 +1,98 @@ +package runner + +import ( + "context" + "fmt" + + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/api/runner/runner_v1alpha" + "miren.dev/runtime/pkg/rpc" +) + +// InstallDiskAccelerator builds and loads the lbd kernel module on a runner. +// +// The work splits across two machines. The toolchain image is built once, +// here, because the coordinator is where BuildKit and the registry live. The +// module itself is built on the target node, because it has to be compiled +// against the kernel running there and loaded into it. +// +// Failures come back in the result rather than as an RPC error, so an operator +// sees why the install did not happen. +func (s *RegistrationServer) InstallDiskAccelerator(ctx context.Context, req *runner_v1alpha.RunnerRegistrationInstallDiskAccelerator) error { + args := req.Args() + results := req.Results() + + if !args.HasQuery() || args.Query() == "" { + results.SetError("runner name or ID is required") + return nil + } + query := args.Query() + + if s.LbdBuilder == nil { + results.SetError("this cluster cannot build the lbd toolchain image, so accelerator mode is unavailable") + return nil + } + if s.RPC == nil { + results.SetError("no rpc state to reach the runner with") + return nil + } + + node, _, err := s.findNodeByQuery(ctx, query) + if err != nil { + s.Log.Error("failed to find runner", "query", query, "error", err) + results.SetError(err.Error()) + return nil + } + if node == nil { + results.SetError(fmt.Sprintf("runner %q not found", query)) + return nil + } + if node.ApiAddress == "" { + results.SetError(fmt.Sprintf("runner %q has no address to reach it on", query)) + return nil + } + + // Built before dialing: a node that pulls an image the cluster has not + // published yet fails with a registry error that says nothing about why. + image, err := s.LbdBuilder.EnsureLbdBuilderImage(ctx) + if err != nil { + s.Log.Error("failed to build the lbd toolchain image", "error", err) + results.SetError(fmt.Sprintf("building the lbd toolchain image: %v", err)) + return nil + } + + s.Log.Info("installing the lbd kernel module on a runner", + "node", node.ID, "address", node.ApiAddress, "image", image) + + cl, err := s.RPC.Connect(node.ApiAddress, string(rpc.ServiceNodeAdmin)) + if err != nil { + results.SetError(fmt.Sprintf("connecting to runner %q at %s: %v", query, node.ApiAddress, err)) + return nil + } + defer cl.Close() + + nc := &nodeadmin_v1alpha.NodeAdminClient{Client: cl} + res, err := nc.InstallDiskAccelerator(ctx, image, args.HasForce() && args.Force()) + if err != nil { + results.SetError(fmt.Sprintf("installing on runner %q: %v", query, err)) + return nil + } + + if res.HasError() && res.Error() != "" { + results.SetError(res.Error()) + return nil + } + + results.SetName(node.Name) + results.SetKernelRelease(res.KernelRelease()) + results.SetLbdVersion(res.LbdVersion()) + return nil +} + +// LbdBuilderImageEnsurer builds the lbd toolchain image into the cluster +// registry if it is not already there. servers/build.Builder implements it; +// this is an interface so the runner registration server does not depend on +// the whole build server to ask one question of it. +type LbdBuilderImageEnsurer interface { + EnsureLbdBuilderImage(ctx context.Context) (string, error) +} diff --git a/servers/runner/registration.go b/servers/runner/registration.go index dd2b3e514..b146a14b1 100644 --- a/servers/runner/registration.go +++ b/servers/runner/registration.go @@ -53,6 +53,16 @@ type RegistrationServerConfig struct { // do not hold the cluster signing key, request tokens from the coordinator // through this server. May be nil when no issuer is configured. WorkloadIssuer *workloadidentity.Issuer + + // LbdBuilder builds the lbd toolchain image into the cluster registry, so + // a node has something to pull before it compiles the kernel module. Nil + // on a cluster with no BuildKit, where accelerator mode is unavailable. + LbdBuilder LbdBuilderImageEnsurer + + // RPC is how the coordinator reaches a specific runner. Installing the + // kernel module has to happen on the node itself, so unlike the rest of + // this server it is not enough to write an entity and wait. + RPC *rpc.State } type RegistrationServer struct { diff --git a/third_party/lbd/README.md b/third_party/lbd/README.md new file mode 100644 index 000000000..c48c3cd7b --- /dev/null +++ b/third_party/lbd/README.md @@ -0,0 +1,17 @@ +# lbd kernel module source + +`src/` is a verbatim copy of `src/` from [miren.dev/lbd], at the version `go.mod` +pins. It is checked in so `pkg/lbdmod` can embed it in the miren binary and hand +it to the lbd builder image, which compiles it against the running kernel. + +**Do not edit anything under `src/`.** It is generated. Changes to the module +belong in the lbd repo. + +To update: bump `miren.dev/lbd` in `go.mod`, then run `hack/sync-lbd-src.sh`. CI +runs `hack/sync-lbd-src.sh --check` so the copy and the pin cannot drift. +`src/VERSION` records which version the current copy came from. + +`src/lz4/` is LZ4 by Yann Collet, vendored by lbd under its own BSD 2-Clause +license. + +[miren.dev/lbd]: https://github.com/mirendev/lbd diff --git a/third_party/lbd/embed.go b/third_party/lbd/embed.go new file mode 100644 index 000000000..3fc32de15 --- /dev/null +++ b/third_party/lbd/embed.go @@ -0,0 +1,32 @@ +// Package lbdsrc carries the lbd kernel module's C source, embedded in the +// miren binary so a node can compile the module without fetching anything. +// +// The source under src/ is generated by hack/sync-lbd-src.sh from the +// miren.dev/lbd version that go.mod pins; see README.md. +package lbdsrc + +import "embed" + +// FS holds the module source rooted at "src". Its "src/VERSION" file records +// the miren.dev/lbd version the tree was copied from. +// +//go:embed all:src +var FS embed.FS + +// Version reports the miren.dev/lbd version this source was copied from. +func Version() string { + data, err := FS.ReadFile("src/VERSION") + if err != nil { + // VERSION is written by the sync script alongside the source it + // describes, so its absence means the embed itself is broken. + return "unknown" + } + return trimNewline(string(data)) +} + +func trimNewline(s string) string { + for len(s) > 0 && (s[len(s)-1] == '\n' || s[len(s)-1] == '\r') { + s = s[:len(s)-1] + } + return s +} diff --git a/third_party/lbd/src/Makefile b/third_party/lbd/src/Makefile new file mode 100644 index 000000000..fb3281ea9 --- /dev/null +++ b/third_party/lbd/src/Makefile @@ -0,0 +1,33 @@ +ifneq ($(KERNELRELEASE),) +# Called from kernel build system +obj-m := lbd.o +lbd-y := lbd_main.o lbd_qcow2.o lz4/lz4.o + +# Newer kernels re-add explicit old_parent/new_parent dentry fields to +# struct renamedata, which vfs_rename() dereferences directly. Distro version +# numbers don't reliably mark when this landed, so probe the headers for the +# field and let lbd_rename_file() populate the parents when it is present. +LBD_RENAME_PARENT := $(shell grep -qs 'old_parent' $(srctree)/include/linux/fs.h && echo 1 || echo 0) + +ccflags-y := -include $(M)/lz4_kcompat.h -DLBD_HAS_RENAME_PARENT=$(LBD_RENAME_PARENT) +CFLAGS_lz4/lz4.o := -Wno-deprecated-declarations -Wframe-larger-than=32768 + +else +# Called from command line +KDIR ?= /lib/modules/$(shell uname -r)/build + +all: lbd.ko lbdctl + +lbd.ko: lbd_main.c lbd.h lbd_qcow2.c lbd_qcow2.h lbd_qcow2_format.h cbor_enc.h cbor_dec.h lz4_kcompat.h lz4/lz4.c lz4/lz4.h + $(MAKE) -C $(KDIR) M=$(CURDIR) KBUILD_MODPOST_WARN=1 modules + +lbdctl: lbdctl.c lbd.h lbd_qcow2_format.h lz4/lz4.c lz4/lz4.h + $(CC) -Wall -Wextra -O2 -o $@ lbdctl.c lz4/lz4.c + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean + rm -f lbdctl + +.PHONY: all clean + +endif diff --git a/third_party/lbd/src/VERSION b/third_party/lbd/src/VERSION new file mode 100644 index 000000000..ff0b426cb --- /dev/null +++ b/third_party/lbd/src/VERSION @@ -0,0 +1 @@ +v0.0.0-20260824210626-be4cec661034 diff --git a/third_party/lbd/src/cbor_dec.h b/third_party/lbd/src/cbor_dec.h new file mode 100644 index 000000000..0e1262e7d --- /dev/null +++ b/third_party/lbd/src/cbor_dec.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CBOR_DEC_H +#define _CBOR_DEC_H + +/* + * Minimal CBOR decoder (RFC 8949) for kernel use. + * Buffer-based companion to cbor_enc.h. + * Supports major types 0 (uint), 3 (text string), 5 (map). + */ + +#include +#include +#include + +struct cbor_dec { + const u8 *buf; + size_t pos; + size_t len; +}; + +static inline void cbor_dec_init(struct cbor_dec *d, const void *buf, + size_t len) +{ + d->buf = buf; + d->pos = 0; + d->len = len; +} + +/* + * Read a CBOR head: major type (0-7) and argument value. + * Returns 0 on success, -EINVAL on truncation or reserved additional info. + */ +static inline int cbor_dec_head(struct cbor_dec *d, u8 *major, u64 *val) +{ + u8 ib, ai; + + if (d->pos >= d->len) + return -EINVAL; + + ib = d->buf[d->pos++]; + *major = ib >> 5; + ai = ib & 0x1F; + + if (ai < 24) { + *val = ai; + } else if (ai == 24) { + if (d->pos + 1 > d->len) + return -EINVAL; + *val = d->buf[d->pos++]; + } else if (ai == 25) { + if (d->pos + 2 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 8) | d->buf[d->pos + 1]; + d->pos += 2; + } else if (ai == 26) { + if (d->pos + 4 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 24) | + ((u64)d->buf[d->pos + 1] << 16) | + ((u64)d->buf[d->pos + 2] << 8) | + d->buf[d->pos + 3]; + d->pos += 4; + } else if (ai == 27) { + if (d->pos + 8 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 56) | + ((u64)d->buf[d->pos + 1] << 48) | + ((u64)d->buf[d->pos + 2] << 40) | + ((u64)d->buf[d->pos + 3] << 32) | + ((u64)d->buf[d->pos + 4] << 24) | + ((u64)d->buf[d->pos + 5] << 16) | + ((u64)d->buf[d->pos + 6] << 8) | + d->buf[d->pos + 7]; + d->pos += 8; + } else { + return -EINVAL; /* indefinite / reserved */ + } + + return 0; +} + +/* Expect a map header (major 5), returns item count via *count */ +static inline int cbor_dec_map(struct cbor_dec *d, u64 *count) +{ + u8 major; + int ret = cbor_dec_head(d, &major, count); + + if (ret) + return ret; + if (major != 5) + return -EINVAL; + return 0; +} + +/* Expect an unsigned integer (major 0) */ +static inline int cbor_dec_uint(struct cbor_dec *d, u64 *val) +{ + u8 major; + int ret = cbor_dec_head(d, &major, val); + + if (ret) + return ret; + if (major != 0) + return -EINVAL; + return 0; +} + +/* Read a text string (major 3) into buf, NUL-terminated */ +static inline int cbor_dec_text(struct cbor_dec *d, char *buf, size_t cap, + size_t *outlen) +{ + u8 major; + u64 slen; + int ret = cbor_dec_head(d, &major, &slen); + + if (ret) + return ret; + if (major != 3) + return -EINVAL; + if (slen >= cap) + return -EINVAL; + if (d->pos + slen > d->len) + return -EINVAL; + + memcpy(buf, d->buf + d->pos, slen); + buf[slen] = '\0'; + d->pos += slen; + if (outlen) + *outlen = slen; + return 0; +} + +#endif /* _CBOR_DEC_H */ diff --git a/third_party/lbd/src/cbor_enc.h b/third_party/lbd/src/cbor_enc.h new file mode 100644 index 000000000..f6c613441 --- /dev/null +++ b/third_party/lbd/src/cbor_enc.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CBOR_ENC_H +#define _CBOR_ENC_H + +/* + * Minimal CBOR encoder (RFC 8949) for kernel use. + * Supports major types 0 (uint), 2 (byte string), 3 (text string), 5 (map). + */ + +#include +#include +#include + +struct cbor_enc { + u8 *buf; + size_t pos; + size_t cap; + int err; /* sticky -ENOSPC */ +}; + +static inline void cbor_enc_init(struct cbor_enc *e, void *buf, size_t cap) +{ + e->buf = buf; + e->pos = 0; + e->cap = cap; + e->err = 0; +} + +static inline size_t cbor_enc_len(const struct cbor_enc *e) +{ + return e->pos; +} + +/* Write one byte, set sticky error on overflow */ +static inline void cbor_put(struct cbor_enc *e, u8 b) +{ + if (e->err) + return; + if (e->pos >= e->cap) { + e->err = -ENOSPC; + return; + } + e->buf[e->pos++] = b; +} + +/* + * Encode a CBOR head: major type (top 3 bits) + value. + * Chooses the shortest encoding automatically. + */ +static inline void cbor_enc_head(struct cbor_enc *e, u8 major, u64 val) +{ + u8 mt = major << 5; + + if (val < 24) { + cbor_put(e, mt | (u8)val); + } else if (val <= 0xFF) { + cbor_put(e, mt | 24); + cbor_put(e, (u8)val); + } else if (val <= 0xFFFF) { + cbor_put(e, mt | 25); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } else if (val <= 0xFFFFFFFF) { + cbor_put(e, mt | 26); + cbor_put(e, (u8)(val >> 24)); + cbor_put(e, (u8)(val >> 16)); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } else { + cbor_put(e, mt | 27); + cbor_put(e, (u8)(val >> 56)); + cbor_put(e, (u8)(val >> 48)); + cbor_put(e, (u8)(val >> 40)); + cbor_put(e, (u8)(val >> 32)); + cbor_put(e, (u8)(val >> 24)); + cbor_put(e, (u8)(val >> 16)); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } +} + +/* Major type 5: map of count pairs */ +static inline void cbor_enc_map(struct cbor_enc *e, u64 count) +{ + cbor_enc_head(e, 5, count); +} + +/* Major type 0: unsigned integer */ +static inline void cbor_enc_uint(struct cbor_enc *e, u64 val) +{ + cbor_enc_head(e, 0, val); +} + +/* Major type 3: text string (UTF-8) */ +static inline void cbor_enc_text(struct cbor_enc *e, const char *s, size_t len) +{ + size_t i; + + cbor_enc_head(e, 3, len); + for (i = 0; i < len; i++) + cbor_put(e, (u8)s[i]); +} + +/* Convenience: encode a text string key (NUL-terminated) */ +static inline void cbor_enc_text_key(struct cbor_enc *e, const char *key) +{ + cbor_enc_text(e, key, strlen(key)); +} + +/* Major type 2: byte string header only (caller appends raw data) */ +static inline void cbor_enc_bytes_hdr(struct cbor_enc *e, u64 len) +{ + cbor_enc_head(e, 2, len); +} + +#endif /* _CBOR_ENC_H */ diff --git a/third_party/lbd/src/dkms.conf b/third_party/lbd/src/dkms.conf new file mode 100644 index 000000000..9000fa5b1 --- /dev/null +++ b/third_party/lbd/src/dkms.conf @@ -0,0 +1,6 @@ +PACKAGE_NAME="lbd" +PACKAGE_VERSION="0.1.0" +MAKE[0]="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build" +BUILT_MODULE_NAME[0]="lbd" +DEST_MODULE_LOCATION[0]="/extra" +AUTOINSTALL="yes" diff --git a/third_party/lbd/src/lbd.h b/third_party/lbd/src/lbd.h new file mode 100644 index 000000000..ad15413a3 --- /dev/null +++ b/third_party/lbd/src/lbd.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LBD_H +#define _LBD_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd_qcow2.h" + +#define LBD_NAME "lbd" +#define LBD_CTL_NAME "lbd-control" +#define LBD_VERSION "0.1.0" + +#define LBD_BLOCK_SIZE 4096 +#define LBD_QUEUE_DEPTH 128 + +/* Log format constants */ +#define LBD_LOG_VERSION 2 +#define LBD_LOG_PATH_MAX 256 +#define LBD_LOG_MAX_SIZE_DEFAULT (64ULL * 1024 * 1024) /* 64 MiB */ +#define LBD_LOG_MAX_AGE_DEFAULT 60 /* seconds */ +#define LBD_LOG_BUF_SIZE (5ULL * 1024 * 1024) /* 5 MiB */ + +/* Device states */ +enum lbd_state { + LBD_STATE_UNBOUND = 0, + LBD_STATE_BOUND, + LBD_STATE_REMOVING, +}; + +/* ioctl interface */ +#define LBD_CTL_MAGIC 'L' +#define LBD_CTL_ADD _IOWR(LBD_CTL_MAGIC, 0, struct lbd_ctl_add) +#define LBD_CTL_REMOVE _IOW(LBD_CTL_MAGIC, 1, struct lbd_ctl_remove) +#define LBD_CTL_INFO _IOWR(LBD_CTL_MAGIC, 2, struct lbd_ctl_info) + +struct lbd_ctl_add { + char path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + char base_path[LBD_LOG_PATH_MAX]; /* empty string = no base */ + __s32 index; /* out: assigned device index */ + __u64 log_max_size; /* 0 = default */ + __u32 log_max_age_secs; /* 0 = default */ +}; + +struct lbd_ctl_remove { + __s32 index; +}; + +struct lbd_ctl_info { + __s32 index; + __u32 state; + __u64 size; + char path[LBD_LOG_PATH_MAX]; +}; + +/* + * CBOR log key constants (numeric map keys for compactness). + * + * Header map(5): { 1:version, 2:block_size, 3:segment_label, + * 4:device_size, 5:backing_path } + * Write map(7): { 1:"W", 2:timestamp_ns, 3:sequence, 4:block, + * 5:length, 6:crc32, 7:data_bytes } + * Trim map(5): { 1:"T", 2:timestamp_ns, 3:sequence, 4:block, + * 5:length } + */ +#define LBD_CBOR_KEY_HDR_VERSION 1 +#define LBD_CBOR_KEY_HDR_BLOCK_SIZE 2 +#define LBD_CBOR_KEY_HDR_SEGMENT_LABEL 3 +#define LBD_CBOR_KEY_HDR_DEVICE_SIZE 4 +#define LBD_CBOR_KEY_HDR_BACKING_PATH 5 + +#define LBD_CBOR_KEY_OP 1 +#define LBD_CBOR_KEY_TIMESTAMP 2 +#define LBD_CBOR_KEY_SEQUENCE 3 +#define LBD_CBOR_KEY_BLOCK 4 +#define LBD_CBOR_KEY_LENGTH 5 +#define LBD_CBOR_KEY_CHECKSUM 6 +#define LBD_CBOR_KEY_DATA 7 + +/* Watch command keys (write path: userspace -> kernel) */ +#define LBD_WATCH_KEY_CMD 1 +#define LBD_WATCH_KEY_DEV 2 +#define LBD_WATCH_KEY_PATH 3 /* text: file path (for swap) */ + +#ifdef CONFIG_LBD_MISS_HANDLER +/* Miss event keys (kernel -> userspace and userspace -> kernel) */ +#define LBD_MISS_KEY_TYPE 1 /* text: event/command type */ +#define LBD_MISS_KEY_DEV 2 /* uint: device index */ +#define LBD_MISS_KEY_CLUSTER 3 /* uint: cluster index */ +#endif + +/* Event keys (read path: kernel -> userspace) */ +#define LBD_EVENT_KEY_TYPE 1 +#define LBD_EVENT_KEY_DEV 2 +#define LBD_EVENT_KEY_LABEL 3 +#define LBD_EVENT_KEY_DIR 4 +#define LBD_EVENT_KEY_SEQ 5 +#define LBD_EVENT_KEY_SIZE 6 + +#ifdef CONFIG_LBD_MISS_HANDLER +/* Forward declaration for miss handler */ +struct lbd_miss_handler; + +enum lbd_miss_action { + LBD_MISS_CONTINUE = 0, + LBD_MISS_RETRY, +}; +#endif + +/* Per-request data (embedded in blk-mq PDU) */ +struct lbd_cmd { + struct list_head list_entry; + int ret; +}; + +/* Per-device state */ +struct lbd_device { + int index; + enum lbd_state state; + atomic_t open_count; + + struct gendisk *gd; + struct blk_mq_tag_set tag_set; + + struct file *backing_file; + struct file *log_file; + char backing_path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + struct path log_dir_path; + loff_t size; /* device size in bytes */ + + struct workqueue_struct *wq; + struct work_struct work; + struct list_head cmd_list; + spinlock_t cmd_lock; + + /* Log segmentation — all protected by log_mutex */ + struct mutex log_mutex; + bool log_has_entries; /* current segment has data entries */ + u64 log_seq; /* global, never resets */ + char log_segment_label[25]; /* current segment TAI64N label */ + u64 log_max_size; /* rotation size threshold */ + u32 log_max_age_secs; /* rotation age threshold */ + struct delayed_work log_rotate_dwork; /* age timer */ + void *log_buf; /* write buffer, allocated once */ + size_t log_buf_used; /* bytes currently in buffer */ + + void *lz4_state; /* LZ4 compression state, allocated once */ + + /* qcow2-lz4 backing store */ + bool is_qcow2; + struct lbd_qcow2 qcow2; + + /* Thin snapshot base layer */ + struct lbd_qcow2_base *base; /* NULL when no base configured */ + char base_path[LBD_LOG_PATH_MAX]; + +#ifdef CONFIG_LBD_MISS_HANDLER + /* Block miss handler (for remote-fetch workflow) */ + struct lbd_miss_handler *miss_handler; /* NULL when no handler */ + spinlock_t miss_handler_lock; /* protects miss_handler pointer */ +#endif + + /* I/O stats (atomic for lock-free sysfs reads) */ + atomic64_t stat_reads; + atomic64_t stat_writes; + atomic64_t stat_trims; + atomic64_t stat_read_bytes; + atomic64_t stat_write_bytes; + atomic64_t stat_trim_bytes; + + /* Allocation stats (qcow2 space reuse) */ + atomic64_t stat_alloc_reused; /* clusters written in-place or via free list */ + atomic64_t stat_alloc_new; /* clusters allocated by appending */ + atomic64_t stat_alloc_freed; /* extents added to the free list */ + atomic64_t stat_compressed; /* clusters stored compressed */ + atomic64_t stat_uncompressed; /* clusters stored uncompressed */ + + /* Log stats */ + atomic64_t stat_log_rotations; + ktime_t segment_start_time; +}; + +/* Kernel version compatibility */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) +#define LBD_HAS_BLK_MQ_ALLOC_DISK 1 +#else +#define LBD_HAS_BLK_MQ_ALLOC_DISK 0 +#endif + +/* 6.3+: struct mnt_idmap replaces struct user_namespace for VFS idmap ops; + * nop_mnt_idmap replaces &init_user_ns */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) +#define LBD_HAS_MNT_IDMAP 1 +#else +#define LBD_HAS_MNT_IDMAP 0 +#endif + +/* 6.5+: block_device_operations.open takes struct gendisk * (was block_device *); + * .release drops the fmode_t argument */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) +#define LBD_HAS_GENDISK_OPEN 1 +#else +#define LBD_HAS_GENDISK_OPEN 0 +#endif + +/* 6.9+: blk_mq_alloc_disk() takes struct queue_limits * as 2nd arg; + * BLK_MQ_F_SHOULD_MERGE removed; blk_queue_* setters removed; + * use BLK_FEAT_WRITE_CACHE in limits.features */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0) +#define LBD_HAS_QUEUE_LIMITS_API 1 +#else +#define LBD_HAS_QUEUE_LIMITS_API 0 +#endif + +/* 6.12+: struct renamedata drops old_dir/new_dir fields (derived from dentries); + * lookup_one_len() removed, use lookup_one(idmap, name, dir, len) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) +#define LBD_HAS_RENAME_NO_DIR 1 +#else +#define LBD_HAS_RENAME_NO_DIR 0 +#endif + +/* 6.15+: lookup_one() takes struct qstr * instead of (const char *, int) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) +#define LBD_HAS_LOOKUP_ONE_QSTR 1 +#else +#define LBD_HAS_LOOKUP_ONE_QSTR 0 +#endif + +/* 6.18+: struct renamedata uses single mnt_idmap field + * (replaces old_mnt_idmap/new_mnt_idmap) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) +#define LBD_HAS_RENAME_SINGLE_IDMAP 1 +#else +#define LBD_HAS_RENAME_SINGLE_IDMAP 0 +#endif + +/* Newer kernels re-add explicit old_parent/new_parent dentry fields to + * struct renamedata; vfs_rename() then dereferences them instead of deriving + * the parents from the dentries, so they must be populated or the rename + * faults on a NULL parent. The exact version varies by distro (the field was + * seen on a 7.0.x tree), so src/Makefile probes the kernel headers for the + * member and defines LBD_HAS_RENAME_PARENT; default off when unprobed (e.g. + * the userspace lbdctl build, which never renames through the VFS). */ +#ifndef LBD_HAS_RENAME_PARENT +#define LBD_HAS_RENAME_PARENT 0 +#endif + +#endif /* _LBD_H */ diff --git a/third_party/lbd/src/lbd_main.c b/third_party/lbd/src/lbd_main.c new file mode 100644 index 000000000..3085a9c14 --- /dev/null +++ b/third_party/lbd/src/lbd_main.c @@ -0,0 +1,2146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LBD - Logging Block Device + * + * A block device backed by a file (like loop) that logs all write + * operations to a companion .log file for change tracking / audit / replay. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd.h" + +#if LBD_HAS_MNT_IDMAP +#include +#endif +#include "lbd_qcow2.h" +#include "cbor_enc.h" +#include "cbor_dec.h" +#include "lz4_kcompat.h" +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("lbd authors"); +MODULE_DESCRIPTION("Logging Block Device"); +MODULE_VERSION(LBD_VERSION); + +static int lbd_major; +static DEFINE_IDR(lbd_devices); +static DEFINE_MUTEX(lbd_devices_mutex); +static struct miscdevice lbd_misc; + +/* ---------------------------------------------------------------- + * Log rotation watchers + * ---------------------------------------------------------------- */ + +#define LBD_WATCHER_QUEUE_SIZE 64 + +struct lbd_watcher_event { + int dev_index; + u64 log_seq; + u64 device_size; + char segment_label[25]; + char log_dir[LBD_LOG_PATH_MAX]; +}; + +struct lbd_watcher { + struct list_head list; + spinlock_t lock; + wait_queue_head_t wq; + struct lbd_watcher_event queue[LBD_WATCHER_QUEUE_SIZE]; + unsigned int head, tail, count; + int filter_dev; /* -1 = all */ +}; + +static LIST_HEAD(lbd_watchers); +static DEFINE_SPINLOCK(lbd_watchers_lock); + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Block miss handler (for remote-fetch workflow) + * ---------------------------------------------------------------- */ + +struct lbd_miss_pending { + struct completion done; + enum lbd_miss_action action; +}; + +struct lbd_miss_handler { + spinlock_t lock; + wait_queue_head_t wq; + int dev_index; + struct lbd_miss_pending *pending; /* NULL = idle */ + bool has_event; + u64 miss_cluster; +}; +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* Control fd state — supports both watch and miss handler on same fd */ +struct lbd_ctl_state { + struct lbd_watcher *watcher; /* NULL until "watch" command */ +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *miss; /* NULL until "manage_misses" command */ +#endif +}; + +/* ---------------------------------------------------------------- + * Block device operations + * ---------------------------------------------------------------- */ + +#if LBD_HAS_GENDISK_OPEN +static int lbd_open(struct gendisk *disk, unsigned int mode) +{ + struct lbd_device *dev = disk->private_data; +#else +static int lbd_open(struct block_device *bdev, fmode_t mode) +{ + struct lbd_device *dev = bdev->bd_disk->private_data; +#endif + + if (dev->state != LBD_STATE_BOUND) + return -ENXIO; + atomic_inc(&dev->open_count); + return 0; +} + +#if LBD_HAS_GENDISK_OPEN +static void lbd_release(struct gendisk *disk) +{ + struct lbd_device *dev = disk->private_data; +#else +static void lbd_release(struct gendisk *disk, fmode_t mode) +{ + struct lbd_device *dev = disk->private_data; +#endif + + atomic_dec(&dev->open_count); +} + +static const struct block_device_operations lbd_fops = { + .owner = THIS_MODULE, + .open = lbd_open, + .release = lbd_release, +}; + +/* ---------------------------------------------------------------- + * I/O: read path + * ---------------------------------------------------------------- */ + +static int lbd_do_read(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + ssize_t ret; + + if (dev->is_qcow2) + return lbd_qcow2_read(dev, rq); + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + + ret = kernel_read(dev->backing_file, mapped + bvec.bv_offset, + bvec.bv_len, &pos); + kunmap_local(mapped); + + if (ret != bvec.bv_len) { + if (ret >= 0) + ret = -EIO; + return ret; + } + } + + atomic64_inc(&dev->stat_reads); + atomic64_add(blk_rq_bytes(rq), &dev->stat_read_bytes); + return 0; +} + +/* Forward declarations */ +static const struct attribute_group lbd_attr_group; + +/* Forward declarations for log segmentation */ +static int lbd_open_log_file(struct lbd_device *dev); +static void lbd_rotate_segment(struct lbd_device *dev); +static void lbd_maybe_rotate_segment(struct lbd_device *dev); +static void lbd_notify_watchers(struct lbd_device *dev); + +/* ---------------------------------------------------------------- + * I/O: log write buffer + * ---------------------------------------------------------------- */ + +static int lbd_log_flush(struct lbd_device *dev) +{ + loff_t pos; + ssize_t ret; + + if (dev->log_buf_used == 0) + return 0; + + /* Lazy-open: create the segment file on first flush */ + if (!dev->log_file) { + int err = lbd_open_log_file(dev); + if (err) { + dev->log_buf_used = 0; + return err; + } + } + + pos = i_size_read(file_inode(dev->log_file)); + ret = kernel_write(dev->log_file, dev->log_buf, + dev->log_buf_used, &pos); + if (ret != dev->log_buf_used) { + pr_warn_ratelimited("lbd%d: log flush failed (%zd)\n", + dev->index, ret); + dev->log_buf_used = 0; + return ret < 0 ? ret : -EIO; + } + dev->log_buf_used = 0; + return 0; +} + +static int lbd_log_buf_append(struct lbd_device *dev, + const void *data, size_t len) +{ + if (dev->log_buf_used + len > LBD_LOG_BUF_SIZE) { + int ret = lbd_log_flush(dev); + if (ret) + return ret; + } + memcpy(dev->log_buf + dev->log_buf_used, data, len); + dev->log_buf_used += len; + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: log write + * ---------------------------------------------------------------- */ + +static void lbd_log_write(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + u32 data_len = blk_rq_bytes(rq); + void *data_buf; + void *comp_buf = NULL; + int comp_len; + int comp_cap; + size_t offset = 0; + u8 tmp[128]; + struct cbor_enc e; + u32 crc; + u64 ts, block; + int ret; + + if (!data_len) + return; + + data_buf = kvmalloc(data_len, GFP_NOIO); + if (!data_buf) { + pr_warn_ratelimited("lbd%d: log alloc failed\n", dev->index); + return; + } + + /* Gather write data from bio vecs */ + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + memcpy(data_buf + offset, mapped + bvec.bv_offset, bvec.bv_len); + kunmap_local(mapped); + offset += bvec.bv_len; + } + + /* CRC on uncompressed data */ + crc = crc32(~0U, data_buf, data_len) ^ ~0U; + + /* LZ4-compress the data */ + comp_cap = LZ4_compressBound(data_len); + comp_buf = kvmalloc(comp_cap, GFP_NOIO); + if (!comp_buf) { + pr_warn_ratelimited("lbd%d: log compress alloc failed\n", + dev->index); + kvfree(data_buf); + return; + } + + comp_len = LZ4_compress_fast_extState(dev->lz4_state, + data_buf, comp_buf, + data_len, comp_cap, 1); + if (comp_len <= 0) { + pr_warn_ratelimited("lbd%d: LZ4 compression failed\n", + dev->index); + kvfree(comp_buf); + kvfree(data_buf); + return; + } + + ts = ktime_get_real_ns(); + block = (loff_t)blk_rq_pos(rq) * 512 / LBD_BLOCK_SIZE; + + mutex_lock(&dev->log_mutex); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 7); + + cbor_enc_uint(&e, LBD_CBOR_KEY_OP); + cbor_enc_text(&e, "W", 1); + + cbor_enc_uint(&e, LBD_CBOR_KEY_TIMESTAMP); + cbor_enc_uint(&e, ts); + + cbor_enc_uint(&e, LBD_CBOR_KEY_SEQUENCE); + cbor_enc_uint(&e, dev->log_seq++); + + cbor_enc_uint(&e, LBD_CBOR_KEY_BLOCK); + cbor_enc_uint(&e, block); + + /* Length = uncompressed size (needed for decompression) */ + cbor_enc_uint(&e, LBD_CBOR_KEY_LENGTH); + cbor_enc_uint(&e, data_len); + + cbor_enc_uint(&e, LBD_CBOR_KEY_CHECKSUM); + cbor_enc_uint(&e, crc); + + /* Data = LZ4-compressed bytes */ + cbor_enc_uint(&e, LBD_CBOR_KEY_DATA); + cbor_enc_bytes_hdr(&e, comp_len); + + if (e.err) { + dev->log_seq--; + goto warn; + } + + ret = lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); + if (ret) + goto warn; + + ret = lbd_log_buf_append(dev, comp_buf, comp_len); + if (ret) + goto warn; + + dev->log_has_entries = true; + lbd_maybe_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + kvfree(comp_buf); + kvfree(data_buf); + return; + +warn: + mutex_unlock(&dev->log_mutex); + kvfree(comp_buf); + kvfree(data_buf); + pr_warn_ratelimited("lbd%d: log write failed (%d)\n", dev->index, ret); +} + +/* ---------------------------------------------------------------- + * I/O: write path + * ---------------------------------------------------------------- */ + +static int lbd_do_write(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + ssize_t ret; + + if (dev->is_qcow2) { + ret = lbd_qcow2_write(dev, rq); + if (ret) + return ret; + lbd_log_write(dev, rq); + atomic64_inc(&dev->stat_writes); + atomic64_add(blk_rq_bytes(rq), &dev->stat_write_bytes); + return 0; + } + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + + ret = kernel_write(dev->backing_file, mapped + bvec.bv_offset, + bvec.bv_len, &pos); + kunmap_local(mapped); + + if (ret != bvec.bv_len) { + if (ret >= 0) + ret = -EIO; + return ret; + } + } + + /* Log the write - failure is non-fatal */ + lbd_log_write(dev, rq); + + atomic64_inc(&dev->stat_writes); + atomic64_add(blk_rq_bytes(rq), &dev->stat_write_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: discard (TRIM) path + * ---------------------------------------------------------------- */ + +static void lbd_log_discard(struct lbd_device *dev, struct request *rq) +{ + u8 tmp[64]; + struct cbor_enc e; + u64 ts = ktime_get_real_ns(); + u64 block = (loff_t)blk_rq_pos(rq) * 512 / LBD_BLOCK_SIZE; + u32 length = blk_rq_bytes(rq); + int ret; + + mutex_lock(&dev->log_mutex); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 5); + + cbor_enc_uint(&e, LBD_CBOR_KEY_OP); + cbor_enc_text(&e, "T", 1); + + cbor_enc_uint(&e, LBD_CBOR_KEY_TIMESTAMP); + cbor_enc_uint(&e, ts); + + cbor_enc_uint(&e, LBD_CBOR_KEY_SEQUENCE); + cbor_enc_uint(&e, dev->log_seq++); + + cbor_enc_uint(&e, LBD_CBOR_KEY_BLOCK); + cbor_enc_uint(&e, block); + + cbor_enc_uint(&e, LBD_CBOR_KEY_LENGTH); + cbor_enc_uint(&e, length); + + if (e.err) { + dev->log_seq--; + goto warn; + } + + ret = lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); + if (ret) + goto warn; + + dev->log_has_entries = true; + lbd_maybe_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + return; + +warn: + mutex_unlock(&dev->log_mutex); + pr_warn_ratelimited("lbd%d: log discard failed (%d)\n", dev->index, ret); +} + +static int lbd_do_discard(struct lbd_device *dev, struct request *rq) +{ + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + unsigned int len = blk_rq_bytes(rq); + int ret; + + if (dev->is_qcow2) { + ret = lbd_qcow2_discard(dev, rq); + if (ret) + return ret; + lbd_log_discard(dev, rq); + atomic64_inc(&dev->stat_trims); + atomic64_add(len, &dev->stat_trim_bytes); + return 0; + } + + if (dev->backing_file->f_op->fallocate) { + ret = dev->backing_file->f_op->fallocate(dev->backing_file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, pos, len); + if (ret && ret != -EINVAL && ret != -EOPNOTSUPP) + return ret; + } + + lbd_log_discard(dev, rq); + + atomic64_inc(&dev->stat_trims); + atomic64_add(len, &dev->stat_trim_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: request dispatch + * ---------------------------------------------------------------- */ + +static void lbd_handle_request(struct lbd_device *dev, struct request *rq) +{ + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + int ret; + + switch (req_op(rq)) { + case REQ_OP_READ: + ret = lbd_do_read(dev, rq); + break; + case REQ_OP_WRITE: + ret = lbd_do_write(dev, rq); + break; + case REQ_OP_DISCARD: + ret = lbd_do_discard(dev, rq); + break; + case REQ_OP_FLUSH: + ret = vfs_fsync(dev->backing_file, 0); + if (!ret) { + mutex_lock(&dev->log_mutex); + if (dev->log_has_entries) + lbd_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + } + break; + default: + ret = -EIO; + break; + } + + cmd->ret = ret; +} + +static void lbd_work_fn(struct work_struct *work) +{ + struct lbd_device *dev = container_of(work, struct lbd_device, work); + struct lbd_cmd *cmd; + LIST_HEAD(local_list); + unsigned int saved_flags = current->flags; + + /* Prevent writeback deadlock (same technique as loop driver) */ + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + + spin_lock_irq(&dev->cmd_lock); + list_splice_init(&dev->cmd_list, &local_list); + spin_unlock_irq(&dev->cmd_lock); + + while (!list_empty(&local_list)) { + cmd = list_first_entry(&local_list, struct lbd_cmd, list_entry); + list_del(&cmd->list_entry); + + lbd_handle_request(dev, + blk_mq_rq_from_pdu(cmd)); + + blk_mq_complete_request(blk_mq_rq_from_pdu(cmd)); + } + + current->flags = (current->flags & ~(PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO)) | + (saved_flags & (PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO)); +} + +static void lbd_complete_rq(struct request *rq) +{ + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + + blk_mq_end_request(rq, cmd->ret ? BLK_STS_IOERR : BLK_STS_OK); +} + +static blk_status_t lbd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) +{ + struct lbd_device *dev = hctx->queue->queuedata; + struct request *rq = bd->rq; + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + + blk_mq_start_request(rq); + + if (dev->state != LBD_STATE_BOUND) + return BLK_STS_IOERR; + + INIT_LIST_HEAD(&cmd->list_entry); + cmd->ret = 0; + + spin_lock_irq(&dev->cmd_lock); + list_add_tail(&cmd->list_entry, &dev->cmd_list); + spin_unlock_irq(&dev->cmd_lock); + + queue_work(dev->wq, &dev->work); + return BLK_STS_OK; +} + +static const struct blk_mq_ops lbd_mq_ops = { + .queue_rq = lbd_queue_rq, + .complete = lbd_complete_rq, +}; + +/* ---------------------------------------------------------------- + * Device lifecycle + * ---------------------------------------------------------------- */ + +static int lbd_write_log_header(struct lbd_device *dev) +{ + u8 tmp[512]; + struct cbor_enc e; + size_t path_len = strlen(dev->backing_path); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 5); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_VERSION); + cbor_enc_uint(&e, LBD_LOG_VERSION); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_BLOCK_SIZE); + cbor_enc_uint(&e, LBD_BLOCK_SIZE); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_SEGMENT_LABEL); + cbor_enc_text(&e, dev->log_segment_label, + strlen(dev->log_segment_label)); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_DEVICE_SIZE); + cbor_enc_uint(&e, dev->size); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_BACKING_PATH); + cbor_enc_text(&e, dev->backing_path, path_len); + + if (e.err) + return e.err; + + return lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); +} + +/* ---------------------------------------------------------------- + * Log segmentation + * ---------------------------------------------------------------- */ + +static int lbd_log_name_tmp(const char *label, char *buf, size_t sz) +{ + return snprintf(buf, sz, "disk.%s.log.tmp", label); +} + +static int lbd_log_name_final(const char *label, char *buf, size_t sz) +{ + return snprintf(buf, sz, "disk.%s.log", label); +} + +/* + * Generate a TAI64N label from the current wall-clock time. + * Format: 16 hex digits (TAI seconds) + 8 hex digits (nanoseconds). + * buf must be at least 25 bytes (24 chars + NUL). + */ +static void lbd_tai64n_label(char *buf) +{ + struct timespec64 ts; + u64 tai_secs; + + ktime_get_real_ts64(&ts); + tai_secs = (u64)ts.tv_sec + 0x4000000000000000ULL; + snprintf(buf, 25, "%016llx%08lx", tai_secs, ts.tv_nsec); +} + +static int lbd_rename_file(struct file *old_file, const char *new_basename) +{ + struct dentry *old_dentry = old_file->f_path.dentry; + struct dentry *parent = old_dentry->d_parent; + struct dentry *new_dentry; + struct renamedata rd; + int ret; + + lock_rename(parent, parent); + +#if LBD_HAS_LOOKUP_ONE_QSTR + { + struct qstr qname = QSTR_INIT(new_basename, + strlen(new_basename)); + new_dentry = lookup_one(&nop_mnt_idmap, &qname, parent); + } +#elif LBD_HAS_RENAME_NO_DIR + new_dentry = lookup_one(&nop_mnt_idmap, new_basename, parent, + strlen(new_basename)); +#else + new_dentry = lookup_one_len(new_basename, parent, strlen(new_basename)); +#endif + if (IS_ERR(new_dentry)) { + ret = PTR_ERR(new_dentry); + goto out_unlock; + } + + memset(&rd, 0, sizeof(rd)); +#if LBD_HAS_RENAME_PARENT + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_parent = parent; + rd.old_dentry = old_dentry; + rd.new_parent = parent; + rd.new_dentry = new_dentry; +#elif LBD_HAS_RENAME_SINGLE_IDMAP + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_dentry = old_dentry; + rd.new_dentry = new_dentry; +#elif LBD_HAS_RENAME_NO_DIR + rd.old_mnt_idmap = &nop_mnt_idmap; + rd.old_dentry = old_dentry; + rd.new_mnt_idmap = &nop_mnt_idmap; + rd.new_dentry = new_dentry; +#elif LBD_HAS_MNT_IDMAP + rd.old_mnt_idmap = &nop_mnt_idmap; + rd.old_dir = d_inode(parent); + rd.old_dentry = old_dentry; + rd.new_mnt_idmap = &nop_mnt_idmap; + rd.new_dir = d_inode(parent); + rd.new_dentry = new_dentry; +#else + rd.old_mnt_userns = &init_user_ns; + rd.old_dir = d_inode(parent); + rd.old_dentry = old_dentry; + rd.new_mnt_userns = &init_user_ns; + rd.new_dir = d_inode(parent); + rd.new_dentry = new_dentry; +#endif + + ret = vfs_rename(&rd); + dput(new_dentry); + +out_unlock: + unlock_rename(parent, parent); + return ret; +} + +/* + * Lazy-open the .log.tmp file for the current segment. Called from + * lbd_log_flush() when buffered data first needs to hit disk. + */ +static int lbd_open_log_file(struct lbd_device *dev) +{ + char name[48]; + struct file *f; + + lbd_log_name_tmp(dev->log_segment_label, name, sizeof(name)); + + /* + * Open relative to the log directory rather than using an + * absolute path. This avoids path resolution failures when + * called from a kworker whose mount namespace differs from + * the process that created the device. + */ + f = file_open_root(&dev->log_dir_path, name, + O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + if (IS_ERR(f)) { + pr_warn("lbd%d: cannot open %s: %ld\n", + dev->index, name, PTR_ERR(f)); + return PTR_ERR(f); + } + + dev->log_file = f; + return 0; +} + +/* + * Begin a new log segment. Resets the buffer, generates a TAI64N + * label, and writes the CBOR header into the buffer. No file is + * created on disk — that happens lazily in lbd_log_flush(). + */ +static void lbd_begin_segment(struct lbd_device *dev) +{ + dev->log_file = NULL; + dev->log_buf_used = 0; + dev->log_has_entries = false; + dev->segment_start_time = ktime_get(); + lbd_tai64n_label(dev->log_segment_label); + lbd_write_log_header(dev); +} + +static void lbd_rotate_segment(struct lbd_device *dev) +{ + char final_name[48]; + int ret; + + if (!dev->log_has_entries) + return; + + /* Flush buffer (lazy-opens .log.tmp) then fsync */ + lbd_log_flush(dev); + if (!dev->log_file) + goto next; + + vfs_fsync(dev->log_file, 0); + + /* Rename disk..log.tmp -> disk..log */ + lbd_log_name_final(dev->log_segment_label, final_name, + sizeof(final_name)); + + ret = lbd_rename_file(dev->log_file, final_name); + if (ret) + pr_warn("lbd%d: rename to %s failed: %d\n", + dev->index, final_name, ret); + + fput(dev->log_file); + dev->log_file = NULL; + +next: + atomic64_inc(&dev->stat_log_rotations); + + /* Notify watchers before lbd_begin_segment() overwrites the label */ + lbd_notify_watchers(dev); + + /* Begin fresh segment (buffer only, no file yet) */ + lbd_begin_segment(dev); + + /* Reschedule age timer */ + if (dev->log_max_age_secs > 0) + mod_delayed_work(system_wq, &dev->log_rotate_dwork, + msecs_to_jiffies(dev->log_max_age_secs * 1000)); +} + +static void lbd_log_rotate_work_fn(struct work_struct *work) +{ + struct lbd_device *dev = container_of(work, struct lbd_device, + log_rotate_dwork.work); + + mutex_lock(&dev->log_mutex); + if (dev->log_has_entries) + lbd_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); +} + +static void lbd_maybe_rotate_segment(struct lbd_device *dev) +{ + loff_t file_size = dev->log_file ? + i_size_read(file_inode(dev->log_file)) : 0; + + if (file_size + dev->log_buf_used >= (loff_t)dev->log_max_size) + lbd_rotate_segment(dev); +} + +static void lbd_finalize_log(struct lbd_device *dev) +{ + char final_name[48]; + + if (dev->log_has_entries) + lbd_log_flush(dev); + + if (!dev->log_file) + return; + + vfs_fsync(dev->log_file, 0); + + if (dev->log_has_entries) { + lbd_log_name_final(dev->log_segment_label, final_name, + sizeof(final_name)); + lbd_rename_file(dev->log_file, final_name); + lbd_notify_watchers(dev); + } + + fput(dev->log_file); + dev->log_file = NULL; +} + +static void lbd_destroy_device(struct lbd_device *dev) +{ + dev->state = LBD_STATE_REMOVING; + + cancel_delayed_work_sync(&dev->log_rotate_dwork); + + device_remove_group(disk_to_dev(dev->gd), &lbd_attr_group); + del_gendisk(dev->gd); + flush_workqueue(dev->wq); + destroy_workqueue(dev->wq); + + lbd_finalize_log(dev); + + if (dev->base) + lbd_qcow2_base_destroy(dev->base); + + if (dev->is_qcow2) + lbd_qcow2_destroy(dev); + + kvfree(dev->lz4_state); + kvfree(dev->log_buf); + path_put(&dev->log_dir_path); + if (dev->backing_file) { + vfs_fsync(dev->backing_file, 0); + fput(dev->backing_file); + } + + put_disk(dev->gd); + blk_mq_free_tag_set(&dev->tag_set); + kfree(dev); +} + +static int lbd_add_device(struct lbd_ctl_add __user *uarg) +{ + struct lbd_ctl_add arg; + struct lbd_device *dev; + struct inode *inode; + int ret, idx; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + arg.path[LBD_LOG_PATH_MAX - 1] = '\0'; + arg.log_dir[LBD_LOG_PATH_MAX - 1] = '\0'; + arg.base_path[LBD_LOG_PATH_MAX - 1] = '\0'; + + if (arg.log_dir[0] == '\0') { + pr_err("lbd: log_dir is required\n"); + return -EINVAL; + } + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + strscpy(dev->backing_path, arg.path, sizeof(dev->backing_path)); + strscpy(dev->log_dir, arg.log_dir, sizeof(dev->log_dir)); + dev->state = LBD_STATE_UNBOUND; + spin_lock_init(&dev->cmd_lock); +#ifdef CONFIG_LBD_MISS_HANDLER + spin_lock_init(&dev->miss_handler_lock); +#endif + INIT_LIST_HEAD(&dev->cmd_list); + INIT_WORK(&dev->work, lbd_work_fn); + mutex_init(&dev->log_mutex); + dev->log_seq = 0; + dev->log_has_entries = false; + dev->log_max_size = arg.log_max_size ? arg.log_max_size + : LBD_LOG_MAX_SIZE_DEFAULT; + dev->log_max_age_secs = arg.log_max_age_secs ? arg.log_max_age_secs + : LBD_LOG_MAX_AGE_DEFAULT; + INIT_DELAYED_WORK(&dev->log_rotate_dwork, lbd_log_rotate_work_fn); + + dev->log_buf = kvmalloc(LBD_LOG_BUF_SIZE, GFP_KERNEL); + if (!dev->log_buf) { + ret = -ENOMEM; + goto err_free; + } + dev->log_buf_used = 0; + + dev->lz4_state = kvmalloc(LZ4_sizeofState(), GFP_KERNEL); + if (!dev->lz4_state) { + ret = -ENOMEM; + goto err_free; + } + + /* Allocate device index */ + mutex_lock(&lbd_devices_mutex); + idx = idr_alloc(&lbd_devices, dev, 0, 256, GFP_KERNEL); + mutex_unlock(&lbd_devices_mutex); + if (idx < 0) { + ret = idx; + goto err_free; + } + dev->index = idx; + + /* Setup blk-mq tag set */ + memset(&dev->tag_set, 0, sizeof(dev->tag_set)); + dev->tag_set.ops = &lbd_mq_ops; + dev->tag_set.nr_hw_queues = 1; + dev->tag_set.queue_depth = LBD_QUEUE_DEPTH; + dev->tag_set.numa_node = NUMA_NO_NODE; + dev->tag_set.cmd_size = sizeof(struct lbd_cmd); +#if LBD_HAS_QUEUE_LIMITS_API + dev->tag_set.flags = 0; +#else + dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; +#endif + + ret = blk_mq_alloc_tag_set(&dev->tag_set); + if (ret) + goto err_idr; + + /* Allocate gendisk - version dependent */ +#if LBD_HAS_QUEUE_LIMITS_API + { + struct queue_limits lim = { + .logical_block_size = LBD_BLOCK_SIZE, + .physical_block_size = LBD_BLOCK_SIZE, + .max_hw_sectors = 256, + .features = BLK_FEAT_WRITE_CACHE, + .max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT, + .discard_granularity = LBD_BLOCK_SIZE, + }; + dev->gd = blk_mq_alloc_disk(&dev->tag_set, &lim, dev); + } + if (IS_ERR(dev->gd)) { + ret = PTR_ERR(dev->gd); + dev->gd = NULL; + goto err_tagset; + } +#elif LBD_HAS_BLK_MQ_ALLOC_DISK + dev->gd = blk_mq_alloc_disk(&dev->tag_set, dev); + if (IS_ERR(dev->gd)) { + ret = PTR_ERR(dev->gd); + dev->gd = NULL; + goto err_tagset; + } +#else + { + struct request_queue *q; + q = blk_mq_init_queue(&dev->tag_set); + if (IS_ERR(q)) { + ret = PTR_ERR(q); + goto err_tagset; + } + dev->gd = alloc_disk(1); + if (!dev->gd) { + blk_cleanup_queue(q); + ret = -ENOMEM; + goto err_tagset; + } + dev->gd->queue = q; + } +#endif + + dev->gd->major = lbd_major; + dev->gd->first_minor = dev->index; + dev->gd->minors = 1; + dev->gd->fops = &lbd_fops; + dev->gd->private_data = dev; + snprintf(dev->gd->disk_name, DISK_NAME_LEN, "lbd%d", dev->index); + +#if !LBD_HAS_BLK_MQ_ALLOC_DISK + dev->gd->queue->queuedata = dev; +#endif + + /* Create workqueue */ + dev->wq = alloc_workqueue("lbd%d", WQ_UNBOUND | WQ_MEM_RECLAIM, 0, + dev->index); + if (!dev->wq) { + ret = -ENOMEM; + goto err_disk; + } + + /* Open backing file */ + dev->backing_file = filp_open(arg.path, O_RDWR | O_LARGEFILE, 0); + if (IS_ERR(dev->backing_file)) { + ret = PTR_ERR(dev->backing_file); + dev->backing_file = NULL; + pr_err("lbd: cannot open backing file '%s': %d\n", arg.path, ret); + goto err_wq; + } + + inode = file_inode(dev->backing_file); + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + pr_err("lbd: backing path must be a regular file\n"); + goto err_backing; + } + + dev->size = i_size_read(inode); + if (dev->size == 0) { + ret = -EINVAL; + pr_err("lbd: backing file is empty\n"); + goto err_backing; + } + + /* Detect qcow2-lz4 format */ + { + u64 magic; + loff_t magic_pos = 0; + ssize_t mret; + + mret = kernel_read(dev->backing_file, &magic, 8, &magic_pos); + if (mret == 8 && be64_to_cpu(magic) == LBD_QCOW2_MAGIC) { + ret = lbd_qcow2_init(dev); + if (ret) + goto err_backing; + dev->is_qcow2 = true; + dev->size = dev->qcow2.virtual_size; + } else { + dev->is_qcow2 = false; + } + } + + /* Initialize base layer if requested */ + if (arg.base_path[0] != '\0') { + if (!dev->is_qcow2) { + pr_err("lbd: base layer requires qcow2-lz4 primary\n"); + ret = -EINVAL; + goto err_backing; + } + strscpy(dev->base_path, arg.base_path, + sizeof(dev->base_path)); + ret = lbd_qcow2_base_init(dev, arg.base_path); + if (ret) + goto err_backing; + } + + /* Resolve log directory */ + ret = kern_path(dev->log_dir, LOOKUP_DIRECTORY, &dev->log_dir_path); + if (ret) { + pr_err("lbd: cannot resolve log directory '%s': %d\n", + dev->log_dir, ret); + goto err_base; + } + + /* Begin initial log segment (file created lazily on first flush) */ + lbd_begin_segment(dev); + + /* Set capacity and activate */ + set_capacity(dev->gd, dev->size >> SECTOR_SHIFT); + + /* Set queue limits (on 6.9+ these are set via struct queue_limits above) */ +#if !LBD_HAS_QUEUE_LIMITS_API + blk_queue_logical_block_size(dev->gd->queue, LBD_BLOCK_SIZE); + blk_queue_physical_block_size(dev->gd->queue, LBD_BLOCK_SIZE); + blk_queue_max_hw_sectors(dev->gd->queue, 256); /* 128K max per request */ + blk_queue_write_cache(dev->gd->queue, true, false); + + blk_queue_max_discard_sectors(dev->gd->queue, UINT_MAX >> SECTOR_SHIFT); + dev->gd->queue->limits.discard_granularity = LBD_BLOCK_SIZE; +#endif + + dev->state = LBD_STATE_BOUND; + + ret = add_disk(dev->gd); + if (ret) + goto err_logdir; + + ret = device_add_group(disk_to_dev(dev->gd), &lbd_attr_group); + if (ret) + pr_warn("lbd%d: failed to create sysfs group: %d\n", + dev->index, ret); + + /* Start age-based rotation timer */ + if (dev->log_max_age_secs > 0) + schedule_delayed_work(&dev->log_rotate_dwork, + msecs_to_jiffies(dev->log_max_age_secs * 1000)); + + /* Return the assigned index to userspace */ + arg.index = dev->index; + if (copy_to_user(uarg, &arg, sizeof(arg))) { + /* Device is live - must tear down */ + lbd_destroy_device(dev); + mutex_lock(&lbd_devices_mutex); + idr_remove(&lbd_devices, idx); + mutex_unlock(&lbd_devices_mutex); + return -EFAULT; + } + + pr_info("lbd%d: attached to %s (%lld bytes)\n", + dev->index, dev->backing_path, dev->size); + return 0; + +err_logdir: + path_put(&dev->log_dir_path); +err_base: + if (dev->base) + lbd_qcow2_base_destroy(dev->base); +err_backing: + if (dev->is_qcow2) + lbd_qcow2_destroy(dev); + fput(dev->backing_file); +err_wq: + destroy_workqueue(dev->wq); +err_disk: +#if !LBD_HAS_BLK_MQ_ALLOC_DISK + blk_cleanup_queue(dev->gd->queue); +#endif + put_disk(dev->gd); +err_tagset: + blk_mq_free_tag_set(&dev->tag_set); +err_idr: + mutex_lock(&lbd_devices_mutex); + idr_remove(&lbd_devices, idx); + mutex_unlock(&lbd_devices_mutex); +err_free: + kvfree(dev->lz4_state); + kvfree(dev->log_buf); + kfree(dev); + return ret; +} + +static int lbd_remove_device(struct lbd_ctl_remove __user *uarg) +{ + struct lbd_ctl_remove arg; + struct lbd_device *dev; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, arg.index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + if (dev->state != LBD_STATE_BOUND) { + mutex_unlock(&lbd_devices_mutex); + return -EBUSY; + } + if (atomic_read(&dev->open_count) > 0) { + mutex_unlock(&lbd_devices_mutex); + return -EBUSY; + } + idr_remove(&lbd_devices, arg.index); + mutex_unlock(&lbd_devices_mutex); + + pr_info("lbd%d: detaching\n", dev->index); + lbd_destroy_device(dev); + return 0; +} + +static int lbd_info_device(struct lbd_ctl_info __user *uarg) +{ + struct lbd_ctl_info info; + struct lbd_device *dev; + + if (copy_from_user(&info, uarg, sizeof(info))) + return -EFAULT; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, info.index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + + info.state = dev->state; + info.size = dev->size; + strscpy(info.path, dev->backing_path, sizeof(info.path)); + mutex_unlock(&lbd_devices_mutex); + + if (copy_to_user(uarg, &info, sizeof(info))) + return -EFAULT; + return 0; +} + +/* ---------------------------------------------------------------- + * Sysfs stats + * ---------------------------------------------------------------- */ + +static ssize_t backing_path_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->backing_path); +} +static DEVICE_ATTR_RO(backing_path); + +static ssize_t base_path_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->base ? dev->base_path : "(none)"); +} +static DEVICE_ATTR_RO(base_path); + +static ssize_t log_dir_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->log_dir); +} +static DEVICE_ATTR_RO(log_dir); + +static ssize_t state_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + const char *s; + + switch (dev->state) { + case LBD_STATE_UNBOUND: s = "unbound"; break; + case LBD_STATE_BOUND: s = "bound"; break; + case LBD_STATE_REMOVING: s = "removing"; break; + default: s = "unknown"; break; + } + return sysfs_emit(buf, "%s\n", s); +} +static DEVICE_ATTR_RO(state); + +static ssize_t device_size_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", dev->size); +} +static DEVICE_ATTR_RO(device_size); + +static ssize_t reads_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_reads)); +} +static DEVICE_ATTR_RO(reads); + +static ssize_t writes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_writes)); +} +static DEVICE_ATTR_RO(writes); + +static ssize_t trims_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_trims)); +} +static DEVICE_ATTR_RO(trims); + +static ssize_t read_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_read_bytes)); +} +static DEVICE_ATTR_RO(read_bytes); + +static ssize_t write_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_write_bytes)); +} +static DEVICE_ATTR_RO(write_bytes); + +static ssize_t trim_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_trim_bytes)); +} +static DEVICE_ATTR_RO(trim_bytes); + +static ssize_t alloc_reused_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_reused)); +} +static DEVICE_ATTR_RO(alloc_reused); + +static ssize_t alloc_new_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_new)); +} +static DEVICE_ATTR_RO(alloc_new); + +static ssize_t alloc_freed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_freed)); +} +static DEVICE_ATTR_RO(alloc_freed); + +static ssize_t compressed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_compressed)); +} +static DEVICE_ATTR_RO(compressed); + +static ssize_t uncompressed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_uncompressed)); +} +static DEVICE_ATTR_RO(uncompressed); + +static ssize_t log_seq_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%llu\n", READ_ONCE(dev->log_seq)); +} +static DEVICE_ATTR_RO(log_seq); + +static ssize_t log_segment_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->log_segment_label); +} +static DEVICE_ATTR_RO(log_segment); + +static ssize_t log_rotations_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_log_rotations)); +} +static DEVICE_ATTR_RO(log_rotations); + +static ssize_t log_buf_used_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%zu\n", READ_ONCE(dev->log_buf_used)); +} +static DEVICE_ATTR_RO(log_buf_used); + +static ssize_t segment_age_secs_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + s64 age = ktime_to_ms(ktime_sub(ktime_get(), dev->segment_start_time)); + + return sysfs_emit(buf, "%lld\n", age / 1000); +} +static DEVICE_ATTR_RO(segment_age_secs); + +static struct attribute *lbd_attrs[] = { + &dev_attr_backing_path.attr, + &dev_attr_base_path.attr, + &dev_attr_log_dir.attr, + &dev_attr_state.attr, + &dev_attr_device_size.attr, + &dev_attr_reads.attr, + &dev_attr_writes.attr, + &dev_attr_trims.attr, + &dev_attr_read_bytes.attr, + &dev_attr_write_bytes.attr, + &dev_attr_trim_bytes.attr, + &dev_attr_alloc_reused.attr, + &dev_attr_alloc_new.attr, + &dev_attr_alloc_freed.attr, + &dev_attr_compressed.attr, + &dev_attr_uncompressed.attr, + &dev_attr_log_seq.attr, + &dev_attr_log_segment.attr, + &dev_attr_log_rotations.attr, + &dev_attr_log_buf_used.attr, + &dev_attr_segment_age_secs.attr, + NULL, +}; + +static const struct attribute_group lbd_attr_group = { + .name = "lbd", + .attrs = lbd_attrs, +}; + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Miss handler helpers + * ---------------------------------------------------------------- */ + +static struct lbd_miss_handler *lbd_miss_handler_alloc(int dev_index) +{ + struct lbd_miss_handler *mh; + + mh = kzalloc(sizeof(*mh), GFP_KERNEL); + if (!mh) + return NULL; + + spin_lock_init(&mh->lock); + init_waitqueue_head(&mh->wq); + mh->dev_index = dev_index; + mh->pending = NULL; + mh->has_event = false; + return mh; +} + +/* + * Called from I/O workqueue when a cluster miss is detected. + * Posts the miss event to the handler and blocks until userspace responds. + * rwsem must be RELEASED before calling this. + */ +enum lbd_miss_action lbd_qcow2_handle_miss(struct lbd_device *dev, + u64 cluster_index) +{ + struct lbd_miss_handler *mh; + struct lbd_miss_pending pending; + + init_completion(&pending.done); + pending.action = LBD_MISS_CONTINUE; + + /* + * Hold miss_handler_lock while setting mh->pending to prevent + * lbd_ctl_release() from freeing mh between our read of the + * pointer and the registration of the pending completion. + * Once pending is registered, release will complete() it + * before kfree(mh), so mh stays alive until we wake up. + */ + spin_lock(&dev->miss_handler_lock); + mh = dev->miss_handler; + if (!mh) { + spin_unlock(&dev->miss_handler_lock); + return LBD_MISS_CONTINUE; + } + + spin_lock(&mh->lock); + mh->pending = &pending; + mh->miss_cluster = cluster_index; + mh->has_event = true; + spin_unlock(&mh->lock); + spin_unlock(&dev->miss_handler_lock); + + wake_up_interruptible(&mh->wq); + + /* Block I/O thread until userspace responds */ + wait_for_completion(&pending.done); + + return pending.action; +} +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* ---------------------------------------------------------------- + * Watcher helpers + * ---------------------------------------------------------------- */ + +static struct lbd_watcher *lbd_watcher_alloc(int filter_dev) +{ + struct lbd_watcher *w; + + w = kzalloc(sizeof(*w), GFP_KERNEL); + if (!w) + return NULL; + + spin_lock_init(&w->lock); + init_waitqueue_head(&w->wq); + w->filter_dev = filter_dev; + w->head = 0; + w->tail = 0; + w->count = 0; + return w; +} + +static void lbd_watcher_enqueue(struct lbd_watcher *w, + const struct lbd_watcher_event *ev) +{ + spin_lock(&w->lock); + if (w->count == LBD_WATCHER_QUEUE_SIZE) { + /* Overflow: drop oldest */ + w->head = (w->head + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count--; + } + w->queue[w->tail] = *ev; + w->tail = (w->tail + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count++; + spin_unlock(&w->lock); + wake_up_interruptible(&w->wq); +} + +/* Copy front event without removing (for encode-then-dequeue pattern) */ +static bool lbd_watcher_peek(struct lbd_watcher *w, + struct lbd_watcher_event *ev) +{ + bool got; + + spin_lock(&w->lock); + got = w->count > 0; + if (got) + *ev = w->queue[w->head]; + spin_unlock(&w->lock); + return got; +} + +/* Remove front event after successful copy_to_user */ +static void lbd_watcher_pop(struct lbd_watcher *w) +{ + spin_lock(&w->lock); + if (w->count > 0) { + w->head = (w->head + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count--; + } + spin_unlock(&w->lock); +} + +/* + * Notify all watchers of a log rotation event. + * Called with dev->log_mutex held (sleepable). + * Acquires lbd_watchers_lock (spin), then w->lock (spin). + */ +static void lbd_notify_watchers(struct lbd_device *dev) +{ + struct lbd_watcher_event ev; + struct lbd_watcher *w; + ev.dev_index = dev->index; + ev.log_seq = dev->log_seq; + ev.device_size = dev->size; + memcpy(ev.segment_label, dev->log_segment_label, + sizeof(ev.segment_label)); + strscpy(ev.log_dir, dev->log_dir, sizeof(ev.log_dir)); + + spin_lock(&lbd_watchers_lock); + list_for_each_entry(w, &lbd_watchers, list) { + if (w->filter_dev >= 0 && w->filter_dev != dev->index) + continue; + lbd_watcher_enqueue(w, &ev); + } + spin_unlock(&lbd_watchers_lock); +} + +/* ---------------------------------------------------------------- + * Control device: watcher fops + * ---------------------------------------------------------------- */ + +static int lbd_ctl_open(struct inode *inode, struct file *file) +{ + struct lbd_ctl_state *state; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + file->private_data = state; + return 0; +} + +static int lbd_ctl_release(struct inode *inode, struct file *file) +{ + struct lbd_ctl_state *state = file->private_data; + + if (!state) + return 0; + + if (state->watcher) { + spin_lock(&lbd_watchers_lock); + list_del(&state->watcher->list); + spin_unlock(&lbd_watchers_lock); + kfree(state->watcher); + } + +#ifdef CONFIG_LBD_MISS_HANDLER + if (state->miss) { + struct lbd_miss_handler *mh = state->miss; + struct lbd_device *dev; + + /* Find the device and clear its handler pointer */ + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, mh->dev_index); + if (dev) { + spin_lock(&dev->miss_handler_lock); + if (dev->miss_handler == mh) + dev->miss_handler = NULL; + spin_unlock(&dev->miss_handler_lock); + } + mutex_unlock(&lbd_devices_mutex); + + /* Complete any pending miss with CONTINUE */ + spin_lock(&mh->lock); + if (mh->pending) { + mh->pending->action = LBD_MISS_CONTINUE; + complete(&mh->pending->done); + mh->pending = NULL; + } + spin_unlock(&mh->lock); + + kfree(mh); + } +#endif + + kfree(state); + file->private_data = NULL; + return 0; +} + +static ssize_t lbd_ctl_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct lbd_ctl_state *state = file->private_data; + u8 kbuf[512]; + struct cbor_dec d; + u64 map_count, key; + char cmd[16]; + int filter_dev = -1; + bool have_cmd = false; + char path[LBD_LOG_PATH_MAX]; + bool have_path = false; + u64 i; + + if (!state) + return -EINVAL; + + if (count > sizeof(kbuf)) + return -EINVAL; + + if (copy_from_user(kbuf, ubuf, count)) + return -EFAULT; + + cbor_dec_init(&d, kbuf, count); + + if (cbor_dec_map(&d, &map_count)) + return -EINVAL; + + path[0] = '\0'; + + for (i = 0; i < map_count; i++) { + if (cbor_dec_uint(&d, &key)) + return -EINVAL; + + switch (key) { + case LBD_WATCH_KEY_CMD: + if (cbor_dec_text(&d, cmd, sizeof(cmd), NULL)) + return -EINVAL; + have_cmd = true; + break; + case LBD_WATCH_KEY_DEV: { + u64 dev_val; + if (cbor_dec_uint(&d, &dev_val)) + return -EINVAL; + filter_dev = (int)dev_val; + break; + } + case LBD_WATCH_KEY_PATH: + if (cbor_dec_text(&d, path, sizeof(path), NULL)) + return -EINVAL; + have_path = true; + break; + default: + return -EINVAL; + } + } + + if (!have_cmd) + return -EINVAL; + + if (strcmp(cmd, "watch") == 0) { + struct lbd_watcher *w; + + if (state->watcher) + return -EBUSY; + + w = lbd_watcher_alloc(filter_dev); + if (!w) + return -ENOMEM; + + spin_lock(&lbd_watchers_lock); + list_add_tail(&w->list, &lbd_watchers); + spin_unlock(&lbd_watchers_lock); + + state->watcher = w; + return count; + } + +#ifdef CONFIG_LBD_MISS_HANDLER + if (strcmp(cmd, "manage_misses") == 0) { + struct lbd_device *dev; + struct lbd_miss_handler *mh; + + if (state->miss) + return -EBUSY; + if (filter_dev < 0) + return -EINVAL; + + mh = lbd_miss_handler_alloc(filter_dev); + if (!mh) + return -ENOMEM; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, filter_dev); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -ENODEV; + } + if (!dev->is_qcow2) { + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -EINVAL; + } + + spin_lock(&dev->miss_handler_lock); + if (dev->miss_handler) { + spin_unlock(&dev->miss_handler_lock); + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -EBUSY; + } + dev->miss_handler = mh; + spin_unlock(&dev->miss_handler_lock); + mutex_unlock(&lbd_devices_mutex); + + state->miss = mh; + return count; + } + + if (strcmp(cmd, "continue") == 0) { + struct lbd_miss_handler *mh = state->miss; + + if (!mh) + return -EINVAL; + + spin_lock(&mh->lock); + if (!mh->pending) { + spin_unlock(&mh->lock); + return -EINVAL; + } + mh->pending->action = LBD_MISS_CONTINUE; + complete(&mh->pending->done); + mh->pending = NULL; + mh->has_event = false; + spin_unlock(&mh->lock); + + return count; + } + + if (strcmp(cmd, "retry") == 0) { + struct lbd_miss_handler *mh = state->miss; + + if (!mh) + return -EINVAL; + + spin_lock(&mh->lock); + if (!mh->pending) { + spin_unlock(&mh->lock); + return -EINVAL; + } + mh->pending->action = LBD_MISS_RETRY; + complete(&mh->pending->done); + mh->pending = NULL; + mh->has_event = false; + spin_unlock(&mh->lock); + + return count; + } + + if (strcmp(cmd, "swap") == 0) { + struct lbd_device *dev; + struct lbd_miss_handler *mh = state->miss; + int ret; + + if (!mh) + return -EINVAL; + if (!have_path || path[0] == '\0') + return -EINVAL; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, mh->dev_index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + mutex_unlock(&lbd_devices_mutex); + + ret = lbd_qcow2_swap_base(dev, path); + if (ret) + return ret; + + return count; + } +#endif /* CONFIG_LBD_MISS_HANDLER */ + + return -EINVAL; +} + +static bool lbd_ctl_has_event(struct lbd_ctl_state *state) +{ + struct lbd_watcher *w = state->watcher; + struct lbd_watcher_event ev; + +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh = state->miss; + + if (mh) { + bool has; + + spin_lock(&mh->lock); + has = mh->has_event; + spin_unlock(&mh->lock); + if (has) + return true; + } +#endif + if (w && lbd_watcher_peek(w, &ev)) + return true; + return false; +} + +static ssize_t lbd_ctl_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct lbd_ctl_state *state = file->private_data; + struct lbd_watcher *w; + u8 tmp[512]; + struct cbor_enc e; + size_t encoded_len; +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh; +#endif + + if (!state) + return -EINVAL; + + w = state->watcher; + +#ifdef CONFIG_LBD_MISS_HANDLER + mh = state->miss; + + if (!mh && !w) + return -EINVAL; +#else + if (!w) + return -EINVAL; +#endif + + /* Block until an event is available */ + if (!lbd_ctl_has_event(state)) { + int ret; + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + +#ifdef CONFIG_LBD_MISS_HANDLER + if (mh) { + ret = wait_event_interruptible(mh->wq, + lbd_ctl_has_event(state)); + } else { + ret = wait_event_interruptible(w->wq, + lbd_ctl_has_event(state)); + } +#else + ret = wait_event_interruptible(w->wq, + lbd_ctl_has_event(state)); +#endif + if (ret) + return -ERESTARTSYS; + } + +#ifdef CONFIG_LBD_MISS_HANDLER + /* Miss events take priority (time-critical, I/O thread blocked) */ + if (mh) { + bool has; + + spin_lock(&mh->lock); + has = mh->has_event; + spin_unlock(&mh->lock); + + if (has) { + u64 cluster; + + spin_lock(&mh->lock); + cluster = mh->miss_cluster; + spin_unlock(&mh->lock); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + cbor_enc_map(&e, 3); + + cbor_enc_uint(&e, LBD_MISS_KEY_TYPE); + cbor_enc_text(&e, "block_miss", 10); + + cbor_enc_uint(&e, LBD_MISS_KEY_DEV); + cbor_enc_uint(&e, (u64)mh->dev_index); + + cbor_enc_uint(&e, LBD_MISS_KEY_CLUSTER); + cbor_enc_uint(&e, cluster); + + if (e.err) + return -EINVAL; + + encoded_len = cbor_enc_len(&e); + if (count < encoded_len) + return -EINVAL; + + if (copy_to_user(ubuf, tmp, encoded_len)) + return -EFAULT; + + return encoded_len; + } + } +#endif /* CONFIG_LBD_MISS_HANDLER */ + + /* Fall through to watcher events */ + if (w) { + struct lbd_watcher_event ev; + size_t label_len, dir_len; + + if (!lbd_watcher_peek(w, &ev)) { + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(w->wq, + lbd_watcher_peek(w, &ev))) + return -ERESTARTSYS; + } + + cbor_enc_init(&e, tmp, sizeof(tmp)); + cbor_enc_map(&e, 6); + + cbor_enc_uint(&e, LBD_EVENT_KEY_TYPE); + cbor_enc_text(&e, "log_rotated", 11); + + cbor_enc_uint(&e, LBD_EVENT_KEY_DEV); + cbor_enc_uint(&e, (u64)ev.dev_index); + + cbor_enc_uint(&e, LBD_EVENT_KEY_LABEL); + label_len = strlen(ev.segment_label); + cbor_enc_text(&e, ev.segment_label, label_len); + + cbor_enc_uint(&e, LBD_EVENT_KEY_DIR); + dir_len = strlen(ev.log_dir); + cbor_enc_text(&e, ev.log_dir, dir_len); + + cbor_enc_uint(&e, LBD_EVENT_KEY_SEQ); + cbor_enc_uint(&e, ev.log_seq); + + cbor_enc_uint(&e, LBD_EVENT_KEY_SIZE); + cbor_enc_uint(&e, ev.device_size); + + if (e.err) + return -EINVAL; + + encoded_len = cbor_enc_len(&e); + if (count < encoded_len) + return -EINVAL; + + if (copy_to_user(ubuf, tmp, encoded_len)) + return -EFAULT; + + lbd_watcher_pop(w); + return encoded_len; + } + + return -EAGAIN; +} + +static __poll_t lbd_ctl_poll(struct file *file, + struct poll_table_struct *wait) +{ + struct lbd_ctl_state *state = file->private_data; + struct lbd_watcher *w; + __poll_t mask = 0; +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh; +#endif + + if (!state) + return 0; + + w = state->watcher; + +#ifdef CONFIG_LBD_MISS_HANDLER + mh = state->miss; + + if (!mh && !w) + return 0; + + if (mh) + poll_wait(file, &mh->wq, wait); +#else + if (!w) + return 0; +#endif + if (w) + poll_wait(file, &w->wq, wait); + +#ifdef CONFIG_LBD_MISS_HANDLER + if (mh) { + spin_lock(&mh->lock); + if (mh->has_event) + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock(&mh->lock); + } +#endif + + if (w) { + spin_lock(&w->lock); + if (w->count > 0) + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock(&w->lock); + } + + return mask; +} + +/* ---------------------------------------------------------------- + * Control device (misc device) + * ---------------------------------------------------------------- */ + +static long lbd_ctl_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case LBD_CTL_ADD: + return lbd_add_device((struct lbd_ctl_add __user *)arg); + case LBD_CTL_REMOVE: + return lbd_remove_device((struct lbd_ctl_remove __user *)arg); + case LBD_CTL_INFO: + return lbd_info_device((struct lbd_ctl_info __user *)arg); + default: + return -ENOTTY; + } +} + +static const struct file_operations lbd_ctl_fops = { + .owner = THIS_MODULE, + .open = lbd_ctl_open, + .release = lbd_ctl_release, + .read = lbd_ctl_read, + .write = lbd_ctl_write, + .poll = lbd_ctl_poll, + .unlocked_ioctl = lbd_ctl_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice lbd_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = LBD_CTL_NAME, + .fops = &lbd_ctl_fops, +}; + +/* ---------------------------------------------------------------- + * Module init / exit + * ---------------------------------------------------------------- */ + +static int __init lbd_init(void) +{ + int ret; + + lbd_major = register_blkdev(0, LBD_NAME); + if (lbd_major < 0) { + pr_err("lbd: failed to register block device\n"); + return lbd_major; + } + + idr_init(&lbd_devices); + + ret = misc_register(&lbd_misc); + if (ret) { + pr_err("lbd: failed to register control device\n"); + idr_destroy(&lbd_devices); + unregister_blkdev(lbd_major, LBD_NAME); + return ret; + } + + pr_info("lbd: module loaded (major=%d)\n", lbd_major); + return 0; +} + +static void __exit lbd_exit(void) +{ + struct lbd_device *dev; + int id; + + misc_deregister(&lbd_misc); + + mutex_lock(&lbd_devices_mutex); + idr_for_each_entry(&lbd_devices, dev, id) { + idr_remove(&lbd_devices, id); + mutex_unlock(&lbd_devices_mutex); + lbd_destroy_device(dev); + mutex_lock(&lbd_devices_mutex); + } + mutex_unlock(&lbd_devices_mutex); + + idr_destroy(&lbd_devices); + unregister_blkdev(lbd_major, LBD_NAME); + + pr_info("lbd: module unloaded\n"); +} + +module_init(lbd_init); +module_exit(lbd_exit); diff --git a/third_party/lbd/src/lbd_qcow2.c b/third_party/lbd/src/lbd_qcow2.c new file mode 100644 index 000000000..d3e81388a --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2.c @@ -0,0 +1,1921 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LBD qcow2-lz4 backing store + * + * Implements a qcow2-inspired format with LZ4-compressed clusters + * as the backing store for LBD block devices. Data is organized in + * 64 KiB clusters with two-level (L1/L2) address translation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd.h" +#include "lbd_qcow2.h" +#include "lz4_kcompat.h" + +/* Forward declaration for base layer read (used by lbd_qcow2_cl_load) */ +static int lbd_qcow2_base_read_cluster(struct lbd_device *dev, + u64 cluster_index, void *buf); + +/* ---------------------------------------------------------------- + * Helpers + * ---------------------------------------------------------------- */ + +static inline u64 lbd_qcow2_lru_tick(struct lbd_qcow2 *q) +{ + return q->lru_tick++; +} + +/* Write header's alloc_offset field to disk */ +static int lbd_qcow2_write_alloc_offset(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[8]; + loff_t pos = LBD_QCOW2_OFF_ALLOC_OFFSET; + ssize_t ret; + + _qcow2_put64(buf, 0, q->alloc_offset); + ret = kernel_write(dev->backing_file, buf, 8, &pos); + if (ret != 8) + return ret < 0 ? ret : -EIO; + return 0; +} + +/* Write header's free_list_head field to disk */ +static int lbd_qcow2_write_free_list_head(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[8]; + loff_t pos = LBD_QCOW2_OFF_FREE_LIST; + ssize_t ret; + + _qcow2_put64(buf, 0, q->free_list_head); + ret = kernel_write(dev->backing_file, buf, 8, &pos); + if (ret != 8) + return ret < 0 ? ret : -EIO; + return 0; +} + +/* + * Compute the on-disk allocation size for an existing L2 entry. + * Returns 0 on error. + */ +static u64 lbd_qcow2_read_old_alloc_size(struct lbd_device *dev, u64 l2_entry) +{ + struct lbd_qcow2 *q = &dev->qcow2; + + if (l2_entry == 0) + return 0; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + u64 phys = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + __be32 comp_size_be; + u32 comp_size; + loff_t pos = phys; + ssize_t ret; + + ret = kernel_read(dev->backing_file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) + return 0; + + comp_size = be32_to_cpu(comp_size_be); + return ALIGN(sizeof(__be32) + comp_size, 4096); + } + + return q->cluster_size; +} + +/* + * Write a tombstone at a physical offset, prepending to the free list. + * extent_size is the total usable space at that location. + */ +static int lbd_qcow2_free_extent(struct lbd_device *dev, loff_t phys_offset, + u64 extent_size) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[16]; + loff_t pos = phys_offset; + ssize_t ret; + + /* Tombstone marker */ + _qcow2_put32(buf, 0, LBD_QCOW2_FREE_TOMBSTONE); + /* Extent size */ + _qcow2_put32(buf, 4, (u32)extent_size); + /* Next free pointer */ + _qcow2_put64(buf, 8, q->free_list_head); + + ret = kernel_write(dev->backing_file, buf, 16, &pos); + if (ret != 16) + return ret < 0 ? ret : -EIO; + + q->free_list_head = phys_offset; + atomic64_inc(&dev->stat_alloc_freed); + return lbd_qcow2_write_free_list_head(dev); +} + +/* Maximum number of free list entries to scan when allocating */ +#define LBD_QCOW2_FREE_SCAN_LIMIT 8 + +/* Minimum free entry size (tombstone header) */ +#define LBD_QCOW2_FREE_ENTRY_MIN 16 + +/* + * Try to allocate space from the free list or append. + * needed: bytes required for the new on-disk data. + * Returns the physical offset to write at via *out_phys. + */ +static int lbd_qcow2_alloc_space(struct lbd_device *dev, u64 needed, + loff_t *out_phys) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t prev_phys = 0; + loff_t cur = q->free_list_head; + int scanned = 0; + int is_first = 1; + + while (cur != 0 && scanned < LBD_QCOW2_FREE_SCAN_LIMIT) { + u8 buf[16]; + loff_t pos = cur; + ssize_t ret; + u32 tombstone, extent_size; + u64 next_free; + + ret = kernel_read(dev->backing_file, buf, 16, &pos); + if (ret != 16) + break; + + tombstone = _qcow2_get32(buf, 0); + if (tombstone != LBD_QCOW2_FREE_TOMBSTONE) + break; + + extent_size = _qcow2_get32(buf, 4); + next_free = _qcow2_get64(buf, 8); + + if (extent_size >= needed) { + u64 remainder = extent_size - needed; + + /* Unlink this entry from the free list */ + if (is_first) { + q->free_list_head = next_free; + } else { + /* Update previous entry's next pointer */ + u8 nbuf[8]; + loff_t npos = prev_phys + 8; + + _qcow2_put64(nbuf, 0, next_free); + ret = kernel_write(dev->backing_file, nbuf, 8, + &npos); + if (ret != 8) + goto append; + } + + /* Split if remainder is large enough */ + if (remainder >= LBD_QCOW2_FREE_ENTRY_MIN) { + loff_t split_phys = cur + needed; + int err; + + /* + * Add the remainder back as a new free entry + * at the head of the free list. + */ + err = lbd_qcow2_free_extent(dev, split_phys, + remainder); + if (err) { + /* + * Non-fatal: we waste the remainder + * but the allocation itself is fine. + */ + lbd_qcow2_write_free_list_head(dev); + } + } else { + lbd_qcow2_write_free_list_head(dev); + } + + *out_phys = cur; + atomic64_inc(&dev->stat_alloc_reused); + return 0; + } + + prev_phys = cur; + cur = next_free; + is_first = 0; + scanned++; + } + +append: + *out_phys = q->alloc_offset; + q->alloc_offset += needed; + atomic64_inc(&dev->stat_alloc_new); + return 0; +} + +/* ---------------------------------------------------------------- + * L2 cache + * ---------------------------------------------------------------- */ + +/* Find or load an L2 table into cache, return pointer to cache entry */ +static struct lbd_l2_cache_entry * +lbd_qcow2_l2_get(struct lbd_device *dev, u32 l1_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_l2_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + /* Check cache for hit */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (e->valid && e->l1_index == l1_index) { + e->lru = lbd_qcow2_lru_tick(q); + return e; + } + } + + /* Cache miss: find LRU entry to evict */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* Flush dirty entry before evicting */ + if (best->valid && best->dirty) { + u64 l2_phys = q->l1_table[best->l1_index]; + if (l2_phys) { + loff_t pos = l2_phys; + __be64 *disk_l2; + int j; + ssize_t ret; + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (disk_l2) { + u8 *raw = (u8 *)disk_l2; + u32 crc; + __be32 crc_be; + + memset(disk_l2, 0, q->cluster_size); + for (j = 0; j < q->l2_entries; j++) + disk_l2[j] = cpu_to_be64(best->table[j]); + + /* CRC32C trailer */ + crc = ~crc32c(~0, raw, q->cluster_size - 4); + crc_be = cpu_to_be32(crc); + memcpy(raw + q->cluster_size - 4, &crc_be, 4); + + ret = kernel_write(dev->backing_file, disk_l2, + q->cluster_size, &pos); + if (ret != q->cluster_size) + pr_warn("lbd%d: L2 flush failed\n", + dev->index); + kvfree(disk_l2); + } + } + best->dirty = false; + } + + /* Load L2 table from disk */ + best->l1_index = l1_index; + best->dirty = false; + best->lru = lbd_qcow2_lru_tick(q); + + if (l1_index < q->l1_size && q->l1_table[l1_index] != 0) { + loff_t pos = q->l1_table[l1_index]; + __be64 *disk_l2; + ssize_t ret; + int j; + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (!disk_l2) { + best->valid = false; + return NULL; + } + + ret = kernel_read(dev->backing_file, disk_l2, + q->cluster_size, &pos); + if (ret != q->cluster_size) { + pr_warn("lbd%d: L2 read failed for l1[%u]\n", + dev->index, l1_index); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + + /* Verify CRC32C trailer */ + { + u8 *raw = (u8 *)disk_l2; + u32 stored_crc = _qcow2_get32(raw, q->cluster_size - 4); + u32 calc_crc = ~crc32c(~0, raw, q->cluster_size - 4); + + if (stored_crc != calc_crc) { + pr_warn("lbd%d: L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + dev->index, l1_index, + stored_crc, calc_crc); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + } + + for (j = 0; j < q->l2_entries; j++) + best->table[j] = be64_to_cpu(disk_l2[j]); + + kvfree(disk_l2); + } else { + /* Unallocated L2: all zeros */ + memset(best->table, 0, q->cluster_size); + } + + best->valid = true; + return best; +} + +/* Flush a specific dirty L2 entry to disk */ +static int lbd_qcow2_l2_flush(struct lbd_device *dev, + struct lbd_l2_cache_entry *e) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u64 l2_phys; + __be64 *disk_l2; + loff_t pos; + ssize_t ret; + int j; + + if (!e->valid || !e->dirty) + return 0; + + l2_phys = q->l1_table[e->l1_index]; + if (!l2_phys) + return -EIO; /* should not happen */ + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (!disk_l2) + return -ENOMEM; + + memset(disk_l2, 0, q->cluster_size); + for (j = 0; j < q->l2_entries; j++) + disk_l2[j] = cpu_to_be64(e->table[j]); + + /* Compute and store CRC32C trailer at end of cluster */ + { + u8 *raw = (u8 *)disk_l2; + u32 crc = ~crc32c(~0, raw, q->cluster_size - 4); + __be32 crc_be = cpu_to_be32(crc); + + memcpy(raw + q->cluster_size - 4, &crc_be, 4); + } + + pos = l2_phys; + ret = kernel_write(dev->backing_file, disk_l2, q->cluster_size, &pos); + kvfree(disk_l2); + + if (ret != q->cluster_size) + return ret < 0 ? ret : -EIO; + + e->dirty = false; + return 0; +} + +/* Allocate a new L2 table on disk if needed */ +static int lbd_qcow2_l2_alloc(struct lbd_device *dev, u32 l1_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t pos; + __be64 val; + ssize_t ret; + + if (l1_index >= q->l1_size) + return -ENOSPC; + + if (q->l1_table[l1_index] != 0) + return 0; /* already allocated */ + + /* Allocate cluster for L2 table at append point */ + q->l1_table[l1_index] = q->alloc_offset; + q->alloc_offset += q->cluster_size; + + /* Write zeroed L2 table to disk (with CRC32C trailer) */ + { + u8 *zeros = kvmalloc(q->cluster_size, GFP_NOIO); + u32 crc; + __be32 crc_be; + + if (!zeros) + return -ENOMEM; + memset(zeros, 0, q->cluster_size); + + /* CRC32C of all-zero data (covers bytes [0, cluster_size-4)) */ + crc = ~crc32c(~0, zeros, q->cluster_size - 4); + crc_be = cpu_to_be32(crc); + memcpy(zeros + q->cluster_size - 4, &crc_be, 4); + + pos = q->l1_table[l1_index]; + ret = kernel_write(dev->backing_file, zeros, + q->cluster_size, &pos); + kvfree(zeros); + if (ret != q->cluster_size) + return ret < 0 ? ret : -EIO; + } + + /* Write L1 entry to disk */ + val = cpu_to_be64(q->l1_table[l1_index]); + pos = q->l1_offset + (loff_t)l1_index * sizeof(__be64); + ret = kernel_write(dev->backing_file, &val, sizeof(val), &pos); + if (ret != sizeof(val)) + return ret < 0 ? ret : -EIO; + + return 0; +} + +/* ---------------------------------------------------------------- + * Cluster cache + * ---------------------------------------------------------------- */ + +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_find(struct lbd_qcow2 *q, u64 cluster_index) +{ + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) { + e->lru = lbd_qcow2_lru_tick(q); + return e; + } + } + return NULL; +} + +static void lbd_qcow2_cl_invalidate(struct lbd_qcow2 *q, u64 cluster_index) +{ + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) { + e->valid = false; + return; + } + } +} + +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_alloc_entry(struct lbd_qcow2 *q) +{ + struct lbd_cl_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* Evict LRU (cluster cache is read-only cache, no flush needed) */ + best->valid = false; + return best; +} + +/* + * Return a zeroed cache entry for write path when cluster is unallocated. + * Caller holds q->rwsem for write. + */ +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_get_zero(struct lbd_device *dev, u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_cl_cache_entry *ce; + + ce = lbd_qcow2_cl_find(q, cluster_index); + if (ce) + return ce; + + ce = lbd_qcow2_cl_alloc_entry(q); + ce->cluster_index = cluster_index; + ce->lru = lbd_qcow2_lru_tick(q); + ce->dirty = false; + memset(ce->data, 0, q->cluster_size); + ce->valid = true; + return ce; +} + +/* Load a cluster into cache from disk via L2 lookup */ +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_load(struct lbd_device *dev, u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_cl_cache_entry *ce; + struct lbd_l2_cache_entry *l2e; + u32 l1_idx, l2_idx; + u64 l2_entry, phys_offset; + ssize_t ret; + + /* Check cache first */ + ce = lbd_qcow2_cl_find(q, cluster_index); + if (ce) + return ce; + + /* L2 lookup */ + l1_idx = cluster_index / q->l2_entries; + l2_idx = cluster_index % q->l2_entries; + + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) + return NULL; + + l2_entry = l2e->table[l2_idx]; + + /* Get a cache entry */ + ce = lbd_qcow2_cl_alloc_entry(q); + ce->cluster_index = cluster_index; + ce->lru = lbd_qcow2_lru_tick(q); + ce->dirty = false; + + if (l2_entry == 0) { + if (dev->base) { + int err = lbd_qcow2_base_read_cluster(dev, + cluster_index, ce->data); + if (err == 1) { + /* Unallocated in both primary and base */ +#ifdef CONFIG_LBD_MISS_HANDLER + if (dev->miss_handler) + return ERR_PTR(-ENODATA); +#endif + /* No handler: zero-fill (backward compatible) */ + memset(ce->data, 0, q->cluster_size); + } else if (err < 0) { + return NULL; /* I/O error */ + } + /* err == 0: data was read successfully */ +#ifdef CONFIG_LBD_MISS_HANDLER + } else if (dev->miss_handler) { + /* No base, but miss handler registered */ + return ERR_PTR(-ENODATA); +#endif + } else { + memset(ce->data, 0, q->cluster_size); + } + ce->valid = true; + return ce; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + /* Compressed cluster: read size header + compressed data */ + __be32 comp_size_be; + u32 comp_size; + int dec_len; + loff_t pos = phys_offset; + + ret = kernel_read(dev->backing_file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + pr_warn("lbd%d: failed to read compressed size\n", + dev->index); + return NULL; + } + + comp_size = be32_to_cpu(comp_size_be); + if (comp_size > LZ4_compressBound(q->cluster_size)) { + pr_warn("lbd%d: invalid compressed size %u\n", + dev->index, comp_size); + return NULL; + } + + ret = kernel_read(dev->backing_file, q->read_buf, + comp_size, &pos); + if (ret != comp_size) { + pr_warn("lbd%d: failed to read compressed data\n", + dev->index); + return NULL; + } + + dec_len = LZ4_decompress_safe(q->read_buf, ce->data, + comp_size, q->cluster_size); + if (dec_len != q->cluster_size) { + pr_warn("lbd%d: LZ4 decompress failed (%d)\n", + dev->index, dec_len); + return NULL; + } + } else { + /* Uncompressed cluster */ + loff_t pos = phys_offset; + + ret = kernel_read(dev->backing_file, ce->data, + q->cluster_size, &pos); + if (ret != q->cluster_size) { + pr_warn("lbd%d: failed to read cluster\n", + dev->index); + return NULL; + } + } + + ce->valid = true; + return ce; +} + +/* + * Flush all dirty L2 cache entries to disk. + */ +static void lbd_qcow2_flush_all_l2(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int i; + + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + if (e->valid && e->dirty) + lbd_qcow2_l2_flush(dev, e); + } +} + +/* ---------------------------------------------------------------- + * Init / Destroy + * ---------------------------------------------------------------- */ + +int lbd_qcow2_init(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 *hdr; + loff_t pos = 0; + ssize_t ret; + __be64 *disk_l1; + int i; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_KERNEL); + if (!hdr) + return -ENOMEM; + + /* Read header */ + ret = kernel_read(dev->backing_file, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + pr_err("lbd%d: qcow2 header read failed\n", dev->index); + kvfree(hdr); + return ret < 0 ? ret : -EIO; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + pr_err("lbd%d: invalid qcow2 magic\n", dev->index); + kvfree(hdr); + return -EINVAL; + } + + if (lbd_qcow2_hdr_version(hdr) != LBD_QCOW2_VERSION) { + pr_err("lbd%d: unsupported qcow2 version %u\n", + dev->index, lbd_qcow2_hdr_version(hdr)); + kvfree(hdr); + return -EINVAL; + } + + q->cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + if (q->cluster_bits < 12 || q->cluster_bits > 24) { + pr_err("lbd%d: invalid cluster_bits %u\n", + dev->index, q->cluster_bits); + kvfree(hdr); + return -EINVAL; + } + + q->cluster_size = 1U << q->cluster_bits; + q->l2_entries = (q->cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / sizeof(u64); + q->virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + q->l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + q->l1_size = lbd_qcow2_hdr_l1_size(hdr); + q->alloc_offset = lbd_qcow2_hdr_alloc_offset(hdr); + q->free_list_head = lbd_qcow2_hdr_free_list(hdr); + q->lru_tick = 0; + + kvfree(hdr); + + init_rwsem(&q->rwsem); + + /* Allocate and read L1 table */ + q->l1_table = kvmalloc_array(q->l1_size, sizeof(u64), GFP_KERNEL); + if (!q->l1_table) + return -ENOMEM; + + disk_l1 = kvmalloc_array(q->l1_size, sizeof(__be64), GFP_KERNEL); + if (!disk_l1) { + kvfree(q->l1_table); + q->l1_table = NULL; + return -ENOMEM; + } + + pos = q->l1_offset; + ret = kernel_read(dev->backing_file, disk_l1, + q->l1_size * sizeof(__be64), &pos); + if (ret != q->l1_size * sizeof(__be64)) { + pr_err("lbd%d: L1 table read failed\n", dev->index); + kvfree(disk_l1); + kvfree(q->l1_table); + q->l1_table = NULL; + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < q->l1_size; i++) + q->l1_table[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + /* Allocate L2 cache entries */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + e->table = kvmalloc(q->cluster_size, GFP_KERNEL); + if (!e->table) + goto err_l2_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate cluster cache entries */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + e->data = kvmalloc(q->cluster_size, GFP_KERNEL); + if (!e->data) + goto err_cl_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate compression buffer */ + q->comp_buf = kvmalloc(LZ4_compressBound(q->cluster_size), GFP_KERNEL); + if (!q->comp_buf) + goto err_cl_cache; + + /* Allocate read buffer for compressed data */ + q->read_buf = kvmalloc(LZ4_compressBound(q->cluster_size), GFP_KERNEL); + if (!q->read_buf) { + kvfree(q->comp_buf); + q->comp_buf = NULL; + goto err_cl_cache; + } + + pr_info("lbd%d: qcow2-lz4 format detected, virtual_size=%llu, " + "cluster_size=%u, l1_size=%u\n", + dev->index, q->virtual_size, q->cluster_size, q->l1_size); + + return 0; + +err_cl_cache: + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + kvfree(q->cl_cache[i].data); +err_l2_cache: + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(q->l2_cache[i].table); + kvfree(q->l1_table); + q->l1_table = NULL; + return -ENOMEM; +} + +void lbd_qcow2_destroy(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int i; + + /* Flush any dirty L2 entries */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (e->valid && e->dirty) + lbd_qcow2_l2_flush(dev, e); + } + + /* Write final alloc_offset and free_list_head */ + lbd_qcow2_write_alloc_offset(dev); + lbd_qcow2_write_free_list_head(dev); + + kvfree(q->read_buf); + kvfree(q->comp_buf); + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + kvfree(q->cl_cache[i].data); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(q->l2_cache[i].table); + + kvfree(q->l1_table); + q->l1_table = NULL; +} + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Miss handler support + * ---------------------------------------------------------------- */ + +/* Declared in lbd.c, defined there to access miss handler internals */ +enum lbd_miss_action lbd_qcow2_handle_miss(struct lbd_device *dev, + u64 cluster_index); + +/* + * Invalidate cached entries for a cluster before retry. + * Called with rwsem released. + */ +static void lbd_qcow2_invalidate_for_retry(struct lbd_device *dev, + u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_qcow2_base *base = dev->base; + int i; + + /* Invalidate primary cluster cache */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) + e->valid = false; + } + + /* Invalidate base L2 cache entry covering this cluster */ + if (base && base->is_qcow2) { + u32 l1_idx = cluster_index / base->l2_entries; + + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (e->valid && e->l1_index == l1_idx) + e->valid = false; + } + } +} +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* ---------------------------------------------------------------- + * Read path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_read(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct req_iterator iter; + struct bio_vec bvec; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + + rq_for_each_segment(bvec, rq, iter) { + void *mapped; + unsigned int remaining = bvec.bv_len; + unsigned int bv_off = bvec.bv_offset; + + mapped = kmap_local_page(bvec.bv_page); + + while (remaining > 0) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, remaining, + q->cluster_size - off_in_cluster); + struct lbd_cl_cache_entry *ce; +#ifdef CONFIG_LBD_MISS_HANDLER + int retries = 0; + +retry_cluster: +#endif + down_read(&q->rwsem); + + ce = lbd_qcow2_cl_load(dev, cluster_idx); +#ifdef CONFIG_LBD_MISS_HANDLER + if (IS_ERR(ce)) { + enum lbd_miss_action action; + + up_read(&q->rwsem); + /* -ENODATA = miss, ask userspace */ + action = lbd_qcow2_handle_miss(dev, + cluster_idx); + if (action == LBD_MISS_RETRY && + retries++ < 3) { + lbd_qcow2_invalidate_for_retry(dev, + cluster_idx); + goto retry_cluster; + } + /* CONTINUE or retry exhausted: zero-fill */ + memset(mapped + bv_off, 0, bytes); + goto next_chunk; + } +#endif + if (!ce) { + /* Real I/O error */ + up_read(&q->rwsem); + kunmap_local(mapped); + return -EIO; + } + + memcpy(mapped + bv_off, ce->data + off_in_cluster, + bytes); + + up_read(&q->rwsem); + +#ifdef CONFIG_LBD_MISS_HANDLER +next_chunk: +#endif + bv_off += bytes; + guest_offset += bytes; + remaining -= bytes; + } + + kunmap_local(mapped); + } + + atomic64_inc(&dev->stat_reads); + atomic64_add(blk_rq_bytes(rq), &dev->stat_read_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * Write path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_write(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct req_iterator iter; + struct bio_vec bvec; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + u32 total_len = blk_rq_bytes(rq); + void *write_data; + size_t offset = 0; + + /* Gather all write data into contiguous buffer */ + write_data = kvmalloc(total_len, GFP_NOIO); + if (!write_data) + return -ENOMEM; + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + memcpy(write_data + offset, mapped + bvec.bv_offset, + bvec.bv_len); + kunmap_local(mapped); + offset += bvec.bv_len; + } + + down_write(&q->rwsem); + + offset = 0; + while (offset < total_len) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, total_len - offset, + q->cluster_size - off_in_cluster); + struct lbd_cl_cache_entry *ce; + struct lbd_l2_cache_entry *l2e; + u32 l1_idx = cluster_idx / q->l2_entries; + u32 l2_idx = cluster_idx % q->l2_entries; + int comp_len; + loff_t phys; + ssize_t ret; + int err; + u64 old_l2_entry; + u64 old_alloc, new_alloc; + + /* Ensure L2 table is allocated */ + err = lbd_qcow2_l2_alloc(dev, l1_idx); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + /* Load existing cluster into cache (or zeros if new) */ + ce = lbd_qcow2_cl_load(dev, cluster_idx); + if (IS_ERR(ce)) { + /* + * -ENODATA: cluster unallocated (miss). For writes we + * don't need the old data — just grab a zeroed cache + * slot and let the write overwrite it. + */ + ce = lbd_qcow2_cl_get_zero(dev, cluster_idx); + if (!ce) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + } else if (!ce) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + + /* Read old L2 entry before modifying */ + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + old_l2_entry = l2e->table[l2_idx]; + old_alloc = lbd_qcow2_read_old_alloc_size(dev, old_l2_entry); + + /* Apply write data to cluster */ + memcpy(ce->data + off_in_cluster, write_data + offset, bytes); + + /* Compress the full cluster */ + comp_len = LZ4_compress_fast_extState( + dev->lz4_state, ce->data, q->comp_buf, + q->cluster_size, + LZ4_compressBound(q->cluster_size), 1); + + if (comp_len > 0 && + (u32)comp_len < q->cluster_size - sizeof(__be32)) { + /* Store compressed */ + __be32 comp_size_be = cpu_to_be32(comp_len); + new_alloc = ALIGN(sizeof(__be32) + comp_len, 4096); + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2_entry & + LBD_QCOW2_L2_OFFSET_MASK; + err = lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + err = lbd_qcow2_alloc_space(dev, new_alloc, + &phys); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + + /* Write size header + compressed data */ + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + + ret = kernel_write(dev->backing_file, + q->comp_buf, comp_len, + &pos); + if (ret != comp_len) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + l2e->dirty = true; + atomic64_inc(&dev->stat_compressed); + } else { + /* Store uncompressed */ + new_alloc = q->cluster_size; + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2_entry & + LBD_QCOW2_L2_OFFSET_MASK; + err = lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + err = lbd_qcow2_alloc_space(dev, new_alloc, + &phys); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + ce->data, q->cluster_size, + &pos); + if (ret != q->cluster_size) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = phys; + l2e->dirty = true; + atomic64_inc(&dev->stat_uncompressed); + } + + /* Flush L2 to disk */ + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + /* Update alloc_offset on disk */ + err = lbd_qcow2_write_alloc_offset(dev); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + offset += bytes; + guest_offset += bytes; + } + + up_write(&q->rwsem); + kvfree(write_data); + return 0; +} + +/* ---------------------------------------------------------------- + * TRIM path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_discard(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + u32 remaining = blk_rq_bytes(rq); + int err; + + down_write(&q->rwsem); + + while (remaining > 0) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, remaining, + q->cluster_size - off_in_cluster); + u32 l1_idx = cluster_idx / q->l2_entries; + u32 l2_idx = cluster_idx % q->l2_entries; + struct lbd_l2_cache_entry *l2e; + + if (off_in_cluster == 0 && bytes == q->cluster_size) { + /* Full-cluster trim */ + if (l1_idx < q->l1_size && + q->l1_table[l1_idx] != 0) { + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (l2e && l2e->table[l2_idx] != 0) { + u64 old_l2 = l2e->table[l2_idx]; + u64 old_sz = lbd_qcow2_read_old_alloc_size( + dev, old_l2); + if (old_sz > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_sz); + } + l2e->table[l2_idx] = 0; + l2e->dirty = true; + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + return err; + } + } + } + lbd_qcow2_cl_invalidate(q, cluster_idx); + } else { + /* Partial-cluster trim: read-modify-write */ + struct lbd_cl_cache_entry *ce; + + err = lbd_qcow2_l2_alloc(dev, l1_idx); + if (err) { + up_write(&q->rwsem); + return err; + } + + ce = lbd_qcow2_cl_load(dev, cluster_idx); + if (IS_ERR(ce)) { + /* + * -ENODATA: cluster unallocated. Trimming + * an unallocated cluster is a no-op. + */ + goto next; + } + if (!ce) { + up_write(&q->rwsem); + return -EIO; + } + + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) { + up_write(&q->rwsem); + return -EIO; + } + + if (l2e->table[l2_idx] == 0) { + goto next; + } + + /* Zero the trimmed portion */ + memset(ce->data + off_in_cluster, 0, bytes); + + /* Recompress and write back */ + { + int comp_len; + loff_t phys; + ssize_t ret; + u64 old_l2 = l2e->table[l2_idx]; + u64 old_alloc = lbd_qcow2_read_old_alloc_size( + dev, old_l2); + u64 new_alloc; + + comp_len = LZ4_compress_fast_extState( + dev->lz4_state, ce->data, q->comp_buf, + q->cluster_size, + LZ4_compressBound(q->cluster_size), 1); + + if (comp_len > 0 && + (u32)comp_len < q->cluster_size - sizeof(__be32)) { + __be32 comp_size_be = cpu_to_be32(comp_len); + new_alloc = ALIGN(sizeof(__be32) + comp_len, 4096); + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2 & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + } + err = lbd_qcow2_alloc_space(dev, + new_alloc, &phys); + if (err) { + up_write(&q->rwsem); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + ret = kernel_write(dev->backing_file, + q->comp_buf, comp_len, + &pos); + if (ret != comp_len) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + atomic64_inc(&dev->stat_compressed); + } else { + new_alloc = q->cluster_size; + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2 & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + } + err = lbd_qcow2_alloc_space(dev, + new_alloc, &phys); + if (err) { + up_write(&q->rwsem); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + ce->data, q->cluster_size, + &pos); + if (ret != q->cluster_size) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = phys; + atomic64_inc(&dev->stat_uncompressed); + } + + l2e->dirty = true; + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + return err; + } + + err = lbd_qcow2_write_alloc_offset(dev); + if (err) { + up_write(&q->rwsem); + return err; + } + } + } + +next: + guest_offset += bytes; + remaining -= bytes; + } + + up_write(&q->rwsem); + return 0; +} + +/* ---------------------------------------------------------------- + * Base layer (thin snapshot) support + * ---------------------------------------------------------------- */ + +static inline u64 lbd_qcow2_base_lru_tick(struct lbd_qcow2_base *base) +{ + return base->lru_tick++; +} + +/* Find or load an L2 table from the base layer into its cache */ +static struct lbd_l2_cache_entry * +lbd_qcow2_base_l2_get(struct lbd_qcow2_base *base, u32 l1_index) +{ + struct lbd_l2_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + /* Check cache for hit */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (e->valid && e->l1_index == l1_index) { + e->lru = lbd_qcow2_base_lru_tick(base); + return e; + } + } + + /* Cache miss: find LRU entry to evict */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* No dirty tracking — just overwrite on eviction */ + best->l1_index = l1_index; + best->dirty = false; + best->lru = lbd_qcow2_base_lru_tick(base); + + if (l1_index < base->l1_size && base->l1_table[l1_index] != 0) { + loff_t pos = base->l1_table[l1_index]; + __be64 *disk_l2; + ssize_t ret; + int j; + + disk_l2 = kvmalloc(base->cluster_size, GFP_NOIO); + if (!disk_l2) { + best->valid = false; + return NULL; + } + + ret = kernel_read(base->file, disk_l2, + base->cluster_size, &pos); + if (ret != base->cluster_size) { + pr_warn("lbd: base L2 read failed for l1[%u]\n", + l1_index); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + + /* Verify CRC32C trailer */ + { + u8 *raw = (u8 *)disk_l2; + u32 stored_crc = _qcow2_get32(raw, base->cluster_size - 4); + u32 calc_crc = ~crc32c(~0, raw, base->cluster_size - 4); + + if (stored_crc != calc_crc) { + pr_warn("lbd: base L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + l1_index, stored_crc, calc_crc); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + } + + for (j = 0; j < base->l2_entries; j++) + best->table[j] = be64_to_cpu(disk_l2[j]); + + kvfree(disk_l2); + } else { + /* Unallocated L2: all zeros */ + memset(best->table, 0, base->cluster_size); + } + + best->valid = true; + return best; +} + +/* + * Read a full cluster from the base layer into buf. + * Returns 0 on success, negative errno on failure. + */ +static int lbd_qcow2_base_read_cluster(struct lbd_device *dev, + u64 cluster_index, void *buf) +{ + struct lbd_qcow2_base *base = dev->base; + u32 cluster_size = dev->qcow2.cluster_size; + + if (!base->is_qcow2) { + /* Raw base: direct read */ + loff_t pos = cluster_index * cluster_size; + ssize_t ret; + + if (pos + cluster_size > base->size) { + /* Unallocated: signal with return 1 */ + return 1; + } + + ret = kernel_read(base->file, buf, cluster_size, &pos); + if (ret != cluster_size) { + pr_warn("lbd%d: base raw read failed at cluster %llu\n", + dev->index, cluster_index); + return ret < 0 ? ret : -EIO; + } + return 0; + } + + /* qcow2 base: L1/L2 lookup */ + { + u32 l1_idx = cluster_index / base->l2_entries; + u32 l2_idx = cluster_index % base->l2_entries; + struct lbd_l2_cache_entry *l2e; + u64 l2_entry, phys_offset; + ssize_t ret; + + l2e = lbd_qcow2_base_l2_get(base, l1_idx); + if (!l2e) + return -EIO; + + l2_entry = l2e->table[l2_idx]; + + if (l2_entry == 0) { + /* Unallocated in base: signal with return 1 */ + return 1; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + __be32 comp_size_be; + u32 comp_size; + int dec_len; + loff_t pos = phys_offset; + + ret = kernel_read(base->file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + pr_warn("lbd%d: base compressed size read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + + comp_size = be32_to_cpu(comp_size_be); + if (comp_size > LZ4_compressBound(cluster_size)) { + pr_warn("lbd%d: base invalid compressed size %u\n", + dev->index, comp_size); + return -EIO; + } + + ret = kernel_read(base->file, base->read_buf, + comp_size, &pos); + if (ret != comp_size) { + pr_warn("lbd%d: base compressed data read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + + dec_len = LZ4_decompress_safe(base->read_buf, buf, + comp_size, cluster_size); + if (dec_len != cluster_size) { + pr_warn("lbd%d: base LZ4 decompress failed (%d)\n", + dev->index, dec_len); + return -EIO; + } + } else { + /* Uncompressed cluster */ + loff_t pos = phys_offset; + + ret = kernel_read(base->file, buf, + cluster_size, &pos); + if (ret != cluster_size) { + pr_warn("lbd%d: base cluster read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + } + } + + return 0; +} + +/* ---------------------------------------------------------------- + * Swap base layer and reload L1 + * ---------------------------------------------------------------- */ + +/* + * Re-read the primary's L1 table from disk. + * Called with rwsem held for write. + */ +static int lbd_qcow2_reload_l1(struct lbd_qcow2 *q, struct lbd_device *dev) +{ + u8 *hdr; + loff_t pos = 0; + ssize_t ret; + __be64 *disk_l1; + u64 *new_l1; + u32 new_l1_size; + loff_t new_l1_offset; + int i; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_NOIO); + if (!hdr) + return -ENOMEM; + + ret = kernel_read(dev->backing_file, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + kvfree(hdr); + return ret < 0 ? ret : -EIO; + } + + new_l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + new_l1_size = lbd_qcow2_hdr_l1_size(hdr); + q->alloc_offset = lbd_qcow2_hdr_alloc_offset(hdr); + q->free_list_head = lbd_qcow2_hdr_free_list(hdr); + kvfree(hdr); + + new_l1 = kvmalloc_array(new_l1_size, sizeof(u64), GFP_NOIO); + if (!new_l1) + return -ENOMEM; + + disk_l1 = kvmalloc_array(new_l1_size, sizeof(__be64), GFP_NOIO); + if (!disk_l1) { + kvfree(new_l1); + return -ENOMEM; + } + + pos = new_l1_offset; + ret = kernel_read(dev->backing_file, disk_l1, + new_l1_size * sizeof(__be64), &pos); + if (ret != new_l1_size * sizeof(__be64)) { + kvfree(disk_l1); + kvfree(new_l1); + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < new_l1_size; i++) + new_l1[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + kvfree(q->l1_table); + q->l1_table = new_l1; + q->l1_size = new_l1_size; + q->l1_offset = new_l1_offset; + + return 0; +} + +/* + * Invalidate all caches (primary + base). + * Called with rwsem held for write. + */ +static void lbd_qcow2_invalidate_all_caches(struct lbd_qcow2 *q, + struct lbd_qcow2_base *base) +{ + int i; + + /* Primary cluster cache */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + q->cl_cache[i].valid = false; + + /* Primary L2 cache */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + q->l2_cache[i].valid = false; + + /* Base layer caches */ + if (base && base->is_qcow2) { + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + base->l2_cache[i].valid = false; + } +} + +int lbd_qcow2_swap_base(struct lbd_device *dev, const char *new_path) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int ret; + + down_write(&q->rwsem); + + /* Flush all dirty L2 tables */ + lbd_qcow2_flush_all_l2(dev); + + /* Close old base */ + if (dev->base) { + lbd_qcow2_base_destroy(dev->base); + dev->base = NULL; + } + + /* Open new base */ + ret = lbd_qcow2_base_init(dev, new_path); + if (ret) { + pr_warn("lbd%d: swap_base failed to open '%s': %d\n", + dev->index, new_path, ret); + up_write(&q->rwsem); + return ret; + } + + /* Reload primary L1 table from disk */ + ret = lbd_qcow2_reload_l1(q, dev); + if (ret) { + pr_warn("lbd%d: swap_base failed to reload L1: %d\n", + dev->index, ret); + up_write(&q->rwsem); + return ret; + } + + /* Invalidate all cached data */ + lbd_qcow2_invalidate_all_caches(q, dev->base); + + /* Update stored path */ + strscpy(dev->base_path, new_path, sizeof(dev->base_path)); + + up_write(&q->rwsem); + + pr_info("lbd%d: base layer swapped to '%s'\n", dev->index, new_path); + return 0; +} + +int lbd_qcow2_base_init(struct lbd_device *dev, const char *path) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_qcow2_base *base; + struct file *f; + struct inode *inode, *primary_inode; + u64 magic; + loff_t pos; + ssize_t ret; + int i; + + f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(f)) { + pr_err("lbd%d: cannot open base file '%s': %ld\n", + dev->index, path, PTR_ERR(f)); + return PTR_ERR(f); + } + + inode = file_inode(f); + if (!S_ISREG(inode->i_mode)) { + pr_err("lbd%d: base path must be a regular file\n", + dev->index); + fput(f); + return -EINVAL; + } + + if (i_size_read(inode) == 0) { + pr_err("lbd%d: base file is empty\n", dev->index); + fput(f); + return -EINVAL; + } + + /* Base and primary must be different files */ + primary_inode = file_inode(dev->backing_file); + if (inode->i_sb == primary_inode->i_sb && + inode->i_ino == primary_inode->i_ino) { + pr_err("lbd%d: base and primary must be different files\n", + dev->index); + fput(f); + return -EINVAL; + } + + base = kzalloc(sizeof(*base), GFP_KERNEL); + if (!base) { + fput(f); + return -ENOMEM; + } + + base->file = f; + base->lru_tick = 0; + + /* Detect raw vs qcow2 via magic */ + pos = 0; + ret = kernel_read(f, &magic, 8, &pos); + if (ret != 8) { + pr_err("lbd%d: cannot read base file header\n", dev->index); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + if (be64_to_cpu(magic) == LBD_QCOW2_MAGIC) { + /* qcow2 base */ + u8 *hdr; + __be64 *disk_l1; + + base->is_qcow2 = true; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_KERNEL); + if (!hdr) { + kfree(base); + fput(f); + return -ENOMEM; + } + + pos = 0; + ret = kernel_read(f, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + pr_err("lbd%d: base qcow2 header read failed\n", + dev->index); + kvfree(hdr); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + if (lbd_qcow2_hdr_version(hdr) != LBD_QCOW2_VERSION) { + pr_err("lbd%d: base qcow2 version mismatch (%u)\n", + dev->index, lbd_qcow2_hdr_version(hdr)); + kvfree(hdr); + kfree(base); + fput(f); + return -EINVAL; + } + + base->cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + if (base->cluster_bits != q->cluster_bits) { + pr_err("lbd%d: base cluster_bits %u != primary %u\n", + dev->index, base->cluster_bits, q->cluster_bits); + kvfree(hdr); + kfree(base); + fput(f); + return -EINVAL; + } + + base->cluster_size = 1U << base->cluster_bits; + base->l2_entries = (base->cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / sizeof(u64); + base->size = lbd_qcow2_hdr_virtual_size(hdr); + base->l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + base->l1_size = lbd_qcow2_hdr_l1_size(hdr); + + kvfree(hdr); + + /* Validate virtual size matches primary */ + if (base->size != q->virtual_size) { + pr_err("lbd%d: base virtual_size %llu != primary %llu\n", + dev->index, base->size, q->virtual_size); + kfree(base); + fput(f); + return -EINVAL; + } + + /* Load L1 table */ + base->l1_table = kvmalloc_array(base->l1_size, sizeof(u64), + GFP_KERNEL); + if (!base->l1_table) { + kfree(base); + fput(f); + return -ENOMEM; + } + + disk_l1 = kvmalloc_array(base->l1_size, sizeof(__be64), + GFP_KERNEL); + if (!disk_l1) { + kvfree(base->l1_table); + kfree(base); + fput(f); + return -ENOMEM; + } + + pos = base->l1_offset; + ret = kernel_read(f, disk_l1, + base->l1_size * sizeof(__be64), &pos); + if (ret != base->l1_size * sizeof(__be64)) { + pr_err("lbd%d: base L1 table read failed\n", + dev->index); + kvfree(disk_l1); + kvfree(base->l1_table); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < base->l1_size; i++) + base->l1_table[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + /* Allocate L2 cache tables */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + e->table = kvmalloc(base->cluster_size, GFP_KERNEL); + if (!e->table) + goto err_l2_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate read buffer for decompression */ + base->read_buf = kvmalloc(LZ4_compressBound(base->cluster_size), + GFP_KERNEL); + if (!base->read_buf) + goto err_l2_cache; + } else { + /* Raw base */ + base->is_qcow2 = false; + base->size = i_size_read(inode); + + /* Validate size matches primary */ + if (base->size != q->virtual_size) { + pr_err("lbd%d: base size %llu != primary virtual_size %llu\n", + dev->index, base->size, q->virtual_size); + kfree(base); + fput(f); + return -EINVAL; + } + } + + dev->base = base; + pr_info("lbd%d: base layer attached (%s, %llu bytes)\n", + dev->index, base->is_qcow2 ? "qcow2-lz4" : "raw", + base->size); + return 0; + +err_l2_cache: + if (base->is_qcow2) { + kvfree(base->read_buf); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(base->l2_cache[i].table); + kvfree(base->l1_table); + } + kfree(base); + fput(f); + return -ENOMEM; +} + +void lbd_qcow2_base_destroy(struct lbd_qcow2_base *base) +{ + int i; + + if (base->is_qcow2) { + kvfree(base->read_buf); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(base->l2_cache[i].table); + kvfree(base->l1_table); + } + + fput(base->file); + kfree(base); +} diff --git a/third_party/lbd/src/lbd_qcow2.h b/third_party/lbd/src/lbd_qcow2.h new file mode 100644 index 000000000..0d743cba8 --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LBD_QCOW2_H +#define _LBD_QCOW2_H + +#include +#include + +#include "lbd_qcow2_format.h" + +/* Cache sizes */ +#define LBD_QCOW2_L2_CACHE_SIZE 16 +#define LBD_QCOW2_CL_CACHE_SIZE 16 +/* ---------------------------------------------------------------- + * In-memory cache structures + * ---------------------------------------------------------------- */ + +struct lbd_l2_cache_entry { + u32 l1_index; + bool valid; + bool dirty; + u64 lru; + u64 *table; /* host-endian L2 entries, kvmalloc(cluster_size) */ +}; + +struct lbd_cl_cache_entry { + u64 cluster_index; + bool valid; + bool dirty; + u64 lru; + void *data; /* kvmalloc(cluster_size) */ +}; + +/* ---------------------------------------------------------------- + * Read-only base layer state (for thin snapshots) + * ---------------------------------------------------------------- */ + +struct lbd_qcow2_base { + struct file *file; + bool is_qcow2; + u64 size; /* raw: file size, qcow2: virtual_size */ + + /* Only valid when is_qcow2 == true */ + u32 cluster_bits; + u32 cluster_size; + u32 l2_entries; + u64 *l1_table; + u32 l1_size; + loff_t l1_offset; + + struct lbd_l2_cache_entry l2_cache[LBD_QCOW2_L2_CACHE_SIZE]; + u64 lru_tick; + + void *read_buf; /* decompression buffer */ +}; + +/* ---------------------------------------------------------------- + * Per-device qcow2 state (embedded in struct lbd_device) + * ---------------------------------------------------------------- */ + +struct lbd_qcow2 { + u32 cluster_bits; + u32 cluster_size; /* 1 << cluster_bits */ + u32 l2_entries; /* (cluster_size - L2_TRAILER_SIZE) / 8 */ + u64 virtual_size; + + u64 *l1_table; /* host-endian, always resident */ + u32 l1_size; + loff_t l1_offset; + + loff_t alloc_offset; /* append-only allocation cursor */ + loff_t free_list_head; /* head of on-disk free list, 0 if empty */ + + struct lbd_l2_cache_entry l2_cache[LBD_QCOW2_L2_CACHE_SIZE]; + struct lbd_cl_cache_entry cl_cache[LBD_QCOW2_CL_CACHE_SIZE]; + u64 lru_tick; /* monotonic counter for LRU */ + + struct rw_semaphore rwsem; /* read-shared, write-exclusive */ + + void *comp_buf; /* LZ4_compressBound(cluster_size) */ + void *read_buf; /* cluster_size, for reading compressed */ +}; + +/* ---------------------------------------------------------------- + * Function declarations + * ---------------------------------------------------------------- */ + +struct lbd_device; +struct request; + +int lbd_qcow2_init(struct lbd_device *dev); +void lbd_qcow2_destroy(struct lbd_device *dev); +int lbd_qcow2_read(struct lbd_device *dev, struct request *rq); +int lbd_qcow2_write(struct lbd_device *dev, struct request *rq); +int lbd_qcow2_discard(struct lbd_device *dev, struct request *rq); + +/* Base layer (thin snapshot) */ +int lbd_qcow2_base_init(struct lbd_device *dev, const char *path); +void lbd_qcow2_base_destroy(struct lbd_qcow2_base *base); +int lbd_qcow2_swap_base(struct lbd_device *dev, const char *new_path); + +#endif /* _LBD_QCOW2_H */ diff --git a/third_party/lbd/src/lbd_qcow2_format.h b/third_party/lbd/src/lbd_qcow2_format.h new file mode 100644 index 000000000..a310c7ab8 --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2_format.h @@ -0,0 +1,254 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * LBD qcow2-lz4 on-disk format definitions. + * + * Shared between kernel module and userspace tools (lbdctl). + * All header fields are big-endian on disk and accessed at explicit + * byte offsets — no struct casting. + */ +#ifndef _LBD_QCOW2_FORMAT_H +#define _LBD_QCOW2_FORMAT_H + +/* ---------------------------------------------------------------- + * Constants + * ---------------------------------------------------------------- */ + +#define LBD_QCOW2_MAGIC 0x4C42444351573200ULL /* "LBDQCW2\0" */ +#define LBD_QCOW2_VERSION 2 +#define LBD_QCOW2_COMP_LZ4 1 +#define LBD_QCOW2_HEADER_SIZE 4096 +#define LBD_QCOW2_L2_TRAILER_SIZE 8 /* 4 bytes reserved + 4 bytes CRC32C */ + +/* Header field byte offsets */ +#define LBD_QCOW2_OFF_MAGIC 0 /* u64 */ +#define LBD_QCOW2_OFF_VERSION 8 /* u32 */ +#define LBD_QCOW2_OFF_CLUSTER_BITS 12 /* u32 */ +#define LBD_QCOW2_OFF_VIRTUAL_SIZE 16 /* u64 */ +#define LBD_QCOW2_OFF_L1_TABLE_OFFSET 24 /* u64 */ +#define LBD_QCOW2_OFF_L1_SIZE 32 /* u32 */ +#define LBD_QCOW2_OFF_ALLOC_OFFSET 36 /* u64 */ +#define LBD_QCOW2_OFF_COMP_TYPE 44 /* u32 */ +#define LBD_QCOW2_OFF_FREE_LIST 48 /* u64 */ + +/* Free list tombstone marker */ +#define LBD_QCOW2_FREE_TOMBSTONE 0xDEADF4EEU + +/* L2 entry flags */ +#define LBD_QCOW2_L2_COMPRESSED (1ULL << 63) +#define LBD_QCOW2_L2_OFFSET_MASK 0x3FFFFFFFFFFFFFFFULL + +/* Default cluster parameters */ +#define LBD_QCOW2_CLUSTER_BITS_DEFAULT 16 /* 64 KiB clusters */ +#define LBD_QCOW2_CLUSTER_SIZE_DEFAULT (1U << LBD_QCOW2_CLUSTER_BITS_DEFAULT) + +/* ---------------------------------------------------------------- + * Byte-order helpers (kernel vs userspace) + * ---------------------------------------------------------------- */ + +#ifdef __KERNEL__ + +#include +#include +#include + +typedef u8 _qcow2_u8; +typedef u32 _qcow2_u32; +typedef u64 _qcow2_u64; + +static inline _qcow2_u64 _qcow2_get64(const void *buf, int off) +{ + __be64 v; + memcpy(&v, (const u8 *)buf + off, 8); + return be64_to_cpu(v); +} + +static inline _qcow2_u32 _qcow2_get32(const void *buf, int off) +{ + __be32 v; + memcpy(&v, (const u8 *)buf + off, 4); + return be32_to_cpu(v); +} + +static inline void _qcow2_put64(void *buf, int off, _qcow2_u64 val) +{ + __be64 v = cpu_to_be64(val); + memcpy((u8 *)buf + off, &v, 8); +} + +static inline void _qcow2_put32(void *buf, int off, _qcow2_u32 val) +{ + __be32 v = cpu_to_be32(val); + memcpy((u8 *)buf + off, &v, 4); +} + +typedef u16 _qcow2_u16; + +static inline _qcow2_u16 _qcow2_get16(const void *buf, int off) +{ + __be16 v; + memcpy(&v, (const u8 *)buf + off, 2); + return be16_to_cpu(v); +} + +static inline void _qcow2_put16(void *buf, int off, _qcow2_u16 val) +{ + __be16 v = cpu_to_be16(val); + memcpy((u8 *)buf + off, &v, 2); +} + +#else /* userspace */ + +#include +#include + +typedef uint8_t _qcow2_u8; +typedef uint32_t _qcow2_u32; +typedef uint64_t _qcow2_u64; + +static inline _qcow2_u64 _qcow2_get64(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint64_t)p[0] << 56) | ((uint64_t)p[1] << 48) | + ((uint64_t)p[2] << 40) | ((uint64_t)p[3] << 32) | + ((uint64_t)p[4] << 24) | ((uint64_t)p[5] << 16) | + ((uint64_t)p[6] << 8) | (uint64_t)p[7]; +} + +static inline _qcow2_u32 _qcow2_get32(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | + ((uint32_t)p[2] << 8) | (uint32_t)p[3]; +} + +static inline void _qcow2_put64(void *buf, int off, _qcow2_u64 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 56); p[1] = (uint8_t)(val >> 48); + p[2] = (uint8_t)(val >> 40); p[3] = (uint8_t)(val >> 32); + p[4] = (uint8_t)(val >> 24); p[5] = (uint8_t)(val >> 16); + p[6] = (uint8_t)(val >> 8); p[7] = (uint8_t)val; +} + +static inline void _qcow2_put32(void *buf, int off, _qcow2_u32 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 24); p[1] = (uint8_t)(val >> 16); + p[2] = (uint8_t)(val >> 8); p[3] = (uint8_t)val; +} + +typedef uint16_t _qcow2_u16; + +static inline _qcow2_u16 _qcow2_get16(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint16_t)p[0] << 8) | (uint16_t)p[1]; +} + +static inline void _qcow2_put16(void *buf, int off, _qcow2_u16 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 8); p[1] = (uint8_t)val; +} + +#endif /* __KERNEL__ */ + +/* ---------------------------------------------------------------- + * Per-field accessors: getters + * ---------------------------------------------------------------- */ + +static inline _qcow2_u64 lbd_qcow2_hdr_magic(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_MAGIC); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_version(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_VERSION); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_cluster_bits(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_CLUSTER_BITS); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_virtual_size(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_VIRTUAL_SIZE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_l1_table_offset(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_L1_TABLE_OFFSET); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_l1_size(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_L1_SIZE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_alloc_offset(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_ALLOC_OFFSET); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_comp_type(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_COMP_TYPE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_free_list(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_FREE_LIST); +} + +/* ---------------------------------------------------------------- + * Per-field accessors: setters + * ---------------------------------------------------------------- */ + +static inline void lbd_qcow2_hdr_set_magic(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_MAGIC, v); +} + +static inline void lbd_qcow2_hdr_set_version(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_VERSION, v); +} + +static inline void lbd_qcow2_hdr_set_cluster_bits(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_CLUSTER_BITS, v); +} + +static inline void lbd_qcow2_hdr_set_virtual_size(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_VIRTUAL_SIZE, v); +} + +static inline void lbd_qcow2_hdr_set_l1_table_offset(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_L1_TABLE_OFFSET, v); +} + +static inline void lbd_qcow2_hdr_set_l1_size(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_L1_SIZE, v); +} + +static inline void lbd_qcow2_hdr_set_alloc_offset(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_ALLOC_OFFSET, v); +} + +static inline void lbd_qcow2_hdr_set_comp_type(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_COMP_TYPE, v); +} + +static inline void lbd_qcow2_hdr_set_free_list(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_FREE_LIST, v); +} + +#endif /* _LBD_QCOW2_FORMAT_H */ diff --git a/third_party/lbd/src/lbdctl.c b/third_party/lbd/src/lbdctl.c new file mode 100644 index 000000000..6a10c642c --- /dev/null +++ b/third_party/lbd/src/lbdctl.c @@ -0,0 +1,2949 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * lbdctl - userspace control tool for LBD (Logging Block Device) + * + * Usage: + * lbdctl add [--json] --log-dir /path/to/logs /path/to/file.img + * lbdctl remove [--json] N - destroy /dev/lbdN + * lbdctl list [--json] - show all active devices + * lbdctl log [--json] /path/to/file.img.log - dump log + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lz4/lz4.h" + +/* Mirror the kernel ioctl structures from lbd.h */ +#define LBD_LOG_PATH_MAX 256 + +/* CBOR log numeric map keys */ +#define LBD_CBOR_KEY_HDR_VERSION 1 +#define LBD_CBOR_KEY_HDR_BLOCK_SIZE 2 +#define LBD_CBOR_KEY_HDR_SEGMENT_LABEL 3 +#define LBD_CBOR_KEY_HDR_DEVICE_SIZE 4 +#define LBD_CBOR_KEY_HDR_BACKING_PATH 5 + +#define LBD_CBOR_KEY_OP 1 +#define LBD_CBOR_KEY_TIMESTAMP 2 +#define LBD_CBOR_KEY_SEQUENCE 3 +#define LBD_CBOR_KEY_BLOCK 4 +#define LBD_CBOR_KEY_LENGTH 5 +#define LBD_CBOR_KEY_CHECKSUM 6 +#define LBD_CBOR_KEY_DATA 7 + +/* Watch command keys (write path) */ +#define LBD_WATCH_KEY_CMD 1 +#define LBD_WATCH_KEY_DEV 2 +#define LBD_WATCH_KEY_PATH 3 + +/* Miss event keys */ +#define LBD_MISS_KEY_TYPE 1 +#define LBD_MISS_KEY_DEV 2 +#define LBD_MISS_KEY_CLUSTER 3 + +/* Event keys (read path) */ +#define LBD_EVENT_KEY_TYPE 1 +#define LBD_EVENT_KEY_DEV 2 +#define LBD_EVENT_KEY_LABEL 3 +#define LBD_EVENT_KEY_DIR 4 +#define LBD_EVENT_KEY_SEQ 5 +#define LBD_EVENT_KEY_SIZE 6 + +#define LBD_CTL_MAGIC 'L' + +struct lbd_ctl_add { + char path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + char base_path[LBD_LOG_PATH_MAX]; /* empty string = no base */ + __s32 index; + __u64 log_max_size; + __u32 log_max_age_secs; +}; + +struct lbd_ctl_remove { + __s32 index; +}; + +struct lbd_ctl_info { + __s32 index; + __u32 state; + __u64 size; + char path[LBD_LOG_PATH_MAX]; +}; + +#define LBD_CTL_ADD _IOWR(LBD_CTL_MAGIC, 0, struct lbd_ctl_add) +#define LBD_CTL_REMOVE _IOW(LBD_CTL_MAGIC, 1, struct lbd_ctl_remove) +#define LBD_CTL_INFO _IOWR(LBD_CTL_MAGIC, 2, struct lbd_ctl_info) + +#define LBD_CTL_PATH "/dev/lbd-control" + +/* ---------------------------------------------------------------- + * CRC32 (IEEE 802.3 polynomial, matches Linux kernel crc32()) + * ---------------------------------------------------------------- */ + +static uint32_t crc32_table[256]; +static int crc32_table_ready; + +static void crc32_init(void) +{ + uint32_t poly = 0xEDB88320; + for (int i = 0; i < 256; i++) { + uint32_t c = i; + for (int j = 0; j < 8; j++) + c = (c >> 1) ^ (poly & (-(c & 1))); + crc32_table[i] = c; + } + crc32_table_ready = 1; +} + +static uint32_t crc32_calc(const void *buf, size_t len) +{ + if (!crc32_table_ready) + crc32_init(); + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFF; + for (size_t i = 0; i < len; i++) + crc = crc32_table[(crc ^ p[i]) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFF; +} + +/* ---------------------------------------------------------------- + * CRC32C (Castagnoli polynomial 0x82F63B78, matches Linux crc32c()) + * ---------------------------------------------------------------- */ + +static uint32_t crc32c_table[256]; +static int crc32c_table_ready; + +static void crc32c_init(void) +{ + uint32_t poly = 0x82F63B78; + for (int i = 0; i < 256; i++) { + uint32_t c = i; + for (int j = 0; j < 8; j++) + c = (c >> 1) ^ (poly & (-(c & 1))); + crc32c_table[i] = c; + } + crc32c_table_ready = 1; +} + +static uint32_t crc32c_calc(const void *buf, size_t len) +{ + if (!crc32c_table_ready) + crc32c_init(); + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFF; + for (size_t i = 0; i < len; i++) + crc = crc32c_table[(crc ^ p[i]) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFF; +} + +/* ---------------------------------------------------------------- + * Helpers + * ---------------------------------------------------------------- */ + +static const char *state_str(unsigned int state) +{ + switch (state) { + case 0: return "unbound"; + case 1: return "bound"; + case 2: return "removing"; + default: return "unknown"; + } +} + +static int open_ctl(void) +{ + int fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + } + return fd; +} + +static int read_exact(int fd, void *buf, size_t len) +{ + size_t done = 0; + while (done < len) { + ssize_t n = read(fd, (char *)buf + done, len - done); + if (n < 0) { + if (errno == EINTR) + continue; + return -1; + } + if (n == 0) + return done > 0 ? -1 : 0; /* EOF */ + done += n; + } + return 1; /* success */ +} + +/* Escape a string for JSON output (handles \, ", and control chars) */ +static void json_print_string(FILE *out, const char *s) +{ + fputc('"', out); + for (; *s; s++) { + unsigned char c = *s; + switch (c) { + case '"': fputs("\\\"", out); break; + case '\\': fputs("\\\\", out); break; + case '\b': fputs("\\b", out); break; + case '\f': fputs("\\f", out); break; + case '\n': fputs("\\n", out); break; + case '\r': fputs("\\r", out); break; + case '\t': fputs("\\t", out); break; + default: + if (c < 0x20) + fprintf(out, "\\u%04x", c); + else + fputc(c, out); + } + } + fputc('"', out); +} + +/* Format size with human-readable units */ +static const char *fmt_size(uint64_t bytes, char *buf, size_t bufsz) +{ + if (bytes >= (1ULL << 30)) + snprintf(buf, bufsz, "%.1f GiB", (double)bytes / (1ULL << 30)); + else if (bytes >= (1ULL << 20)) + snprintf(buf, bufsz, "%.1f MiB", (double)bytes / (1ULL << 20)); + else if (bytes >= (1ULL << 10)) + snprintf(buf, bufsz, "%.1f KiB", (double)bytes / (1ULL << 10)); + else + snprintf(buf, bufsz, "%llu B", (unsigned long long)bytes); + return buf; +} + +/* Print hex dump of data (16 bytes per line) */ +static void hex_dump(FILE *out, const uint8_t *data, size_t len, + const char *indent) +{ + for (size_t off = 0; off < len; off += 16) { + fprintf(out, "%s%08zx ", indent, off); + /* hex */ + for (size_t i = 0; i < 16; i++) { + if (off + i < len) + fprintf(out, "%02x ", data[off + i]); + else + fputs(" ", out); + if (i == 7) + fputc(' ', out); + } + fputs(" |", out); + /* ascii */ + for (size_t i = 0; i < 16 && off + i < len; i++) { + uint8_t c = data[off + i]; + fputc((c >= 0x20 && c < 0x7f) ? c : '.', out); + } + fputs("|\n", out); + } +} + +/* ---------------------------------------------------------------- + * Device control commands + * ---------------------------------------------------------------- */ + +static int cmd_add(int argc, char **argv) +{ + struct lbd_ctl_add arg; + char resolved[PATH_MAX]; + const char *path = NULL; + const char *log_dir = NULL; + const char *base = NULL; + int fd, ret; + int json = 0; + + memset(&arg, 0, sizeof(arg)); + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else if (strcmp(argv[i], "--log-max-size") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-max-size requires a value\n"); + return 1; + } + arg.log_max_size = strtoull(argv[i], NULL, 0); + } else if (strcmp(argv[i], "--log-max-age") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-max-age requires a value\n"); + return 1; + } + arg.log_max_age_secs = strtoul(argv[i], NULL, 0); + } else if (strcmp(argv[i], "--log-dir") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-dir requires a value\n"); + return 1; + } + log_dir = argv[i]; + } else if (strcmp(argv[i], "--base") == 0) { + if (++i >= argc) { + fprintf(stderr, "--base requires a value\n"); + return 1; + } + base = argv[i]; + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "add requires a path argument\n"); + return 1; + } + + if (!log_dir) { + fprintf(stderr, "add requires --log-dir \n"); + return 1; + } + + if (!realpath(path, resolved)) { + fprintf(stderr, "Cannot resolve path '%s': %s\n", + path, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Path too long (max %d)\n", LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.path, LBD_LOG_PATH_MAX, "%s", resolved); + + if (!realpath(log_dir, resolved)) { + fprintf(stderr, "Cannot resolve log directory '%s': %s\n", + log_dir, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Log directory path too long (max %d)\n", + LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.log_dir, LBD_LOG_PATH_MAX, "%s", resolved); + + if (base) { + if (!realpath(base, resolved)) { + fprintf(stderr, "Cannot resolve base path '%s': %s\n", + base, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Base path too long (max %d)\n", + LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.base_path, LBD_LOG_PATH_MAX, "%s", resolved); + } + + fd = open_ctl(); + if (fd < 0) + return 1; + + ret = ioctl(fd, LBD_CTL_ADD, &arg); + if (ret < 0) { + fprintf(stderr, "LBD_CTL_ADD failed: %s\n", strerror(errno)); + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d}\n", + arg.index, arg.index); + } else { + printf("Created /dev/lbd%d\n", arg.index); + } + close(fd); + return 0; +} + +static int cmd_remove(int argc, char **argv) +{ + struct lbd_ctl_remove arg; + int fd, ret; + int json = 0; + const char *index_str = NULL; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else if (!index_str) + index_str = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!index_str) { + fprintf(stderr, "remove requires an index argument\n"); + return 1; + } + + arg.index = atoi(index_str); + + fd = open_ctl(); + if (fd < 0) + return 1; + + ret = ioctl(fd, LBD_CTL_REMOVE, &arg); + if (ret < 0) { + fprintf(stderr, "LBD_CTL_REMOVE failed: %s\n", strerror(errno)); + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d}\n", + arg.index, arg.index); + } else { + printf("Removed /dev/lbd%d\n", arg.index); + } + close(fd); + return 0; +} + +static int cmd_list(int argc, char **argv) +{ + struct lbd_ctl_info info; + int fd, i, found = 0; + int json = 0; + + for (i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + fd = open_ctl(); + if (fd < 0) + return 1; + + if (json) { + printf("["); + } else { + printf("%-8s %-10s %-12s %s\n", "DEVICE", "STATE", "SIZE", "BACKING"); + printf("%-8s %-10s %-12s %s\n", "------", "-----", "----", "-------"); + } + + for (i = 0; i < 256; i++) { + memset(&info, 0, sizeof(info)); + info.index = i; + + if (ioctl(fd, LBD_CTL_INFO, &info) < 0) + continue; + + if (json) { + if (found > 0) + printf(","); + printf("\n {\"device\": \"/dev/lbd%d\", \"index\": %d, \"state\": \"%s\", \"size\": %llu, \"backing\": ", + info.index, info.index, state_str(info.state), + (unsigned long long)info.size); + json_print_string(stdout, info.path); + printf("}"); + } else { + printf("lbd%-5d %-10s %-12llu %s\n", + info.index, state_str(info.state), + (unsigned long long)info.size, info.path); + } + found++; + } + + if (json) { + printf("\n]\n"); + } else if (!found) { + printf("(no devices)\n"); + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * CBOR decoder (streaming, reads from fd) + * ---------------------------------------------------------------- */ + +/* + * Read a CBOR head: returns major type (0-7) and argument value. + * Returns 0 on success, -1 on EOF, -2 on error. + */ +static int cbor_read_head(int fd, uint8_t *major_out, uint64_t *val_out) +{ + uint8_t ib; + int ret = read_exact(fd, &ib, 1); + if (ret == 0) + return -1; /* EOF */ + if (ret < 0) + return -2; + + *major_out = ib >> 5; + uint8_t ai = ib & 0x1F; + + if (ai < 24) { + *val_out = ai; + } else if (ai == 24) { + uint8_t b; + if (read_exact(fd, &b, 1) <= 0) return -2; + *val_out = b; + } else if (ai == 25) { + uint8_t b[2]; + if (read_exact(fd, b, 2) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 8) | b[1]; + } else if (ai == 26) { + uint8_t b[4]; + if (read_exact(fd, b, 4) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 24) | ((uint64_t)b[1] << 16) | + ((uint64_t)b[2] << 8) | b[3]; + } else if (ai == 27) { + uint8_t b[8]; + if (read_exact(fd, b, 8) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 56) | ((uint64_t)b[1] << 48) | + ((uint64_t)b[2] << 40) | ((uint64_t)b[3] << 32) | + ((uint64_t)b[4] << 24) | ((uint64_t)b[5] << 16) | + ((uint64_t)b[6] << 8) | b[7]; + } else { + return -2; /* indefinite / reserved */ + } + + return 0; +} + +/* Expect a uint (major 0), returns 0 on success */ +static int cbor_read_uint(int fd, uint64_t *val_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, val_out); + if (ret) return ret; + if (major != 0) return -2; + return 0; +} + +/* Read a text string (major 3) into buf, NUL-terminated. Returns 0 on success */ +static int cbor_read_text(int fd, char *buf, size_t cap, uint64_t *len_out) +{ + uint8_t major; + uint64_t len; + int ret = cbor_read_head(fd, &major, &len); + if (ret) return ret; + if (major != 3) return -2; + if (len >= cap) return -2; /* too long */ + if (len > 0 && read_exact(fd, buf, len) <= 0) return -2; + buf[len] = '\0'; + if (len_out) *len_out = len; + return 0; +} + +/* Read a byte string header (major 2), returns length */ +static int cbor_read_bytes_hdr(int fd, uint64_t *len_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, len_out); + if (ret) return ret; + if (major != 2) return -2; + return 0; +} + +/* Read a map header (major 5), returns count */ +static int cbor_read_map(int fd, uint64_t *count_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, count_out); + if (ret) return ret; + if (major != 5) return -2; + return 0; +} + +/* Skip one CBOR data item (recursively handles maps, arrays, etc.) */ +static int cbor_skip(int fd) +{ + uint8_t major; + uint64_t val; + int ret = cbor_read_head(fd, &major, &val); + if (ret) return ret; + + switch (major) { + case 0: /* uint */ + case 1: /* negint */ + case 7: /* simple/float */ + return 0; + case 2: /* byte string */ + case 3: /* text string */ + if (val > 0) { + /* Skip val bytes */ + while (val > 0) { + uint8_t tmp[256]; + size_t chunk = val < sizeof(tmp) ? (size_t)val : sizeof(tmp); + if (read_exact(fd, tmp, chunk) <= 0) return -2; + val -= chunk; + } + } + return 0; + case 4: /* array */ + for (uint64_t i = 0; i < val; i++) { + if (cbor_skip(fd)) return -2; + } + return 0; + case 5: /* map */ + for (uint64_t i = 0; i < val; i++) { + if (cbor_skip(fd)) return -2; /* key */ + if (cbor_skip(fd)) return -2; /* value */ + } + return 0; + default: + return -2; + } +} + +/* ---------------------------------------------------------------- + * Log reading (CBOR format) + * ---------------------------------------------------------------- */ + +static int cmd_log_cbor(int fd, const char *path, int json, int show_data) +{ + uint64_t map_count, key, val; + uint32_t version = 0, block_size = 4096; + char segment_label[32]; + uint64_t device_size = 0; + char backing_path[LBD_LOG_PATH_MAX]; + int entry_count = 0; + char sizebuf[32]; + uint8_t *data = NULL; + size_t data_cap = 0; + + memset(segment_label, 0, sizeof(segment_label)); + memset(backing_path, 0, sizeof(backing_path)); + + /* Read header map */ + if (cbor_read_map(fd, &map_count)) { + fprintf(stderr, "Failed to read CBOR header map\n"); + return 1; + } + + for (uint64_t i = 0; i < map_count; i++) { + if (cbor_read_uint(fd, &key)) { + fprintf(stderr, "Failed to read header key\n"); + return 1; + } + switch (key) { + case LBD_CBOR_KEY_HDR_VERSION: + if (cbor_read_uint(fd, &val)) return 1; + version = (uint32_t)val; + break; + case LBD_CBOR_KEY_HDR_BLOCK_SIZE: + if (cbor_read_uint(fd, &val)) return 1; + block_size = (uint32_t)val; + break; + case LBD_CBOR_KEY_HDR_SEGMENT_LABEL: + if (cbor_read_text(fd, segment_label, + sizeof(segment_label), NULL)) + return 1; + break; + case LBD_CBOR_KEY_HDR_DEVICE_SIZE: + if (cbor_read_uint(fd, &val)) return 1; + device_size = val; + break; + case LBD_CBOR_KEY_HDR_BACKING_PATH: + if (cbor_read_text(fd, backing_path, sizeof(backing_path), NULL)) + return 1; + break; + default: + if (cbor_skip(fd)) return 1; + break; + } + } + + if (json) { + printf("{\n"); + printf(" \"header\": {\n"); + printf(" \"version\": %u,\n", version); + printf(" \"block_size\": %u,\n", block_size); + printf(" \"segment_label\": "); + json_print_string(stdout, segment_label); + printf(",\n"); + printf(" \"device_size\": %llu,\n", + (unsigned long long)device_size); + printf(" \"backing_path\": "); + json_print_string(stdout, backing_path); + printf("\n },\n"); + printf(" \"entries\": [\n"); + } else { + printf("=== LBD Log: %s ===\n", path); + printf("Version: %u\n", version); + printf("Block size: %u\n", block_size); + printf("Segment: %s\n", segment_label); + printf("Device size: %s (%llu bytes)\n", + fmt_size(device_size, sizebuf, sizeof(sizebuf)), + (unsigned long long)device_size); + printf("Backing file: %s\n", backing_path); + printf("\n"); + } + + /* Read entry maps until EOF */ + while (1) { + uint8_t peek_major; + uint64_t entry_map_count; + int ret; + + /* Try to read next map header; EOF is normal end */ + ret = cbor_read_head(fd, &peek_major, &entry_map_count); + if (ret == -1) + break; /* clean EOF */ + if (ret < 0) { + fprintf(stderr, "Error reading entry %d\n", entry_count); + break; + } + if (peek_major != 5) { + fprintf(stderr, "Expected CBOR map at entry %d, got major %u\n", + entry_count, peek_major); + break; + } + + /* Parse entry fields */ + char op = 0; + uint64_t timestamp_ns = 0, sequence = 0, block = 0; + uint32_t length = 0, checksum = 0; + int has_checksum = 0, has_data = 0; + uint64_t data_len = 0; + uint64_t comp_size = 0; /* compressed size from CBOR */ + + for (uint64_t i = 0; i < entry_map_count; i++) { + if (cbor_read_uint(fd, &key)) { + fprintf(stderr, "Failed to read entry key at entry %d\n", + entry_count); + goto done; + } + switch (key) { + case LBD_CBOR_KEY_OP: { + char tbuf[4]; + if (cbor_read_text(fd, tbuf, sizeof(tbuf), NULL)) + goto done; + op = tbuf[0]; + break; + } + case LBD_CBOR_KEY_TIMESTAMP: + if (cbor_read_uint(fd, ×tamp_ns)) goto done; + break; + case LBD_CBOR_KEY_SEQUENCE: + if (cbor_read_uint(fd, &sequence)) goto done; + break; + case LBD_CBOR_KEY_BLOCK: + if (cbor_read_uint(fd, &block)) goto done; + break; + case LBD_CBOR_KEY_LENGTH: + if (cbor_read_uint(fd, &val)) goto done; + length = (uint32_t)val; + break; + case LBD_CBOR_KEY_CHECKSUM: + if (cbor_read_uint(fd, &val)) goto done; + checksum = (uint32_t)val; + has_checksum = 1; + break; + case LBD_CBOR_KEY_DATA: + if (cbor_read_bytes_hdr(fd, &data_len)) goto done; + has_data = 1; + if (data_len > data_cap) { + free(data); + data_cap = (size_t)data_len; + data = malloc(data_cap); + if (!data) { + fprintf(stderr, + "Out of memory for %llu byte payload\n", + (unsigned long long)data_len); + goto done; + } + } + if (data_len > 0 && + read_exact(fd, data, (size_t)data_len) <= 0) { + fprintf(stderr, + "Truncated data at entry %d\n", + entry_count); + goto done; + } + break; + default: + if (cbor_skip(fd)) goto done; + break; + } + } + + int is_trim = (op == 'T'); + + /* Decompress LZ4 data */ + comp_size = data_len; + if (has_data && length > 0 && data_len > 0) { + uint8_t *decompressed = malloc(length); + if (!decompressed) { + fprintf(stderr, + "Out of memory for %u byte decompression buffer\n", + length); + goto done; + } + int dec_len = LZ4_decompress_safe( + (const char *)data, (char *)decompressed, + (int)data_len, (int)length); + if (dec_len < 0) { + fprintf(stderr, + "LZ4 decompression failed at entry %d\n", + entry_count); + free(decompressed); + goto done; + } + /* Replace compressed data with decompressed */ + if ((size_t)length > data_cap) { + free(data); + data_cap = length; + data = malloc(data_cap); + if (!data) { + free(decompressed); + data = NULL; + data_cap = 0; + goto done; + } + } + memcpy(data, decompressed, dec_len); + data_len = dec_len; + free(decompressed); + } + + /* Validate CRC on decompressed data */ + uint32_t computed_crc = 0; + int crc_ok = 1; + if (has_data && has_checksum) { + computed_crc = crc32_calc(data, (size_t)data_len); + crc_ok = (computed_crc == checksum); + } + + if (json) { + if (entry_count > 0) + printf(",\n"); + printf(" {\n"); + printf(" \"type\": \"%s\",\n", + is_trim ? "trim" : "write"); + printf(" \"sequence\": %llu,\n", + (unsigned long long)sequence); + printf(" \"timestamp_ns\": %llu,\n", + (unsigned long long)timestamp_ns); + + time_t secs = timestamp_ns / 1000000000ULL; + unsigned long ns = timestamp_ns % 1000000000ULL; + struct tm tm; + gmtime_r(&secs, &tm); + char tbuf[64]; + strftime(tbuf, sizeof(tbuf), "%Y-%m-%dT%H:%M:%S", &tm); + printf(" \"timestamp\": \"%s.%09luZ\",\n", tbuf, ns); + + printf(" \"block\": %llu,\n", + (unsigned long long)block); + printf(" \"block_end\": %llu,\n", + (unsigned long long)(block + length / block_size - 1)); + printf(" \"extent\": \"%llu-%llu\",\n", + (unsigned long long)block, + (unsigned long long)(block + length / block_size - 1)); + printf(" \"offset_bytes\": %llu,\n", + (unsigned long long)block * block_size); + printf(" \"length\": %u", length); + if (has_data && comp_size > 0) { + printf(",\n \"compressed_size\": %llu", + (unsigned long long)comp_size); + printf(",\n \"compression_ratio\": %.1f", + length > 0 ? (1.0 - (double)comp_size / length) * 100.0 : 0.0); + } + if (has_checksum) { + printf(",\n \"checksum\": \"0x%08x\",\n", + checksum); + printf(" \"checksum_valid\": %s", + crc_ok ? "true" : "false"); + } + if (show_data && has_data) { + printf(",\n \"data_hex\": \""); + for (uint64_t i = 0; i < data_len; i++) + printf("%02x", data[i]); + printf("\""); + } + printf("\n }"); + } else { + time_t secs = timestamp_ns / 1000000000ULL; + unsigned long ns = timestamp_ns % 1000000000ULL; + struct tm tm; + gmtime_r(&secs, &tm); + char tbuf[64]; + strftime(tbuf, sizeof(tbuf), "%Y-%m-%d %H:%M:%S", &tm); + + printf("--- Entry #%llu [%s] ---\n", + (unsigned long long)sequence, + is_trim ? "TRIM" : "WRITE"); + printf(" Time: %s.%09lu UTC\n", tbuf, ns); + printf(" Extent: %llu-%llu (%u blocks)\n", + (unsigned long long)block, + (unsigned long long)(block + length / block_size - 1), + length / block_size); + printf(" Offset: 0x%llx-0x%llx\n", + (unsigned long long)block * block_size, + (unsigned long long)(block * block_size + length - 1)); + printf(" Length: %s (%u bytes)\n", + fmt_size(length, sizebuf, sizeof(sizebuf)), + length); + if (has_data && comp_size > 0) { + printf(" LZ4: %s compressed (%.1f%% reduction)\n", + fmt_size(comp_size, sizebuf, sizeof(sizebuf)), + length > 0 ? (1.0 - (double)comp_size / length) * 100.0 : 0.0); + } + if (has_checksum) { + printf(" CRC32: 0x%08x %s\n", checksum, + crc_ok ? "(OK)" : "(MISMATCH - computed 0x%08x)"); + if (!crc_ok) + printf(" computed: 0x%08x\n", + computed_crc); + } + if (show_data && has_data) { + printf(" Data:\n"); + hex_dump(stdout, data, (size_t)data_len, " "); + } + printf("\n"); + } + + entry_count++; + } + +done: + if (json) { + printf("\n ],\n"); + printf(" \"entry_count\": %d\n", entry_count); + printf("}\n"); + } else { + printf("Total entries: %d\n", entry_count); + } + + free(data); + return 0; +} + +static int cmd_log(const char *path, int json, int show_data) +{ + int fd, ret; + + fd = open(path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open log file '%s': %s\n", + path, strerror(errno)); + return 1; + } + + ret = cmd_log_cbor(fd, path, json, show_data); + close(fd); + return ret; +} + +/* ---------------------------------------------------------------- + * qcow2-lz4 format support + * ---------------------------------------------------------------- */ + +#include "lbd_qcow2_format.h" + +static inline uint64_t htobe64_val(uint64_t x) +{ + uint8_t buf[8]; + _qcow2_put64(buf, 0, x); + uint64_t v; + memcpy(&v, buf, 8); + return v; +} + +static inline uint64_t be64toh_val(uint64_t x) +{ + uint8_t buf[8]; + memcpy(buf, &x, 8); + return _qcow2_get64(buf, 0); +} + +static inline uint32_t htobe32_val(uint32_t x) +{ + uint8_t buf[4]; + _qcow2_put32(buf, 0, x); + uint32_t v; + memcpy(&v, buf, 4); + return v; +} + +static inline uint32_t be32toh_val(uint32_t x) +{ + uint8_t buf[4]; + memcpy(buf, &x, 4); + return _qcow2_get32(buf, 0); +} + +static int is_all_zero(const void *buf, size_t len) +{ + const uint8_t *p = buf; + for (size_t i = 0; i < len; i++) + if (p[i] != 0) + return 0; + return 1; +} + +static uint64_t parse_size(const char *s) +{ + char *end; + uint64_t val = strtoull(s, &end, 0); + + switch (*end) { + case 'k': case 'K': val *= 1024; break; + case 'm': case 'M': val *= 1024 * 1024; break; + case 'g': case 'G': val *= 1024ULL * 1024 * 1024; break; + case 't': case 'T': val *= 1024ULL * 1024 * 1024 * 1024; break; + } + return val; +} + +/* + * lbdctl create --size [--cluster-bits 16] + * Create an empty qcow2-lz4 image file. + */ +static int cmd_create(int argc, char **argv) +{ + const char *path = NULL; + uint64_t virtual_size = 0; + uint32_t cluster_bits = 16; + uint32_t cluster_size, l2_entries, l1_size; + uint64_t l1_table_offset, alloc_offset; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + int fd; + ssize_t n; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--size") == 0) { + if (++i >= argc) { + fprintf(stderr, "--size requires a value\n"); + return 1; + } + virtual_size = parse_size(argv[i]); + } else if (strcmp(argv[i], "--cluster-bits") == 0) { + if (++i >= argc) { + fprintf(stderr, "--cluster-bits requires a value\n"); + return 1; + } + cluster_bits = atoi(argv[i]); + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "create requires an output path\n"); + return 1; + } + + if (virtual_size == 0) { + fprintf(stderr, "create requires --size \n"); + return 1; + } + + if (cluster_bits < 12 || cluster_bits > 24) { + fprintf(stderr, "cluster_bits must be between 12 and 24\n"); + return 1; + } + + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + + /* Compute L1 size: one entry per L2 table needed */ + l1_size = (virtual_size + (uint64_t)l2_entries * cluster_size - 1) / + ((uint64_t)l2_entries * cluster_size); + if (l1_size == 0) + l1_size = 1; + + l1_table_offset = LBD_QCOW2_HEADER_SIZE; + alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + /* Align alloc_offset to cluster boundary */ + alloc_offset = (alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Build header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", path, strerror(errno)); + return 1; + } + + /* Write header */ + n = write(fd, hdr, LBD_QCOW2_HEADER_SIZE); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to write header: %s\n", strerror(errno)); + close(fd); + return 1; + } + + /* Write zeroed L1 table */ + { + size_t l1_bytes = l1_size * sizeof(uint64_t); + void *zeros = calloc(1, l1_bytes); + if (!zeros) { + fprintf(stderr, "Out of memory\n"); + close(fd); + return 1; + } + n = write(fd, zeros, l1_bytes); + free(zeros); + if (n != (ssize_t)l1_bytes) { + fprintf(stderr, "Failed to write L1 table: %s\n", + strerror(errno)); + close(fd); + return 1; + } + } + + /* Extend file to alloc_offset */ + if (ftruncate(fd, alloc_offset) < 0) { + fprintf(stderr, "Failed to extend file: %s\n", strerror(errno)); + close(fd); + return 1; + } + + close(fd); + + printf("Created qcow2-lz4 image: %s\n", path); + printf(" Virtual size: %llu bytes (%s)\n", + (unsigned long long)virtual_size, + fmt_size(virtual_size, (char[32]){0}, 32)); + printf(" Cluster size: %u bytes (bits=%u)\n", cluster_size, cluster_bits); + printf(" L1 entries: %u\n", l1_size); + printf(" L2 entries: %u per table\n", l2_entries); + printf(" Alloc offset: %llu\n", (unsigned long long)alloc_offset); + + return 0; +} + +/* + * lbdctl convert + * Convert a flat (raw) image file to qcow2-lz4 format. + */ +static int cmd_convert(const char *flat_path, const char *qcow2_path) +{ + int in_fd, out_fd; + struct stat st; + uint64_t virtual_size; + uint32_t cluster_bits = 16; + uint32_t cluster_size = 1U << cluster_bits; + uint32_t l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + uint32_t l1_size; + uint64_t l1_table_offset, alloc_offset; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t *l1_table; + uint64_t **l2_tables; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + uint64_t total_clusters, clusters_written = 0, clusters_zero = 0; + ssize_t n; + + in_fd = open(flat_path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", flat_path, strerror(errno)); + return 1; + } + + if (fstat(in_fd, &st) < 0) { + fprintf(stderr, "Cannot stat '%s': %s\n", flat_path, strerror(errno)); + close(in_fd); + return 1; + } + + virtual_size = st.st_size; + if (virtual_size == 0) { + fprintf(stderr, "Input file is empty\n"); + close(in_fd); + return 1; + } + + total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + l1_size = (virtual_size + (uint64_t)l2_entries * cluster_size - 1) / + ((uint64_t)l2_entries * cluster_size); + if (l1_size == 0) + l1_size = 1; + + l1_table_offset = LBD_QCOW2_HEADER_SIZE; + alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + alloc_offset = (alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Allocate tables */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + l2_tables = calloc(l1_size, sizeof(uint64_t *)); + cluster_buf = malloc(cluster_size); + comp_cap = LZ4_compressBound(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !l2_tables || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + for (uint32_t i = 0; i < l1_size; i++) { + l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + } + + out_fd = open(qcow2_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", qcow2_path, strerror(errno)); + close(in_fd); + return 1; + } + + /* Reserve space for header + L1 table (written at end) */ + if (ftruncate(out_fd, alloc_offset) < 0) { + fprintf(stderr, "Failed to extend output: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + if (lseek(out_fd, alloc_offset, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Process each cluster */ + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + size_t to_read = cluster_size; + ssize_t rd; + + /* Handle last partial cluster */ + if ((ci + 1) * cluster_size > virtual_size) + to_read = virtual_size - ci * cluster_size; + + memset(cluster_buf, 0, cluster_size); + rd = pread(in_fd, cluster_buf, to_read, ci * cluster_size); + if (rd < 0) { + fprintf(stderr, "Read error at cluster %llu: %s\n", + (unsigned long long)ci, strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Skip all-zero clusters (leave L2 = 0 for sparse) */ + if (is_all_zero(cluster_buf, cluster_size)) { + clusters_zero++; + continue; + } + + /* Allocate L2 table if needed */ + if (l1_table[l1_idx] == 0) { + l1_table[l1_idx] = alloc_offset; + alloc_offset += cluster_size; + } + + /* Compress */ + int comp_len = LZ4_compress_default( + (const char *)cluster_buf, (char *)comp_buf, + cluster_size, comp_cap); + + if (comp_len > 0 && + (uint32_t)comp_len < cluster_size - sizeof(uint32_t)) { + /* Store compressed */ + uint32_t total_on_disk = ((sizeof(uint32_t) + comp_len) + 4095) & ~4095U; + uint64_t phys = alloc_offset; + uint32_t comp_size_be = htobe32_val(comp_len); + + /* Write size header + compressed data */ + if (pwrite(out_fd, &comp_size_be, sizeof(comp_size_be), phys) != + sizeof(comp_size_be)) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + if (pwrite(out_fd, comp_buf, comp_len, + phys + sizeof(comp_size_be)) != comp_len) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + + l2_tables[l1_idx][l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + alloc_offset += total_on_disk; + } else { + /* Store uncompressed */ + uint64_t phys = alloc_offset; + + if (pwrite(out_fd, cluster_buf, cluster_size, phys) != + cluster_size) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + + l2_tables[l1_idx][l2_idx] = phys; + alloc_offset += cluster_size; + } + + clusters_written++; + } + + /* Write L2 tables to their allocated positions */ + for (uint32_t i = 0; i < l1_size; i++) { + if (l1_table[i] == 0) + continue; + + /* Convert L2 entries to big-endian on disk */ + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + + memset(disk_l2, 0, cluster_size); + for (uint32_t j = 0; j < l2_entries; j++) + disk_l2[j] = htobe64_val(l2_tables[i][j]); + + /* Compute and store CRC32C trailer */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t crc = crc32c_calc(raw, cluster_size - 4); + _qcow2_put32(raw, cluster_size - 4, crc); + } + + n = pwrite(out_fd, disk_l2, cluster_size, l1_table[i]); + free(disk_l2); + if (n != cluster_size) { + fprintf(stderr, "Failed to write L2 table %u\n", i); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Write header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + n = pwrite(out_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to write header\n"); + close(in_fd); + close(out_fd); + return 1; + } + + /* Write L1 table (big-endian) */ + { + uint64_t *disk_l1 = calloc(l1_size, sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + disk_l1[i] = htobe64_val(l1_table[i]); + + n = pwrite(out_fd, disk_l1, l1_size * sizeof(uint64_t), + l1_table_offset); + free(disk_l1); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to write L1 table\n"); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Truncate file to final size */ + if (ftruncate(out_fd, alloc_offset) < 0) + fprintf(stderr, "Warning: ftruncate failed: %s\n", strerror(errno)); + + close(in_fd); + close(out_fd); + + /* Cleanup */ + for (uint32_t i = 0; i < l1_size; i++) + free(l2_tables[i]); + free(l2_tables); + free(l1_table); + free(cluster_buf); + free(comp_buf); + + printf("Converted %s -> %s\n", flat_path, qcow2_path); + printf(" Virtual size: %llu bytes\n", (unsigned long long)virtual_size); + printf(" Total clusters: %llu\n", (unsigned long long)total_clusters); + printf(" Written: %llu (data)\n", (unsigned long long)clusters_written); + printf(" Zero (sparse): %llu\n", (unsigned long long)clusters_zero); + + { + struct stat out_st; + if (stat(qcow2_path, &out_st) == 0) { + printf(" File size: %llu bytes (%s)\n", + (unsigned long long)out_st.st_size, + fmt_size(out_st.st_size, (char[32]){0}, 32)); + if (virtual_size > 0) + printf(" Compression: %.1f%%\n", + (1.0 - (double)out_st.st_size / virtual_size) * 100.0); + } + } + + return 0; +} + +/* + * lbdctl extract + * Decompress all clusters from a qcow2-lz4 image to a flat file. + */ +static int cmd_extract(const char *qcow2_path, const char *flat_path) +{ + int in_fd, out_fd; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t virtual_size, l1_table_offset, alloc_off; + uint32_t cluster_bits, cluster_size, l2_entries, l1_size; + uint64_t *l1_table; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + ssize_t n; + + in_fd = open(qcow2_path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", qcow2_path, strerror(errno)); + return 1; + } + + /* Read header */ + n = pread(in_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to read header\n"); + close(in_fd); + return 1; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + fprintf(stderr, "Not a qcow2-lz4 image\n"); + close(in_fd); + return 1; + } + + cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + l1_table_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + l1_size = lbd_qcow2_hdr_l1_size(hdr); + alloc_off = lbd_qcow2_hdr_alloc_offset(hdr); + (void)alloc_off; + + /* Read L1 table */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + comp_cap = LZ4_compressBound(cluster_size); + cluster_buf = malloc(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + { + uint64_t *disk_l1 = malloc(l1_size * sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to read L1 table\n"); + free(disk_l1); + close(in_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + l1_table[i] = be64toh_val(disk_l1[i]); + free(disk_l1); + } + + /* Create output file */ + out_fd = open(flat_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", flat_path, strerror(errno)); + close(in_fd); + return 1; + } + + /* Pre-allocate the output */ + if (ftruncate(out_fd, virtual_size) < 0) { + fprintf(stderr, "Failed to extend output: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Read and verify all L2 tables */ + uint64_t **ext_l2_tables = calloc(l1_size, sizeof(uint64_t *)); + if (!ext_l2_tables) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) { + ext_l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!ext_l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + if (l1_table[i] == 0) + continue; + + uint8_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + n = pread(in_fd, disk_l2, cluster_size, l1_table[i]); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read L2 table %u\n", i); + free(disk_l2); + close(in_fd); + close(out_fd); + return 1; + } + + /* Verify CRC32C */ + { + uint32_t stored_crc = _qcow2_get32(disk_l2, cluster_size - 4); + uint32_t calc_crc = crc32c_calc(disk_l2, cluster_size - 4); + if (stored_crc != calc_crc) { + fprintf(stderr, "L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + i, stored_crc, calc_crc); + free(disk_l2); + close(in_fd); + close(out_fd); + return 1; + } + } + + for (uint32_t j = 0; j < l2_entries; j++) + ext_l2_tables[i][j] = be64toh_val(((uint64_t *)disk_l2)[j]); + free(disk_l2); + } + + /* Extract each cluster */ + uint64_t total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + uint64_t l2_entry; + uint64_t phys_offset; + size_t write_len = cluster_size; + + if (l1_idx >= l1_size || l1_table[l1_idx] == 0) { + /* Unallocated L2 table: output zeros (already zero from ftruncate) */ + continue; + } + + l2_entry = ext_l2_tables[l1_idx][l2_idx]; + + if (l2_entry == 0) { + /* Unallocated cluster: zeros */ + continue; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + /* Handle last partial cluster */ + if ((ci + 1) * cluster_size > virtual_size) + write_len = virtual_size - ci * cluster_size; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + /* Compressed cluster */ + uint32_t comp_size_be, comp_size; + + n = pread(in_fd, &comp_size_be, sizeof(comp_size_be), + phys_offset); + if (n != sizeof(comp_size_be)) { + fprintf(stderr, "Failed to read compressed size at cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + + comp_size = be32toh_val(comp_size_be); + + n = pread(in_fd, comp_buf, comp_size, + phys_offset + sizeof(comp_size_be)); + if (n != (ssize_t)comp_size) { + fprintf(stderr, "Failed to read compressed data at cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + + int dec_len = LZ4_decompress_safe( + (const char *)comp_buf, (char *)cluster_buf, + comp_size, cluster_size); + if (dec_len != (int)cluster_size) { + fprintf(stderr, "LZ4 decompression failed at cluster %llu (got %d, expected %u)\n", + (unsigned long long)ci, dec_len, cluster_size); + close(in_fd); + close(out_fd); + return 1; + } + } else { + /* Uncompressed cluster */ + n = pread(in_fd, cluster_buf, cluster_size, phys_offset); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + } + + n = pwrite(out_fd, cluster_buf, write_len, ci * cluster_size); + if (n != (ssize_t)write_len) { + fprintf(stderr, "Failed to write cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + } + + close(in_fd); + close(out_fd); + for (uint32_t i = 0; i < l1_size; i++) + free(ext_l2_tables[i]); + free(ext_l2_tables); + free(l1_table); + free(cluster_buf); + free(comp_buf); + + printf("Extracted %s -> %s (%llu bytes)\n", + qcow2_path, flat_path, (unsigned long long)virtual_size); + + return 0; +} + +/* + * lbdctl compact [--inplace] + * Rewrite the image, eliminating dead space from overwrites. + */ +static int cmd_compact(int argc, char **argv) +{ + const char *path = NULL; + int in_fd, out_fd; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t virtual_size, l1_table_offset; + uint32_t cluster_bits, cluster_size, l2_entries, l1_size; + uint64_t *l1_table; + uint64_t **l2_tables; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + uint64_t new_alloc_offset; + ssize_t n; + char tmp_path[PATH_MAX]; + + for (int i = 0; i < argc; i++) { + if (!path) + path = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "compact requires a qcow2 file path\n"); + return 1; + } + + in_fd = open(path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", path, strerror(errno)); + return 1; + } + + /* Read header */ + n = pread(in_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to read header\n"); + close(in_fd); + return 1; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + fprintf(stderr, "Not a qcow2-lz4 image\n"); + close(in_fd); + return 1; + } + + cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + l1_table_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + l1_size = lbd_qcow2_hdr_l1_size(hdr); + + /* Read L1 */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + l2_tables = calloc(l1_size, sizeof(uint64_t *)); + cluster_buf = malloc(cluster_size); + comp_cap = LZ4_compressBound(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !l2_tables || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + { + uint64_t *disk_l1 = malloc(l1_size * sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to read L1 table\n"); + free(disk_l1); + close(in_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + l1_table[i] = be64toh_val(disk_l1[i]); + free(disk_l1); + } + + /* Read all L2 tables */ + for (uint32_t i = 0; i < l1_size; i++) { + l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + if (l1_table[i] == 0) + continue; + + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l2, cluster_size, l1_table[i]); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read L2 table %u\n", i); + free(disk_l2); + close(in_fd); + return 1; + } + + /* Verify CRC32C */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t stored_crc = _qcow2_get32(raw, cluster_size - 4); + uint32_t calc_crc = crc32c_calc(raw, cluster_size - 4); + if (stored_crc != calc_crc) { + fprintf(stderr, "L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + i, stored_crc, calc_crc); + free(disk_l2); + close(in_fd); + return 1; + } + } + + for (uint32_t j = 0; j < l2_entries; j++) + l2_tables[i][j] = be64toh_val(disk_l2[j]); + free(disk_l2); + } + + /* Create temp output file */ + snprintf(tmp_path, sizeof(tmp_path), "%s.compact.tmp", path); + out_fd = open(tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create temp file: %s\n", strerror(errno)); + close(in_fd); + return 1; + } + + /* New layout: header + L1 table, then data */ + new_alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + new_alloc_offset = (new_alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Reserve space for header + L1 */ + if (ftruncate(out_fd, new_alloc_offset) < 0) { + fprintf(stderr, "Failed to extend temp file\n"); + close(in_fd); + close(out_fd); + return 1; + } + + /* New L1 and L2 tables */ + uint64_t *new_l1 = calloc(l1_size, sizeof(uint64_t)); + uint64_t **new_l2 = calloc(l1_size, sizeof(uint64_t *)); + if (!new_l1 || !new_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) { + new_l2[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!new_l2[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Walk all clusters and re-pack */ + uint64_t total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + uint64_t l2_entry; + uint64_t phys_offset; + + if (l1_idx >= l1_size || l1_table[l1_idx] == 0) + continue; + + l2_entry = l2_tables[l1_idx][l2_idx]; + if (l2_entry == 0) + continue; + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + /* Read cluster data (decompress if needed) */ + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + uint32_t comp_size_be, comp_size; + + n = pread(in_fd, &comp_size_be, sizeof(comp_size_be), phys_offset); + if (n != sizeof(comp_size_be)) { + fprintf(stderr, "Read error\n"); + goto compact_err; + } + comp_size = be32toh_val(comp_size_be); + + n = pread(in_fd, comp_buf, comp_size, + phys_offset + sizeof(comp_size_be)); + if (n != (ssize_t)comp_size) { + fprintf(stderr, "Read error\n"); + goto compact_err; + } + + int dec_len = LZ4_decompress_safe( + (const char *)comp_buf, (char *)cluster_buf, + comp_size, cluster_size); + if (dec_len != (int)cluster_size) { + fprintf(stderr, "Decompression failed at cluster %llu\n", + (unsigned long long)ci); + goto compact_err; + } + } else { + n = pread(in_fd, cluster_buf, cluster_size, phys_offset); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Read error at cluster %llu\n", + (unsigned long long)ci); + goto compact_err; + } + } + + /* Skip zero clusters */ + if (is_all_zero(cluster_buf, cluster_size)) + continue; + + /* Allocate L2 table for new image if needed */ + if (new_l1[l1_idx] == 0) { + new_l1[l1_idx] = new_alloc_offset; + new_alloc_offset += cluster_size; + } + + /* Recompress and write */ + int comp_len = LZ4_compress_default( + (const char *)cluster_buf, (char *)comp_buf, + cluster_size, comp_cap); + + if (comp_len > 0 && + (uint32_t)comp_len < cluster_size - sizeof(uint32_t)) { + uint32_t total_on_disk = ((sizeof(uint32_t) + comp_len) + 4095) & ~4095U; + uint64_t phys = new_alloc_offset; + uint32_t comp_size_be = htobe32_val(comp_len); + + if (pwrite(out_fd, &comp_size_be, sizeof(comp_size_be), phys) != + sizeof(comp_size_be)) + goto compact_err; + if (pwrite(out_fd, comp_buf, comp_len, + phys + sizeof(comp_size_be)) != comp_len) + goto compact_err; + + new_l2[l1_idx][l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + new_alloc_offset += total_on_disk; + } else { + uint64_t phys = new_alloc_offset; + + if (pwrite(out_fd, cluster_buf, cluster_size, phys) != + (ssize_t)cluster_size) + goto compact_err; + + new_l2[l1_idx][l2_idx] = phys; + new_alloc_offset += cluster_size; + } + } + + /* Write L2 tables */ + for (uint32_t i = 0; i < l1_size; i++) { + if (new_l1[i] == 0) + continue; + + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) + goto compact_err; + + memset(disk_l2, 0, cluster_size); + for (uint32_t j = 0; j < l2_entries; j++) + disk_l2[j] = htobe64_val(new_l2[i][j]); + + /* Compute and store CRC32C trailer */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t crc = crc32c_calc(raw, cluster_size - 4); + _qcow2_put32(raw, cluster_size - 4, crc); + } + + n = pwrite(out_fd, disk_l2, cluster_size, new_l1[i]); + free(disk_l2); + if (n != (ssize_t)cluster_size) + goto compact_err; + } + + /* Write header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, new_alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + if (pwrite(out_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0) != LBD_QCOW2_HEADER_SIZE) + goto compact_err; + + /* Write L1 table */ + { + uint64_t *disk_l1 = calloc(l1_size, sizeof(uint64_t)); + if (!disk_l1) + goto compact_err; + for (uint32_t i = 0; i < l1_size; i++) + disk_l1[i] = htobe64_val(new_l1[i]); + n = pwrite(out_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + free(disk_l1); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) + goto compact_err; + } + + if (ftruncate(out_fd, new_alloc_offset) < 0) + fprintf(stderr, "Warning: ftruncate failed\n"); + + close(in_fd); + close(out_fd); + + /* Replace original with compacted */ + if (rename(tmp_path, path) < 0) { + fprintf(stderr, "Failed to rename %s -> %s: %s\n", + tmp_path, path, strerror(errno)); + return 1; + } + + printf("Compacted %s\n", path); + { + struct stat st; + if (stat(path, &st) == 0) { + printf(" File size: %llu bytes (%s)\n", + (unsigned long long)st.st_size, + fmt_size(st.st_size, (char[32]){0}, 32)); + } + } + + /* Cleanup */ + for (uint32_t i = 0; i < l1_size; i++) { + free(l2_tables[i]); + free(new_l2[i]); + } + free(l2_tables); + free(new_l2); + free(l1_table); + free(new_l1); + free(cluster_buf); + free(comp_buf); + return 0; + +compact_err: + fprintf(stderr, "Compaction failed\n"); + close(in_fd); + close(out_fd); + unlink(tmp_path); + for (uint32_t i = 0; i < l1_size; i++) { + free(l2_tables[i]); + free(new_l2[i]); + } + free(l2_tables); + free(new_l2); + free(l1_table); + free(new_l1); + free(cluster_buf); + free(comp_buf); + return 1; +} + +/* ---------------------------------------------------------------- + * Watch command (log rotation notifications) + * ---------------------------------------------------------------- */ + +static volatile sig_atomic_t watch_running = 1; + +static void watch_sigint(int sig) +{ + (void)sig; + watch_running = 0; +} + +/* + * Minimal CBOR encoder for userspace (write path). + * Encodes a CBOR head: major type (top 3 bits) + value. + */ +static size_t cbor_write_head(uint8_t *buf, uint8_t major, uint64_t val) +{ + uint8_t mt = major << 5; + + if (val < 24) { + buf[0] = mt | (uint8_t)val; + return 1; + } else if (val <= 0xFF) { + buf[0] = mt | 24; + buf[1] = (uint8_t)val; + return 2; + } else if (val <= 0xFFFF) { + buf[0] = mt | 25; + buf[1] = (uint8_t)(val >> 8); + buf[2] = (uint8_t)val; + return 3; + } else if (val <= 0xFFFFFFFF) { + buf[0] = mt | 26; + buf[1] = (uint8_t)(val >> 24); + buf[2] = (uint8_t)(val >> 16); + buf[3] = (uint8_t)(val >> 8); + buf[4] = (uint8_t)val; + return 5; + } else { + buf[0] = mt | 27; + buf[1] = (uint8_t)(val >> 56); + buf[2] = (uint8_t)(val >> 48); + buf[3] = (uint8_t)(val >> 40); + buf[4] = (uint8_t)(val >> 32); + buf[5] = (uint8_t)(val >> 24); + buf[6] = (uint8_t)(val >> 16); + buf[7] = (uint8_t)(val >> 8); + buf[8] = (uint8_t)val; + return 9; + } +} + +/* + * Encode and write a watch command to the control device fd. + * dev_index < 0 means watch all devices. + */ +static int cbor_write_watch_cmd(int fd, int dev_index) +{ + uint8_t buf[64]; + size_t pos = 0; + const char *cmd_str = "watch"; + size_t cmd_len = 5; + int map_items = (dev_index >= 0) ? 2 : 1; + ssize_t n; + + /* map(1 or 2) */ + pos += cbor_write_head(buf + pos, 5, map_items); + + /* key 1: "watch" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 2: dev_index (optional) */ + if (dev_index >= 0) { + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_DEV); + pos += cbor_write_head(buf + pos, 0, (uint64_t)dev_index); + } + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write watch command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on watch command\n"); + return -1; + } + return 0; +} + +/* + * Decode a CBOR-encoded event message from a buffer. + * Uses the existing fd-based cbor_read_* functions via a temporary + * approach: we decode inline from the buffer. + */ +static int decode_watch_event(const uint8_t *buf, size_t len, + int *dev_index, char *label, size_t label_cap, + char *dir, size_t dir_cap, + uint64_t *log_seq, uint64_t *device_size) +{ + size_t pos = 0; + uint8_t ib, major, ai; + uint64_t map_count, key, val; + uint64_t slen; + + /* Inline buffer-based CBOR decoder (mirrors cbor_dec.h logic) */ +#define BUF_HEAD(major_out, val_out) do { \ + if (pos >= len) return -1; \ + ib = buf[pos++]; \ + *(major_out) = ib >> 5; \ + ai = ib & 0x1F; \ + if (ai < 24) { *(val_out) = ai; } \ + else if (ai == 24) { \ + if (pos + 1 > len) return -1; \ + *(val_out) = buf[pos++]; \ + } else if (ai == 25) { \ + if (pos + 2 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 8) | buf[pos+1]; \ + pos += 2; \ + } else if (ai == 26) { \ + if (pos + 4 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 24) | \ + ((uint64_t)buf[pos+1] << 16) | \ + ((uint64_t)buf[pos+2] << 8) | \ + buf[pos+3]; \ + pos += 4; \ + } else if (ai == 27) { \ + if (pos + 8 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 56) | \ + ((uint64_t)buf[pos+1] << 48) | \ + ((uint64_t)buf[pos+2] << 40) | \ + ((uint64_t)buf[pos+3] << 32) | \ + ((uint64_t)buf[pos+4] << 24) | \ + ((uint64_t)buf[pos+5] << 16) | \ + ((uint64_t)buf[pos+6] << 8) | \ + buf[pos+7]; \ + pos += 8; \ + } else { return -1; } \ +} while (0) + + /* Read map header */ + BUF_HEAD(&major, &map_count); + if (major != 5) + return -1; + + *dev_index = -1; + *log_seq = 0; + *device_size = 0; + label[0] = '\0'; + dir[0] = '\0'; + + for (uint64_t i = 0; i < map_count; i++) { + /* Read key (uint) */ + BUF_HEAD(&major, &key); + if (major != 0) + return -1; + + switch (key) { + case LBD_EVENT_KEY_TYPE: + /* text string — skip it */ + BUF_HEAD(&major, &slen); + if (major != 3 || pos + slen > len) + return -1; + pos += slen; + break; + case LBD_EVENT_KEY_DEV: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *dev_index = (int)val; + break; + case LBD_EVENT_KEY_LABEL: + BUF_HEAD(&major, &slen); + if (major != 3 || slen >= label_cap || pos + slen > len) + return -1; + memcpy(label, buf + pos, slen); + label[slen] = '\0'; + pos += slen; + break; + case LBD_EVENT_KEY_DIR: + BUF_HEAD(&major, &slen); + if (major != 3 || slen >= dir_cap || pos + slen > len) + return -1; + memcpy(dir, buf + pos, slen); + dir[slen] = '\0'; + pos += slen; + break; + case LBD_EVENT_KEY_SEQ: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *log_seq = val; + break; + case LBD_EVENT_KEY_SIZE: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *device_size = val; + break; + default: + return -1; + } + } + +#undef BUF_HEAD + return 0; +} + +static int cmd_watch(int argc, char **argv) +{ + int fd; + int dev_filter = -1; + int json = 0; + struct pollfd pfd; + struct sigaction sa; + uint8_t rbuf[512]; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_filter = atoi(argv[i]); + } else if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + /* Send watch command */ + if (cbor_write_watch_cmd(fd, dev_filter) < 0) { + close(fd); + return 1; + } + + if (!json) { + if (dev_filter >= 0) + fprintf(stderr, "Watching lbd%d for log rotations...\n", + dev_filter); + else + fprintf(stderr, "Watching all devices for log rotations...\n"); + } + + /* Setup SIGINT handler for clean exit */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = watch_sigint; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + pfd.fd = fd; + pfd.events = POLLIN; + + while (watch_running) { + int ret = poll(&pfd, 1, 1000); + + if (ret < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "poll error: %s\n", strerror(errno)); + break; + } + if (ret == 0) + continue; + + if (pfd.revents & POLLIN) { + ssize_t n = read(fd, rbuf, sizeof(rbuf)); + + if (n < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "read error: %s\n", + strerror(errno)); + break; + } + if (n == 0) + break; + + int dev_idx; + char label[32]; + char dir[LBD_LOG_PATH_MAX]; + uint64_t log_seq, device_size; + + if (decode_watch_event(rbuf, (size_t)n, + &dev_idx, label, sizeof(label), + dir, sizeof(dir), + &log_seq, &device_size) < 0) { + fprintf(stderr, "Failed to decode event\n"); + continue; + } + + if (json) { + printf("{\"type\":\"log_rotated\"," + "\"dev\":%d," + "\"segment_label\":\"%s\"," + "\"log_dir\":", + dev_idx, label); + json_print_string(stdout, dir); + printf(",\"log_seq\":%llu," + "\"device_size\":%llu}\n", + (unsigned long long)log_seq, + (unsigned long long)device_size); + } else { + printf("lbd%d: %s/disk.%s.log (seq=%llu)\n", + dev_idx, dir, label, + (unsigned long long)log_seq); + } + fflush(stdout); + } + + if (pfd.revents & (POLLERR | POLLHUP)) + break; + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Miss handler command + * ---------------------------------------------------------------- */ + +/* + * Encode and write a manage_misses command. + */ +static int cbor_write_manage_misses_cmd(int fd, int dev_index) +{ + uint8_t buf[64]; + size_t pos = 0; + const char *cmd_str = "manage_misses"; + size_t cmd_len = strlen(cmd_str); + ssize_t n; + + /* map(2) */ + pos += cbor_write_head(buf + pos, 5, 2); + + /* key 1: "manage_misses" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 2: dev_index */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_DEV); + pos += cbor_write_head(buf + pos, 0, (uint64_t)dev_index); + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write manage_misses command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on manage_misses command\n"); + return -1; + } + return 0; +} + +/* + * Encode and write a continue/retry command. + */ +static int cbor_write_miss_response(int fd, const char *action) +{ + uint8_t buf[64]; + size_t pos = 0; + size_t cmd_len = strlen(action); + ssize_t n; + + /* map(1) */ + pos += cbor_write_head(buf + pos, 5, 1); + + /* key 1: action */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, action, cmd_len); + pos += cmd_len; + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write %s command: %s\n", + action, strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on %s command\n", action); + return -1; + } + return 0; +} + +/* + * Decode a miss event from a CBOR buffer. + * Returns 0 on success, -1 on error. + */ +static int decode_miss_event(const uint8_t *buf, size_t len, + int *dev_index, uint64_t *cluster) +{ + size_t pos = 0; + uint8_t ib, major, ai; + uint64_t map_count, key, val; + uint64_t slen; + +#define BUF_HEAD2(major_out, val_out) do { \ + if (pos >= len) return -1; \ + ib = buf[pos++]; \ + *(major_out) = ib >> 5; \ + ai = ib & 0x1F; \ + if (ai < 24) { *(val_out) = ai; } \ + else if (ai == 24) { \ + if (pos + 1 > len) return -1; \ + *(val_out) = buf[pos++]; \ + } else if (ai == 25) { \ + if (pos + 2 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 8) | buf[pos+1]; \ + pos += 2; \ + } else if (ai == 26) { \ + if (pos + 4 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 24) | \ + ((uint64_t)buf[pos+1] << 16) | \ + ((uint64_t)buf[pos+2] << 8) | \ + buf[pos+3]; \ + pos += 4; \ + } else if (ai == 27) { \ + if (pos + 8 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 56) | \ + ((uint64_t)buf[pos+1] << 48) | \ + ((uint64_t)buf[pos+2] << 40) | \ + ((uint64_t)buf[pos+3] << 32) | \ + ((uint64_t)buf[pos+4] << 24) | \ + ((uint64_t)buf[pos+5] << 16) | \ + ((uint64_t)buf[pos+6] << 8) | \ + buf[pos+7]; \ + pos += 8; \ + } else { return -1; } \ +} while (0) + + BUF_HEAD2(&major, &map_count); + if (major != 5) + return -1; + + *dev_index = -1; + *cluster = 0; + + for (uint64_t i = 0; i < map_count; i++) { + BUF_HEAD2(&major, &key); + if (major != 0) + return -1; + + switch (key) { + case LBD_MISS_KEY_TYPE: + BUF_HEAD2(&major, &slen); + if (major != 3 || pos + slen > len) + return -1; + pos += slen; + break; + case LBD_MISS_KEY_DEV: + BUF_HEAD2(&major, &val); + if (major != 0) + return -1; + *dev_index = (int)val; + break; + case LBD_MISS_KEY_CLUSTER: + BUF_HEAD2(&major, &val); + if (major != 0) + return -1; + *cluster = val; + break; + default: + return -1; + } + } + +#undef BUF_HEAD2 + return 0; +} + +static int cmd_miss_handler(int argc, char **argv) +{ + int fd; + int dev_index = -1; + int json = 0; + struct pollfd pfd; + struct sigaction sa; + uint8_t rbuf[512]; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_index = atoi(argv[i]); + } else if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (dev_index < 0) { + fprintf(stderr, "miss-handler requires --dev N\n"); + return 1; + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + if (cbor_write_manage_misses_cmd(fd, dev_index) < 0) { + close(fd); + return 1; + } + + if (!json) + fprintf(stderr, "Handling block misses for lbd%d...\n", + dev_index); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = watch_sigint; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + pfd.fd = fd; + pfd.events = POLLIN; + + while (watch_running) { + int ret = poll(&pfd, 1, 1000); + + if (ret < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "poll error: %s\n", strerror(errno)); + break; + } + if (ret == 0) + continue; + + if (pfd.revents & POLLIN) { + ssize_t n = read(fd, rbuf, sizeof(rbuf)); + + if (n < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "read error: %s\n", + strerror(errno)); + break; + } + if (n == 0) + break; + + int miss_dev; + uint64_t miss_cluster; + + if (decode_miss_event(rbuf, (size_t)n, + &miss_dev, &miss_cluster) < 0) { + fprintf(stderr, "Failed to decode miss event\n"); + continue; + } + + if (json) { + printf("{\"type\":\"block_miss\"," + "\"dev\":%d," + "\"cluster\":%llu}\n", + miss_dev, + (unsigned long long)miss_cluster); + } else { + printf("lbd%d: block miss at cluster %llu\n", + miss_dev, + (unsigned long long)miss_cluster); + } + fflush(stdout); + + /* Respond with continue */ + if (cbor_write_miss_response(fd, "continue") < 0) + break; + } + + if (pfd.revents & (POLLERR | POLLHUP)) + break; + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Swap command + * ---------------------------------------------------------------- */ + +/* + * Encode and write a swap command with a path. + */ +static int cbor_write_swap_cmd(int fd, const char *path) +{ + uint8_t buf[512]; + size_t pos = 0; + const char *cmd_str = "swap"; + size_t cmd_len = strlen(cmd_str); + size_t path_len = strlen(path); + ssize_t n; + + /* map(2) */ + pos += cbor_write_head(buf + pos, 5, 2); + + /* key 1: "swap" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 3: path */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_PATH); + pos += cbor_write_head(buf + pos, 3, path_len); + if (pos + path_len > sizeof(buf)) { + fprintf(stderr, "Path too long for CBOR buffer\n"); + return -1; + } + memcpy(buf + pos, path, path_len); + pos += path_len; + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write swap command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on swap command\n"); + return -1; + } + return 0; +} + +static int cmd_swap(int argc, char **argv) +{ + int fd; + int dev_index = -1; + const char *path = NULL; + char resolved[PATH_MAX]; + int json = 0; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_index = atoi(argv[i]); + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (dev_index < 0) { + fprintf(stderr, "swap requires --dev N\n"); + return 1; + } + if (!path) { + fprintf(stderr, "swap requires a path argument\n"); + return 1; + } + + if (!realpath(path, resolved)) { + fprintf(stderr, "Cannot resolve path '%s': %s\n", + path, strerror(errno)); + return 1; + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + /* First register as miss handler for the device */ + if (cbor_write_manage_misses_cmd(fd, dev_index) < 0) { + close(fd); + return 1; + } + + /* Send swap command */ + if (cbor_write_swap_cmd(fd, resolved) < 0) { + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d, \"base\": ", + dev_index, dev_index); + json_print_string(stdout, resolved); + printf("}\n"); + } else { + printf("Swapped base layer for lbd%d to %s\n", dev_index, resolved); + } + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Main + * ---------------------------------------------------------------- */ + +static void usage(void) +{ + fprintf(stderr, + "Usage:\n" + " lbdctl add [opts] Create a new lbd device backed by \n" + " lbdctl remove Remove /dev/lbdN\n" + " lbdctl list List all active lbd devices\n" + " lbdctl watch [opts] Watch for log rotation events\n" + " lbdctl miss-handler [opts] Handle block miss events\n" + " lbdctl swap [opts] Swap base layer for a device\n" + " lbdctl log [opts] Read and display a log file\n" + " lbdctl create --size Create empty qcow2-lz4 image\n" + " lbdctl convert Convert flat image to qcow2-lz4\n" + " lbdctl extract Extract qcow2-lz4 to flat image\n" + " lbdctl compact Compact qcow2-lz4 image (reclaim space)\n" + "\n" + "Global options:\n" + " --json Output as JSON\n" + "\n" + "Add options:\n" + " --log-dir Directory to write log files (required)\n" + " --base Read-only base layer image (thin snapshot)\n" + " --log-max-size Segment rotation size (default 64 MiB)\n" + " --log-max-age Segment rotation age (default 60s)\n" + "\n" + "Watch options:\n" + " --dev Only watch device N (default: all)\n" + " --json Output events as JSON\n" + "\n" + "Miss handler options:\n" + " --dev Device to handle misses for (required)\n" + " --json Output events as JSON\n" + "\n" + "Swap options:\n" + " --dev Device to swap base for (required)\n" + "\n" + "Create options:\n" + " --size Virtual device size (supports K/M/G/T suffixes)\n" + " --cluster-bits log2(cluster_size), default 16 (64 KiB)\n" + "\n" + "Log options:\n" + " --json Output as JSON\n" + " --data Include hex data in output\n"); +} + +int main(int argc, char **argv) +{ + if (argc < 2) { + usage(); + return 1; + } + + if (strcmp(argv[1], "add") == 0) { + if (argc < 3) { + fprintf(stderr, "add requires a path argument\n"); + return 1; + } + return cmd_add(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "remove") == 0) { + return cmd_remove(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "list") == 0) { + return cmd_list(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "watch") == 0) { + return cmd_watch(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "miss-handler") == 0) { + return cmd_miss_handler(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "swap") == 0) { + return cmd_swap(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "create") == 0) { + if (argc < 3) { + fprintf(stderr, "create requires --size and a path\n"); + return 1; + } + return cmd_create(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "convert") == 0) { + if (argc < 4) { + fprintf(stderr, "convert requires \n"); + return 1; + } + return cmd_convert(argv[2], argv[3]); + } + + if (strcmp(argv[1], "extract") == 0) { + if (argc < 4) { + fprintf(stderr, "extract requires \n"); + return 1; + } + return cmd_extract(argv[2], argv[3]); + } + + if (strcmp(argv[1], "compact") == 0) { + if (argc < 3) { + fprintf(stderr, "compact requires a qcow2 file path\n"); + return 1; + } + return cmd_compact(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "log") == 0) { + int json = 0, show_data = 0; + const char *logpath = NULL; + + for (int i = 2; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else if (strcmp(argv[i], "--data") == 0) + show_data = 1; + else if (!logpath) + logpath = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", + argv[i]); + return 1; + } + } + + if (!logpath) { + fprintf(stderr, "log requires a log file path\n"); + return 1; + } + return cmd_log(logpath, json, show_data); + } + + fprintf(stderr, "Unknown command: %s\n", argv[1]); + usage(); + return 1; +} diff --git a/third_party/lbd/src/lz4/lz4.c b/third_party/lbd/src/lz4/lz4.c new file mode 100644 index 000000000..72b349f88 --- /dev/null +++ b/third_party/lbd/src/lz4/lz4.c @@ -0,0 +1,2842 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (c) Yann Collet. All rights reserved. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4_HEAPMODE : + * Select how stateless compression functions like `LZ4_compress_default()` + * allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * LZ4_ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) \ + || (defined(__riscv) && defined(__riscv_zicclsm)) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + + +/*-************************************ +* Dependency +**************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +# define LZ4_STATIC_LINKING_ONLY +#endif +#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ +# pragma warning(disable : 6239) /* disable: C6239: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6240) /* disable: C6240: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6326) /* disable: C6326: Potential comparison of a constant with another constant */ +#endif /* _MSC_VER */ + +#ifndef LZ4_FORCE_INLINE +# if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + + +/*-************************************ +* Memory routines +**************************************/ + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " %i: ", __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + + +/*-************************************ +* Types +**************************************/ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) + /* types provided by the embedding environment */ + typedef unsigned char BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef size_t uptrval; +#else +#include +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef unsigned char BYTE; /*uint8_t not necessarily blessed to alias arbitrary type*/ + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif +#endif /* LZ4_FREESTANDING */ + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + +/*-************************************ +* Reading and writing into memory +**************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif +#endif + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define LZ4_PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__)) +#elif defined(_MSC_VER) +#define LZ4_PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop)) +#endif + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16; +LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32; +LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST; + +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalignST*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign16*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign32*)memPtr)->u32 = value; } + +#else /* safe and portable access using memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] | (p[1]<<8)); + } +} + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT +static U32 LZ4_readLE32(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read32(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U32)p[0] | (p[1]<<8) | (p[2]<<16) | (p[3]<<24); + } +} +#endif + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 12 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif + LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(pop) +#endif + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif + + +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# endif +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +#endif +# endif + } else /* 32 bits */ { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } + } +} + + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } + + +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT + return LZ4_hash4(LZ4_readLE32(p), tableType); +#else + return LZ4_hash4(LZ4_read32(p), tableType); +#endif +} + +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +/* LZ4_putPosition*() : only used in byPtr mode */ +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType) +{ + const BYTE** const hashTable = (const BYTE**)tableBase; + assert(tableType == byPtr); (void)tableType; + hashTable[h] = p; +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType); +} + +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + assert(tableType == byPtr); (void)tableType; + { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } +} + +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType); +} + +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", (void*)cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic_validated() : + * inlined, to ensure branches are decided at compilation time. + * The following conditions are presumed already validated: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int* inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + int result; + const BYTE* ip = (const BYTE*)source; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*)source - startIndex; + const BYTE* lowLimit; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = + (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with indexes in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE* const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 offset = 0; + U32 forwardH; + + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + if (tableType == byU16) assert(inputSize= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, byPtr); + } else { + LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); + } } + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + const BYTE* filledIp; + + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); + } + + /* Catch up */ + filledIp = ip; + assert(ip > anchor); /* this is always true as ip has been advanced before entering the main loop */ + if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) { + do { ip--; match--; } while (((ip > anchor) & (match > lowLimit)) && (unlikely(ip[-1] == match[-1]))); + } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + + /* Encode Offset */ + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } + + /* Encode MatchLength */ + { unsigned matchCode; + + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); + } + + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); + + anchor = ip; + + /* Test end of chunk */ + if (ip >= mflimitPlusOne) break; + + /* Fill table */ + { U32 const h = LZ4_hashPosition(ip-2, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); + } else { + U32 const idx = (U32)((ip-2) - base); + LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); + } } + + /* Test next position */ + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType); + LZ4_putPosition(ip, cctx->hashTable, tableType); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + assert(ctx != NULL); + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + + +int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, src, dest, srcSize, dstCapacity, acceleration); + +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast(src, dst, srcSize, dstCapacity, 1); +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration); + } } +} + +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration); + /* clean the state on exit */ + LZ4_initStream(state, sizeof (LZ4_stream_t)); + return r; +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* const ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); + +#if (LZ4_HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/*-****************************** +* Streaming functions +********************************/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); + DEBUGLOG(4, "LZ4_createStream %p", (void*)lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} +#endif + +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", (void*)LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", (void*)LZ4_stream); + FREEMEM(LZ4_stream); + return (0); +} +#endif + + +typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, + const char* dictionary, int dictSize, + LoadDict_mode_e _ld) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + U32 idx32; + + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, (void*)dictionary, (void*)LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = (U32)tableType; + idx32 = dict->currentOffset - dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + /* Note: overwriting => favors positions end of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + p+=3; idx32+=3; + } + + if (_ld == _ld_slow) { + /* Fill hash table with additional references, to improve compression capability */ + p = dict->dictionary; + idx32 = dict->currentOffset - dict->dictSize; + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + U32 const limit = dict->currentOffset - 64 KB; + if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) { + /* Note: not overwriting => favors positions beginning of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + } + p++; idx32++; + } + } + + return (int)dict->dictSize; +} + +int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast); +} + +int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow); +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + (void*)workingStream, (void*)dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) +{ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + DEBUGLOG(4, "LZ4_renormDictT"); + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + const tableType_t tableType = byU32; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, (void*)streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = source; + } + + /* Check overlapping input/dictionary space */ + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == source) { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); + else + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); + } + + /* external dictionary mode */ + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) +{ + LZ4_stream_t_internal* const streamPtr = &LZ4_dict->internal_donotuse; + int result; + + LZ4_renormDictT(streamPtr, srcSize); + + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)srcSize; + + return result; +} + + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, (void*)safeBuffer); + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +static size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; + } + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + if (likely(s != 255)) return length; + do { + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + } while (s == 255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. + */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + if ((src == NULL) || (outputSize < 0)) { return -1; } + + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(64 KB)); + + + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + + /* Special cases */ + assert(lowPrefix <= op); + if (unlikely(outputSize==0)) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if (unlikely(srcSize==0)) { return -1; } + + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "move to safe decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using fast decode loop"); + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long literal length"); + goto _output_error; + } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, op+length); + ip += length; op += length; + } else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) { + /* We don't need to check oend, since we check it once for each loop below */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); + ip += length; op += length; + } else { + goto safe_literal_copy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + DEBUGLOG(6, "blockPos%6u: offset = %u", (unsigned)(op-(BYTE*)dst), (unsigned)offset); + match = op - offset; + assert(match <= op); /* overflow check */ + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH); + + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { + DEBUGLOG(5, "error reading long match length"); + goto _output_error; + } + length += addl; + length += MINMATCH; + DEBUGLOG(7, " long match length == %u", (unsigned)length); + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length); + goto safe_match_copy; + } + + /* Fastpath check: skip LZ4_wildCopy32 when true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { + DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); + goto _output_error; + } + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + DEBUGLOG(6, "end-of-block condition violated") + goto _output_error; + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using safe decode loop"); + while (1) { + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (length != RUN_MASK) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((ip < shortiend) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, 16); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4); + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + /* copy literals */ + cpy = op+length; + + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((ip+length != iend) || (cpy > oend)) { + DEBUGLOG(5, "should have been last run of literals") + DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", (void*)ip, (int)length, (void*)(ip+length), (void*)iend); + DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", (void*)cpy, (void*)(oend-MFLIMIT)); + DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize); + goto _output_error; + } + } + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + _copy_match: + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } + + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ + } + + /* end of decoding */ + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } +} + + +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + decode_full_block, noDict, + (BYTE*)dest, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + partial_decode, + noDict, (BYTE*)dst, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; +} +#endif + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +LZ4_FORCE_O2 +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); + int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + DEBUGLOG(5, "prefix becomes extDict"); + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); +} + + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} + +/* +These decompression functions are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_create (char* inputBuffer) +{ + (void)inputBuffer; + return LZ4_createStream(); +} +#endif + +char* LZ4_slideInputBuffer (void* state) +{ + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; +} + +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/third_party/lbd/src/lz4/lz4.h b/third_party/lbd/src/lz4/lz4.h new file mode 100644 index 000000000..b25d83f0b --- /dev/null +++ b/third_party/lbd/src/lz4/lz4.h @@ -0,0 +1,894 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (c) Yann Collet. All rights reserved. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#if !defined(LZ4_FREESTANDING) || (LZ4_FREESTANDING != 1) +#include /* size_t */ +#endif + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. +*/ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ + + +/*-************************************ +* Tuning memory usage +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Can be selected at compile time, by setting LZ4_MEMORY_USAGE. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB) + * Increasing memory usage improves compression ratio, generally at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. + * Default value is 14, for 16KB, which nicely fits into most L1 caches. + */ +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +/* These are absolute limits, they should not be changed by users */ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * @compressedSize : is the exact complete size of the compressed block. + * @dstCapacity : is the size of destination buffer (which must be already allocated), + * presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). +*/ +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'dstCapacity'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : in+out parameter. Initially contains size of input. + * Will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails. + * + * Note : 'targetDstSize' must be >= 1, because it's the smallest valid lz4 payload. + * + * Note 2:from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): + * the produced compressed content could, in rare circumstances, + * require to be decompressed into a destination buffer + * larger by at least 1 byte than decompressesSize. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize); + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic. + * When in doubt, employ the Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_loadDictSlow() : v1.10.0+ + * Same as LZ4_loadDict(), + * but uses a bit more cpu to reference the dictionary content more thoroughly. + * This is expected to slightly improve compression ratio. + * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_attach_dictionary() : stable since v1.10.0 + * + * This allows efficient re-use of a static dictionary multiple times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references @dictionaryStream in-place. + * + * Several assumptions are made about the state of @dictionaryStream. + * Currently, only states which have been prepared by LZ4_loadDict() or + * LZ4_loadDictSlow() should be expected to work. + * + * Alternatively, the provided @dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. + * @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the compression session. + * + * Note: there is no equivalent LZ4_attach_*() method on the decompression side + * because there is no initialization cost, hence no need to share the cost across multiple sessions. + * To decompress LZ4 blocks using dictionary, attached or not, + * just employ the regular LZ4_setStreamDecode() for streaming, + * or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression. + */ +LZ4LIB_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_safe_continue() : + * This decoding function allows decompression of consecutive blocks in "streaming" mode. + * The difference with the usual independent blocks is that + * new blocks are allowed to find references into former blocks. + * A block is an unsplittable entity, and must be presented entirely to the decompression function. + * LZ4_decompress_safe_continue() only accepts one block at a time. + * It's modeled after `LZ4_decompress_safe()` and behaves similarly. + * + * @LZ4_streamDecode : decompression state, tracking the position in memory of past data + * @compressedSize : exact complete size of one compressed block. + * @dstCapacity : size of destination buffer (which must be already allocated), + * must be an upper bound of decompressed size. + * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * + * The last 64KB of previously decoded data *must* remain available and unmodified + * at the memory position where they were previously decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. +*/ +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); + + +/*! LZ4_decompress_safe_usingDict() : + * Works the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() + * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +/*! LZ4_decompress_safe_partial_usingDict() : + * Behaves the same as LZ4_decompress_safe_partial() + * with the added ability to specify a memory segment for past data. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +# define LZ4LIB_STATIC_API LZ4LIB_API +#else +# define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize_extState() : introduced in v1.10.0 + * Same as LZ4_compress_destSize(), but using an externally allocated state. + * Also: exposes @acceleration + */ +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly constrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) + /* types provided by the embedding environment (e.g. ) */ + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#elif defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef int8_t LZ4_i8; + typedef unsigned char LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif + +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ +}; + +#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */ +union LZ4_stream_u { + char minStateSize[LZ4_STREAM_MINSIZE]; + LZ4_stream_t_internal internal_donotuse; +}; /* previously typedef'd to LZ4_stream_t */ + + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead +**/ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size); + + +/*! LZ4_streamDecode_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 +union LZ4_streamDecode_u { + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + + +/*-************************************ +* Obsolete Functions +**************************************/ + +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# else +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. " + "Note that the contract will change (requires block's compressed size, instead of decompressed size)") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + +#endif /* LZ4_H_98237428734687 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/third_party/lbd/src/lz4_kcompat.h b/third_party/lbd/src/lz4_kcompat.h new file mode 100644 index 000000000..d1e7703d4 --- /dev/null +++ b/third_party/lbd/src/lz4_kcompat.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LZ4_KCOMPAT_H +#define _LZ4_KCOMPAT_H + +#ifdef __KERNEL__ +#define LZ4_FREESTANDING 1 +#define LZ4_memcpy __builtin_memcpy +#define LZ4_memmove __builtin_memmove +#define LZ4_memset __builtin_memset +#define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +#include +#endif + +#include "lz4/lz4.h" + +#endif /* _LZ4_KCOMPAT_H */