From 90639e18b704002a9daa463418fb2a8b5d8d1012 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 22:10:17 -0700 Subject: [PATCH 01/15] Vendor the lbd kernel module source into the tree The lbd kernel module lives in its own repo, and until now nothing in miren carried it. That is fine for the Go half, which arrives as an ordinary module dependency, but the C source is what a node actually needs in order to build the module for its own kernel. It turns out the full source already travels inside the miren.dev/lbd module zip under src/, so there is nothing to fetch. This copies that tree into third_party/lbd via a sync script and embeds it, which keeps the version locked to whatever go.mod pins and lets a bare miren binary build the module with nothing else present. CI runs the script with --check so the copy and the pin cannot drift apart. The pin moves at the same time. The version we were on predates the header probe lbd uses to build against 6.12 and newer kernels, so the old one would have failed on any current host. --- .gitattributes | 1 + .github/workflows/test.yml | 3 + go.mod | 2 +- go.sum | 4 +- hack/sync-lbd-src.sh | 86 + third_party/lbd/README.md | 17 + third_party/lbd/embed.go | 32 + third_party/lbd/src/Makefile | 33 + third_party/lbd/src/VERSION | 1 + third_party/lbd/src/cbor_dec.h | 133 ++ third_party/lbd/src/cbor_enc.h | 116 + third_party/lbd/src/dkms.conf | 6 + third_party/lbd/src/lbd.h | 261 +++ third_party/lbd/src/lbd_main.c | 2146 +++++++++++++++++ third_party/lbd/src/lbd_qcow2.c | 1921 +++++++++++++++ third_party/lbd/src/lbd_qcow2.h | 101 + third_party/lbd/src/lbd_qcow2_format.h | 254 ++ third_party/lbd/src/lbdctl.c | 2949 ++++++++++++++++++++++++ third_party/lbd/src/lz4/lz4.c | 2842 +++++++++++++++++++++++ third_party/lbd/src/lz4/lz4.h | 894 +++++++ third_party/lbd/src/lz4_kcompat.h | 16 + 21 files changed, 11815 insertions(+), 3 deletions(-) create mode 100644 .gitattributes create mode 100755 hack/sync-lbd-src.sh create mode 100644 third_party/lbd/README.md create mode 100644 third_party/lbd/embed.go create mode 100644 third_party/lbd/src/Makefile create mode 100644 third_party/lbd/src/VERSION create mode 100644 third_party/lbd/src/cbor_dec.h create mode 100644 third_party/lbd/src/cbor_enc.h create mode 100644 third_party/lbd/src/dkms.conf create mode 100644 third_party/lbd/src/lbd.h create mode 100644 third_party/lbd/src/lbd_main.c create mode 100644 third_party/lbd/src/lbd_qcow2.c create mode 100644 third_party/lbd/src/lbd_qcow2.h create mode 100644 third_party/lbd/src/lbd_qcow2_format.h create mode 100644 third_party/lbd/src/lbdctl.c create mode 100644 third_party/lbd/src/lz4/lz4.c create mode 100644 third_party/lbd/src/lz4/lz4.h create mode 100644 third_party/lbd/src/lz4_kcompat.h diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..4a2a89003 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +third_party/** linguist-vendored diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d482a0b6..bc15ab52a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,6 +82,9 @@ jobs: with: version: v2.12.2 + - name: Check the vendored lbd source matches go.mod + run: ./hack/sync-lbd-src.sh --check + - name: Check go.mod tidiness run: | # Run go mod tidy and check if there are any changes diff --git a/go.mod b/go.mod index f9368c78d..8d945089f 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 k8s.io/klog/v2 v2.130.1 - miren.dev/lbd v0.0.0-20260224020427-8914d8db2233 + miren.dev/lbd v0.0.0-20260824210626-be4cec661034 miren.dev/mflags v0.0.0-20260913023833-b11db8e63177 modernc.org/sqlite v1.45.0 sigs.k8s.io/knftables v0.0.21 diff --git a/go.sum b/go.sum index b011fe5dd..52ac0541c 100644 --- a/go.sum +++ b/go.sum @@ -1949,8 +1949,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -miren.dev/lbd v0.0.0-20260224020427-8914d8db2233 h1:9DxH7Dhnmu7hn1OA2JC5fHpLuVgAqBySws9GLNssLl4= -miren.dev/lbd v0.0.0-20260224020427-8914d8db2233/go.mod h1:+x9fy2p45csBnGUJdqxCUmzlUTCipoVDbv6zIapTgDA= +miren.dev/lbd v0.0.0-20260824210626-be4cec661034 h1:+wGXxapZA6prIbxGLlVsgaGqu29OgBe6OMUH+P+D8EM= +miren.dev/lbd v0.0.0-20260824210626-be4cec661034/go.mod h1:+x9fy2p45csBnGUJdqxCUmzlUTCipoVDbv6zIapTgDA= miren.dev/mflags v0.0.0-20260913023833-b11db8e63177 h1:7w7JpoGLuFwv3WQHDeYQHyruWA6y+SLNuZIO4LOZggk= miren.dev/mflags v0.0.0-20260913023833-b11db8e63177/go.mod h1:G1eQ/upWVdO6BGT6dlh5Yqjt+9ncH5RUAKX6UKi1F9Q= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= diff --git a/hack/sync-lbd-src.sh b/hack/sync-lbd-src.sh new file mode 100755 index 000000000..b4c1a1425 --- /dev/null +++ b/hack/sync-lbd-src.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Sync the lbd kernel module source from the miren.dev/lbd module into +# third_party/lbd, so it can be embedded in the miren binary and handed to the +# builder image. +# +# The version is whatever go.mod pins -- never a hardcoded string here. Run +# without arguments to update the checked-in tree; run with --check to verify it +# matches, which is what CI does. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Only src/ is generated. third_party/lbd also holds a hand-written README.md +# and embed.go, which this script must leave alone. +DEST="third_party/lbd/src" +MODULE="miren.dev/lbd" + +check_only=0 +if [ "${1:-}" = "--check" ]; then + check_only=1 +elif [ -n "${1:-}" ]; then + echo "usage: $0 [--check]" >&2 + exit 2 +fi + +version="$(go list -m -f '{{.Version}}' "$MODULE")" +if [ -z "$version" ]; then + echo "could not resolve the $MODULE version from go.mod" >&2 + exit 1 +fi + +# Ensure the module is in the cache, then ask go where it landed. The cache is +# read-only, so everything copied out of it needs its mode fixed up. +go mod download "$MODULE" +src="$(go list -m -f '{{.Dir}}' "$MODULE")/src" +if [ ! -d "$src" ]; then + echo "no src/ directory in $MODULE $version (looked in $src)" >&2 + exit 1 +fi + +staging="$(mktemp -d)" +trap 'rm -rf "$staging"' EXIT + +# Everything the module build needs, and nothing else: the module's own C, the +# vendored LZ4, the Makefile, and dkms.conf. README.md and test_lbd.sh are +# developer files that belong in the lbd repo, not in the binary. +for f in \ + Makefile \ + dkms.conf \ + lbd.h \ + lbd_main.c \ + lbd_qcow2.c \ + lbd_qcow2.h \ + lbd_qcow2_format.h \ + lbdctl.c \ + cbor_dec.h \ + cbor_enc.h \ + lz4_kcompat.h \ + lz4/lz4.c \ + lz4/lz4.h; do + if [ ! -f "$src/$f" ]; then + echo "$MODULE $version is missing src/$f" >&2 + echo "the file list in $0 needs updating to match the module" >&2 + exit 1 + fi + mkdir -p "$staging/$(dirname "$f")" + install -m 0644 "$src/$f" "$staging/$f" +done + +printf '%s\n' "$version" >"$staging/VERSION" + +if [ "$check_only" -eq 1 ]; then + if diff -ru "$DEST" "$staging"; then + echo "$DEST is in sync with $MODULE $version" + exit 0 + fi + echo >&2 + echo "$DEST does not match $MODULE $version -- run hack/sync-lbd-src.sh" >&2 + exit 1 +fi + +rm -rf "$DEST" +mkdir -p "$(dirname "$DEST")" +cp -R "$staging" "$DEST" +echo "synced $DEST from $MODULE $version" diff --git a/third_party/lbd/README.md b/third_party/lbd/README.md new file mode 100644 index 000000000..c48c3cd7b --- /dev/null +++ b/third_party/lbd/README.md @@ -0,0 +1,17 @@ +# lbd kernel module source + +`src/` is a verbatim copy of `src/` from [miren.dev/lbd], at the version `go.mod` +pins. It is checked in so `pkg/lbdmod` can embed it in the miren binary and hand +it to the lbd builder image, which compiles it against the running kernel. + +**Do not edit anything under `src/`.** It is generated. Changes to the module +belong in the lbd repo. + +To update: bump `miren.dev/lbd` in `go.mod`, then run `hack/sync-lbd-src.sh`. CI +runs `hack/sync-lbd-src.sh --check` so the copy and the pin cannot drift. +`src/VERSION` records which version the current copy came from. + +`src/lz4/` is LZ4 by Yann Collet, vendored by lbd under its own BSD 2-Clause +license. + +[miren.dev/lbd]: https://github.com/mirendev/lbd diff --git a/third_party/lbd/embed.go b/third_party/lbd/embed.go new file mode 100644 index 000000000..3fc32de15 --- /dev/null +++ b/third_party/lbd/embed.go @@ -0,0 +1,32 @@ +// Package lbdsrc carries the lbd kernel module's C source, embedded in the +// miren binary so a node can compile the module without fetching anything. +// +// The source under src/ is generated by hack/sync-lbd-src.sh from the +// miren.dev/lbd version that go.mod pins; see README.md. +package lbdsrc + +import "embed" + +// FS holds the module source rooted at "src". Its "src/VERSION" file records +// the miren.dev/lbd version the tree was copied from. +// +//go:embed all:src +var FS embed.FS + +// Version reports the miren.dev/lbd version this source was copied from. +func Version() string { + data, err := FS.ReadFile("src/VERSION") + if err != nil { + // VERSION is written by the sync script alongside the source it + // describes, so its absence means the embed itself is broken. + return "unknown" + } + return trimNewline(string(data)) +} + +func trimNewline(s string) string { + for len(s) > 0 && (s[len(s)-1] == '\n' || s[len(s)-1] == '\r') { + s = s[:len(s)-1] + } + return s +} diff --git a/third_party/lbd/src/Makefile b/third_party/lbd/src/Makefile new file mode 100644 index 000000000..fb3281ea9 --- /dev/null +++ b/third_party/lbd/src/Makefile @@ -0,0 +1,33 @@ +ifneq ($(KERNELRELEASE),) +# Called from kernel build system +obj-m := lbd.o +lbd-y := lbd_main.o lbd_qcow2.o lz4/lz4.o + +# Newer kernels re-add explicit old_parent/new_parent dentry fields to +# struct renamedata, which vfs_rename() dereferences directly. Distro version +# numbers don't reliably mark when this landed, so probe the headers for the +# field and let lbd_rename_file() populate the parents when it is present. +LBD_RENAME_PARENT := $(shell grep -qs 'old_parent' $(srctree)/include/linux/fs.h && echo 1 || echo 0) + +ccflags-y := -include $(M)/lz4_kcompat.h -DLBD_HAS_RENAME_PARENT=$(LBD_RENAME_PARENT) +CFLAGS_lz4/lz4.o := -Wno-deprecated-declarations -Wframe-larger-than=32768 + +else +# Called from command line +KDIR ?= /lib/modules/$(shell uname -r)/build + +all: lbd.ko lbdctl + +lbd.ko: lbd_main.c lbd.h lbd_qcow2.c lbd_qcow2.h lbd_qcow2_format.h cbor_enc.h cbor_dec.h lz4_kcompat.h lz4/lz4.c lz4/lz4.h + $(MAKE) -C $(KDIR) M=$(CURDIR) KBUILD_MODPOST_WARN=1 modules + +lbdctl: lbdctl.c lbd.h lbd_qcow2_format.h lz4/lz4.c lz4/lz4.h + $(CC) -Wall -Wextra -O2 -o $@ lbdctl.c lz4/lz4.c + +clean: + $(MAKE) -C $(KDIR) M=$(CURDIR) clean + rm -f lbdctl + +.PHONY: all clean + +endif diff --git a/third_party/lbd/src/VERSION b/third_party/lbd/src/VERSION new file mode 100644 index 000000000..ff0b426cb --- /dev/null +++ b/third_party/lbd/src/VERSION @@ -0,0 +1 @@ +v0.0.0-20260824210626-be4cec661034 diff --git a/third_party/lbd/src/cbor_dec.h b/third_party/lbd/src/cbor_dec.h new file mode 100644 index 000000000..0e1262e7d --- /dev/null +++ b/third_party/lbd/src/cbor_dec.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CBOR_DEC_H +#define _CBOR_DEC_H + +/* + * Minimal CBOR decoder (RFC 8949) for kernel use. + * Buffer-based companion to cbor_enc.h. + * Supports major types 0 (uint), 3 (text string), 5 (map). + */ + +#include +#include +#include + +struct cbor_dec { + const u8 *buf; + size_t pos; + size_t len; +}; + +static inline void cbor_dec_init(struct cbor_dec *d, const void *buf, + size_t len) +{ + d->buf = buf; + d->pos = 0; + d->len = len; +} + +/* + * Read a CBOR head: major type (0-7) and argument value. + * Returns 0 on success, -EINVAL on truncation or reserved additional info. + */ +static inline int cbor_dec_head(struct cbor_dec *d, u8 *major, u64 *val) +{ + u8 ib, ai; + + if (d->pos >= d->len) + return -EINVAL; + + ib = d->buf[d->pos++]; + *major = ib >> 5; + ai = ib & 0x1F; + + if (ai < 24) { + *val = ai; + } else if (ai == 24) { + if (d->pos + 1 > d->len) + return -EINVAL; + *val = d->buf[d->pos++]; + } else if (ai == 25) { + if (d->pos + 2 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 8) | d->buf[d->pos + 1]; + d->pos += 2; + } else if (ai == 26) { + if (d->pos + 4 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 24) | + ((u64)d->buf[d->pos + 1] << 16) | + ((u64)d->buf[d->pos + 2] << 8) | + d->buf[d->pos + 3]; + d->pos += 4; + } else if (ai == 27) { + if (d->pos + 8 > d->len) + return -EINVAL; + *val = ((u64)d->buf[d->pos] << 56) | + ((u64)d->buf[d->pos + 1] << 48) | + ((u64)d->buf[d->pos + 2] << 40) | + ((u64)d->buf[d->pos + 3] << 32) | + ((u64)d->buf[d->pos + 4] << 24) | + ((u64)d->buf[d->pos + 5] << 16) | + ((u64)d->buf[d->pos + 6] << 8) | + d->buf[d->pos + 7]; + d->pos += 8; + } else { + return -EINVAL; /* indefinite / reserved */ + } + + return 0; +} + +/* Expect a map header (major 5), returns item count via *count */ +static inline int cbor_dec_map(struct cbor_dec *d, u64 *count) +{ + u8 major; + int ret = cbor_dec_head(d, &major, count); + + if (ret) + return ret; + if (major != 5) + return -EINVAL; + return 0; +} + +/* Expect an unsigned integer (major 0) */ +static inline int cbor_dec_uint(struct cbor_dec *d, u64 *val) +{ + u8 major; + int ret = cbor_dec_head(d, &major, val); + + if (ret) + return ret; + if (major != 0) + return -EINVAL; + return 0; +} + +/* Read a text string (major 3) into buf, NUL-terminated */ +static inline int cbor_dec_text(struct cbor_dec *d, char *buf, size_t cap, + size_t *outlen) +{ + u8 major; + u64 slen; + int ret = cbor_dec_head(d, &major, &slen); + + if (ret) + return ret; + if (major != 3) + return -EINVAL; + if (slen >= cap) + return -EINVAL; + if (d->pos + slen > d->len) + return -EINVAL; + + memcpy(buf, d->buf + d->pos, slen); + buf[slen] = '\0'; + d->pos += slen; + if (outlen) + *outlen = slen; + return 0; +} + +#endif /* _CBOR_DEC_H */ diff --git a/third_party/lbd/src/cbor_enc.h b/third_party/lbd/src/cbor_enc.h new file mode 100644 index 000000000..f6c613441 --- /dev/null +++ b/third_party/lbd/src/cbor_enc.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CBOR_ENC_H +#define _CBOR_ENC_H + +/* + * Minimal CBOR encoder (RFC 8949) for kernel use. + * Supports major types 0 (uint), 2 (byte string), 3 (text string), 5 (map). + */ + +#include +#include +#include + +struct cbor_enc { + u8 *buf; + size_t pos; + size_t cap; + int err; /* sticky -ENOSPC */ +}; + +static inline void cbor_enc_init(struct cbor_enc *e, void *buf, size_t cap) +{ + e->buf = buf; + e->pos = 0; + e->cap = cap; + e->err = 0; +} + +static inline size_t cbor_enc_len(const struct cbor_enc *e) +{ + return e->pos; +} + +/* Write one byte, set sticky error on overflow */ +static inline void cbor_put(struct cbor_enc *e, u8 b) +{ + if (e->err) + return; + if (e->pos >= e->cap) { + e->err = -ENOSPC; + return; + } + e->buf[e->pos++] = b; +} + +/* + * Encode a CBOR head: major type (top 3 bits) + value. + * Chooses the shortest encoding automatically. + */ +static inline void cbor_enc_head(struct cbor_enc *e, u8 major, u64 val) +{ + u8 mt = major << 5; + + if (val < 24) { + cbor_put(e, mt | (u8)val); + } else if (val <= 0xFF) { + cbor_put(e, mt | 24); + cbor_put(e, (u8)val); + } else if (val <= 0xFFFF) { + cbor_put(e, mt | 25); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } else if (val <= 0xFFFFFFFF) { + cbor_put(e, mt | 26); + cbor_put(e, (u8)(val >> 24)); + cbor_put(e, (u8)(val >> 16)); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } else { + cbor_put(e, mt | 27); + cbor_put(e, (u8)(val >> 56)); + cbor_put(e, (u8)(val >> 48)); + cbor_put(e, (u8)(val >> 40)); + cbor_put(e, (u8)(val >> 32)); + cbor_put(e, (u8)(val >> 24)); + cbor_put(e, (u8)(val >> 16)); + cbor_put(e, (u8)(val >> 8)); + cbor_put(e, (u8)val); + } +} + +/* Major type 5: map of count pairs */ +static inline void cbor_enc_map(struct cbor_enc *e, u64 count) +{ + cbor_enc_head(e, 5, count); +} + +/* Major type 0: unsigned integer */ +static inline void cbor_enc_uint(struct cbor_enc *e, u64 val) +{ + cbor_enc_head(e, 0, val); +} + +/* Major type 3: text string (UTF-8) */ +static inline void cbor_enc_text(struct cbor_enc *e, const char *s, size_t len) +{ + size_t i; + + cbor_enc_head(e, 3, len); + for (i = 0; i < len; i++) + cbor_put(e, (u8)s[i]); +} + +/* Convenience: encode a text string key (NUL-terminated) */ +static inline void cbor_enc_text_key(struct cbor_enc *e, const char *key) +{ + cbor_enc_text(e, key, strlen(key)); +} + +/* Major type 2: byte string header only (caller appends raw data) */ +static inline void cbor_enc_bytes_hdr(struct cbor_enc *e, u64 len) +{ + cbor_enc_head(e, 2, len); +} + +#endif /* _CBOR_ENC_H */ diff --git a/third_party/lbd/src/dkms.conf b/third_party/lbd/src/dkms.conf new file mode 100644 index 000000000..9000fa5b1 --- /dev/null +++ b/third_party/lbd/src/dkms.conf @@ -0,0 +1,6 @@ +PACKAGE_NAME="lbd" +PACKAGE_VERSION="0.1.0" +MAKE[0]="make -C ${kernel_source_dir} M=${dkms_tree}/${PACKAGE_NAME}/${PACKAGE_VERSION}/build" +BUILT_MODULE_NAME[0]="lbd" +DEST_MODULE_LOCATION[0]="/extra" +AUTOINSTALL="yes" diff --git a/third_party/lbd/src/lbd.h b/third_party/lbd/src/lbd.h new file mode 100644 index 000000000..ad15413a3 --- /dev/null +++ b/third_party/lbd/src/lbd.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LBD_H +#define _LBD_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd_qcow2.h" + +#define LBD_NAME "lbd" +#define LBD_CTL_NAME "lbd-control" +#define LBD_VERSION "0.1.0" + +#define LBD_BLOCK_SIZE 4096 +#define LBD_QUEUE_DEPTH 128 + +/* Log format constants */ +#define LBD_LOG_VERSION 2 +#define LBD_LOG_PATH_MAX 256 +#define LBD_LOG_MAX_SIZE_DEFAULT (64ULL * 1024 * 1024) /* 64 MiB */ +#define LBD_LOG_MAX_AGE_DEFAULT 60 /* seconds */ +#define LBD_LOG_BUF_SIZE (5ULL * 1024 * 1024) /* 5 MiB */ + +/* Device states */ +enum lbd_state { + LBD_STATE_UNBOUND = 0, + LBD_STATE_BOUND, + LBD_STATE_REMOVING, +}; + +/* ioctl interface */ +#define LBD_CTL_MAGIC 'L' +#define LBD_CTL_ADD _IOWR(LBD_CTL_MAGIC, 0, struct lbd_ctl_add) +#define LBD_CTL_REMOVE _IOW(LBD_CTL_MAGIC, 1, struct lbd_ctl_remove) +#define LBD_CTL_INFO _IOWR(LBD_CTL_MAGIC, 2, struct lbd_ctl_info) + +struct lbd_ctl_add { + char path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + char base_path[LBD_LOG_PATH_MAX]; /* empty string = no base */ + __s32 index; /* out: assigned device index */ + __u64 log_max_size; /* 0 = default */ + __u32 log_max_age_secs; /* 0 = default */ +}; + +struct lbd_ctl_remove { + __s32 index; +}; + +struct lbd_ctl_info { + __s32 index; + __u32 state; + __u64 size; + char path[LBD_LOG_PATH_MAX]; +}; + +/* + * CBOR log key constants (numeric map keys for compactness). + * + * Header map(5): { 1:version, 2:block_size, 3:segment_label, + * 4:device_size, 5:backing_path } + * Write map(7): { 1:"W", 2:timestamp_ns, 3:sequence, 4:block, + * 5:length, 6:crc32, 7:data_bytes } + * Trim map(5): { 1:"T", 2:timestamp_ns, 3:sequence, 4:block, + * 5:length } + */ +#define LBD_CBOR_KEY_HDR_VERSION 1 +#define LBD_CBOR_KEY_HDR_BLOCK_SIZE 2 +#define LBD_CBOR_KEY_HDR_SEGMENT_LABEL 3 +#define LBD_CBOR_KEY_HDR_DEVICE_SIZE 4 +#define LBD_CBOR_KEY_HDR_BACKING_PATH 5 + +#define LBD_CBOR_KEY_OP 1 +#define LBD_CBOR_KEY_TIMESTAMP 2 +#define LBD_CBOR_KEY_SEQUENCE 3 +#define LBD_CBOR_KEY_BLOCK 4 +#define LBD_CBOR_KEY_LENGTH 5 +#define LBD_CBOR_KEY_CHECKSUM 6 +#define LBD_CBOR_KEY_DATA 7 + +/* Watch command keys (write path: userspace -> kernel) */ +#define LBD_WATCH_KEY_CMD 1 +#define LBD_WATCH_KEY_DEV 2 +#define LBD_WATCH_KEY_PATH 3 /* text: file path (for swap) */ + +#ifdef CONFIG_LBD_MISS_HANDLER +/* Miss event keys (kernel -> userspace and userspace -> kernel) */ +#define LBD_MISS_KEY_TYPE 1 /* text: event/command type */ +#define LBD_MISS_KEY_DEV 2 /* uint: device index */ +#define LBD_MISS_KEY_CLUSTER 3 /* uint: cluster index */ +#endif + +/* Event keys (read path: kernel -> userspace) */ +#define LBD_EVENT_KEY_TYPE 1 +#define LBD_EVENT_KEY_DEV 2 +#define LBD_EVENT_KEY_LABEL 3 +#define LBD_EVENT_KEY_DIR 4 +#define LBD_EVENT_KEY_SEQ 5 +#define LBD_EVENT_KEY_SIZE 6 + +#ifdef CONFIG_LBD_MISS_HANDLER +/* Forward declaration for miss handler */ +struct lbd_miss_handler; + +enum lbd_miss_action { + LBD_MISS_CONTINUE = 0, + LBD_MISS_RETRY, +}; +#endif + +/* Per-request data (embedded in blk-mq PDU) */ +struct lbd_cmd { + struct list_head list_entry; + int ret; +}; + +/* Per-device state */ +struct lbd_device { + int index; + enum lbd_state state; + atomic_t open_count; + + struct gendisk *gd; + struct blk_mq_tag_set tag_set; + + struct file *backing_file; + struct file *log_file; + char backing_path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + struct path log_dir_path; + loff_t size; /* device size in bytes */ + + struct workqueue_struct *wq; + struct work_struct work; + struct list_head cmd_list; + spinlock_t cmd_lock; + + /* Log segmentation — all protected by log_mutex */ + struct mutex log_mutex; + bool log_has_entries; /* current segment has data entries */ + u64 log_seq; /* global, never resets */ + char log_segment_label[25]; /* current segment TAI64N label */ + u64 log_max_size; /* rotation size threshold */ + u32 log_max_age_secs; /* rotation age threshold */ + struct delayed_work log_rotate_dwork; /* age timer */ + void *log_buf; /* write buffer, allocated once */ + size_t log_buf_used; /* bytes currently in buffer */ + + void *lz4_state; /* LZ4 compression state, allocated once */ + + /* qcow2-lz4 backing store */ + bool is_qcow2; + struct lbd_qcow2 qcow2; + + /* Thin snapshot base layer */ + struct lbd_qcow2_base *base; /* NULL when no base configured */ + char base_path[LBD_LOG_PATH_MAX]; + +#ifdef CONFIG_LBD_MISS_HANDLER + /* Block miss handler (for remote-fetch workflow) */ + struct lbd_miss_handler *miss_handler; /* NULL when no handler */ + spinlock_t miss_handler_lock; /* protects miss_handler pointer */ +#endif + + /* I/O stats (atomic for lock-free sysfs reads) */ + atomic64_t stat_reads; + atomic64_t stat_writes; + atomic64_t stat_trims; + atomic64_t stat_read_bytes; + atomic64_t stat_write_bytes; + atomic64_t stat_trim_bytes; + + /* Allocation stats (qcow2 space reuse) */ + atomic64_t stat_alloc_reused; /* clusters written in-place or via free list */ + atomic64_t stat_alloc_new; /* clusters allocated by appending */ + atomic64_t stat_alloc_freed; /* extents added to the free list */ + atomic64_t stat_compressed; /* clusters stored compressed */ + atomic64_t stat_uncompressed; /* clusters stored uncompressed */ + + /* Log stats */ + atomic64_t stat_log_rotations; + ktime_t segment_start_time; +}; + +/* Kernel version compatibility */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 14, 0) +#define LBD_HAS_BLK_MQ_ALLOC_DISK 1 +#else +#define LBD_HAS_BLK_MQ_ALLOC_DISK 0 +#endif + +/* 6.3+: struct mnt_idmap replaces struct user_namespace for VFS idmap ops; + * nop_mnt_idmap replaces &init_user_ns */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 3, 0) +#define LBD_HAS_MNT_IDMAP 1 +#else +#define LBD_HAS_MNT_IDMAP 0 +#endif + +/* 6.5+: block_device_operations.open takes struct gendisk * (was block_device *); + * .release drops the fmode_t argument */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 5, 0) +#define LBD_HAS_GENDISK_OPEN 1 +#else +#define LBD_HAS_GENDISK_OPEN 0 +#endif + +/* 6.9+: blk_mq_alloc_disk() takes struct queue_limits * as 2nd arg; + * BLK_MQ_F_SHOULD_MERGE removed; blk_queue_* setters removed; + * use BLK_FEAT_WRITE_CACHE in limits.features */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0) +#define LBD_HAS_QUEUE_LIMITS_API 1 +#else +#define LBD_HAS_QUEUE_LIMITS_API 0 +#endif + +/* 6.12+: struct renamedata drops old_dir/new_dir fields (derived from dentries); + * lookup_one_len() removed, use lookup_one(idmap, name, dir, len) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) +#define LBD_HAS_RENAME_NO_DIR 1 +#else +#define LBD_HAS_RENAME_NO_DIR 0 +#endif + +/* 6.15+: lookup_one() takes struct qstr * instead of (const char *, int) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0) +#define LBD_HAS_LOOKUP_ONE_QSTR 1 +#else +#define LBD_HAS_LOOKUP_ONE_QSTR 0 +#endif + +/* 6.18+: struct renamedata uses single mnt_idmap field + * (replaces old_mnt_idmap/new_mnt_idmap) */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) +#define LBD_HAS_RENAME_SINGLE_IDMAP 1 +#else +#define LBD_HAS_RENAME_SINGLE_IDMAP 0 +#endif + +/* Newer kernels re-add explicit old_parent/new_parent dentry fields to + * struct renamedata; vfs_rename() then dereferences them instead of deriving + * the parents from the dentries, so they must be populated or the rename + * faults on a NULL parent. The exact version varies by distro (the field was + * seen on a 7.0.x tree), so src/Makefile probes the kernel headers for the + * member and defines LBD_HAS_RENAME_PARENT; default off when unprobed (e.g. + * the userspace lbdctl build, which never renames through the VFS). */ +#ifndef LBD_HAS_RENAME_PARENT +#define LBD_HAS_RENAME_PARENT 0 +#endif + +#endif /* _LBD_H */ diff --git a/third_party/lbd/src/lbd_main.c b/third_party/lbd/src/lbd_main.c new file mode 100644 index 000000000..3085a9c14 --- /dev/null +++ b/third_party/lbd/src/lbd_main.c @@ -0,0 +1,2146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LBD - Logging Block Device + * + * A block device backed by a file (like loop) that logs all write + * operations to a companion .log file for change tracking / audit / replay. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd.h" + +#if LBD_HAS_MNT_IDMAP +#include +#endif +#include "lbd_qcow2.h" +#include "cbor_enc.h" +#include "cbor_dec.h" +#include "lz4_kcompat.h" +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("lbd authors"); +MODULE_DESCRIPTION("Logging Block Device"); +MODULE_VERSION(LBD_VERSION); + +static int lbd_major; +static DEFINE_IDR(lbd_devices); +static DEFINE_MUTEX(lbd_devices_mutex); +static struct miscdevice lbd_misc; + +/* ---------------------------------------------------------------- + * Log rotation watchers + * ---------------------------------------------------------------- */ + +#define LBD_WATCHER_QUEUE_SIZE 64 + +struct lbd_watcher_event { + int dev_index; + u64 log_seq; + u64 device_size; + char segment_label[25]; + char log_dir[LBD_LOG_PATH_MAX]; +}; + +struct lbd_watcher { + struct list_head list; + spinlock_t lock; + wait_queue_head_t wq; + struct lbd_watcher_event queue[LBD_WATCHER_QUEUE_SIZE]; + unsigned int head, tail, count; + int filter_dev; /* -1 = all */ +}; + +static LIST_HEAD(lbd_watchers); +static DEFINE_SPINLOCK(lbd_watchers_lock); + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Block miss handler (for remote-fetch workflow) + * ---------------------------------------------------------------- */ + +struct lbd_miss_pending { + struct completion done; + enum lbd_miss_action action; +}; + +struct lbd_miss_handler { + spinlock_t lock; + wait_queue_head_t wq; + int dev_index; + struct lbd_miss_pending *pending; /* NULL = idle */ + bool has_event; + u64 miss_cluster; +}; +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* Control fd state — supports both watch and miss handler on same fd */ +struct lbd_ctl_state { + struct lbd_watcher *watcher; /* NULL until "watch" command */ +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *miss; /* NULL until "manage_misses" command */ +#endif +}; + +/* ---------------------------------------------------------------- + * Block device operations + * ---------------------------------------------------------------- */ + +#if LBD_HAS_GENDISK_OPEN +static int lbd_open(struct gendisk *disk, unsigned int mode) +{ + struct lbd_device *dev = disk->private_data; +#else +static int lbd_open(struct block_device *bdev, fmode_t mode) +{ + struct lbd_device *dev = bdev->bd_disk->private_data; +#endif + + if (dev->state != LBD_STATE_BOUND) + return -ENXIO; + atomic_inc(&dev->open_count); + return 0; +} + +#if LBD_HAS_GENDISK_OPEN +static void lbd_release(struct gendisk *disk) +{ + struct lbd_device *dev = disk->private_data; +#else +static void lbd_release(struct gendisk *disk, fmode_t mode) +{ + struct lbd_device *dev = disk->private_data; +#endif + + atomic_dec(&dev->open_count); +} + +static const struct block_device_operations lbd_fops = { + .owner = THIS_MODULE, + .open = lbd_open, + .release = lbd_release, +}; + +/* ---------------------------------------------------------------- + * I/O: read path + * ---------------------------------------------------------------- */ + +static int lbd_do_read(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + ssize_t ret; + + if (dev->is_qcow2) + return lbd_qcow2_read(dev, rq); + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + + ret = kernel_read(dev->backing_file, mapped + bvec.bv_offset, + bvec.bv_len, &pos); + kunmap_local(mapped); + + if (ret != bvec.bv_len) { + if (ret >= 0) + ret = -EIO; + return ret; + } + } + + atomic64_inc(&dev->stat_reads); + atomic64_add(blk_rq_bytes(rq), &dev->stat_read_bytes); + return 0; +} + +/* Forward declarations */ +static const struct attribute_group lbd_attr_group; + +/* Forward declarations for log segmentation */ +static int lbd_open_log_file(struct lbd_device *dev); +static void lbd_rotate_segment(struct lbd_device *dev); +static void lbd_maybe_rotate_segment(struct lbd_device *dev); +static void lbd_notify_watchers(struct lbd_device *dev); + +/* ---------------------------------------------------------------- + * I/O: log write buffer + * ---------------------------------------------------------------- */ + +static int lbd_log_flush(struct lbd_device *dev) +{ + loff_t pos; + ssize_t ret; + + if (dev->log_buf_used == 0) + return 0; + + /* Lazy-open: create the segment file on first flush */ + if (!dev->log_file) { + int err = lbd_open_log_file(dev); + if (err) { + dev->log_buf_used = 0; + return err; + } + } + + pos = i_size_read(file_inode(dev->log_file)); + ret = kernel_write(dev->log_file, dev->log_buf, + dev->log_buf_used, &pos); + if (ret != dev->log_buf_used) { + pr_warn_ratelimited("lbd%d: log flush failed (%zd)\n", + dev->index, ret); + dev->log_buf_used = 0; + return ret < 0 ? ret : -EIO; + } + dev->log_buf_used = 0; + return 0; +} + +static int lbd_log_buf_append(struct lbd_device *dev, + const void *data, size_t len) +{ + if (dev->log_buf_used + len > LBD_LOG_BUF_SIZE) { + int ret = lbd_log_flush(dev); + if (ret) + return ret; + } + memcpy(dev->log_buf + dev->log_buf_used, data, len); + dev->log_buf_used += len; + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: log write + * ---------------------------------------------------------------- */ + +static void lbd_log_write(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + u32 data_len = blk_rq_bytes(rq); + void *data_buf; + void *comp_buf = NULL; + int comp_len; + int comp_cap; + size_t offset = 0; + u8 tmp[128]; + struct cbor_enc e; + u32 crc; + u64 ts, block; + int ret; + + if (!data_len) + return; + + data_buf = kvmalloc(data_len, GFP_NOIO); + if (!data_buf) { + pr_warn_ratelimited("lbd%d: log alloc failed\n", dev->index); + return; + } + + /* Gather write data from bio vecs */ + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + memcpy(data_buf + offset, mapped + bvec.bv_offset, bvec.bv_len); + kunmap_local(mapped); + offset += bvec.bv_len; + } + + /* CRC on uncompressed data */ + crc = crc32(~0U, data_buf, data_len) ^ ~0U; + + /* LZ4-compress the data */ + comp_cap = LZ4_compressBound(data_len); + comp_buf = kvmalloc(comp_cap, GFP_NOIO); + if (!comp_buf) { + pr_warn_ratelimited("lbd%d: log compress alloc failed\n", + dev->index); + kvfree(data_buf); + return; + } + + comp_len = LZ4_compress_fast_extState(dev->lz4_state, + data_buf, comp_buf, + data_len, comp_cap, 1); + if (comp_len <= 0) { + pr_warn_ratelimited("lbd%d: LZ4 compression failed\n", + dev->index); + kvfree(comp_buf); + kvfree(data_buf); + return; + } + + ts = ktime_get_real_ns(); + block = (loff_t)blk_rq_pos(rq) * 512 / LBD_BLOCK_SIZE; + + mutex_lock(&dev->log_mutex); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 7); + + cbor_enc_uint(&e, LBD_CBOR_KEY_OP); + cbor_enc_text(&e, "W", 1); + + cbor_enc_uint(&e, LBD_CBOR_KEY_TIMESTAMP); + cbor_enc_uint(&e, ts); + + cbor_enc_uint(&e, LBD_CBOR_KEY_SEQUENCE); + cbor_enc_uint(&e, dev->log_seq++); + + cbor_enc_uint(&e, LBD_CBOR_KEY_BLOCK); + cbor_enc_uint(&e, block); + + /* Length = uncompressed size (needed for decompression) */ + cbor_enc_uint(&e, LBD_CBOR_KEY_LENGTH); + cbor_enc_uint(&e, data_len); + + cbor_enc_uint(&e, LBD_CBOR_KEY_CHECKSUM); + cbor_enc_uint(&e, crc); + + /* Data = LZ4-compressed bytes */ + cbor_enc_uint(&e, LBD_CBOR_KEY_DATA); + cbor_enc_bytes_hdr(&e, comp_len); + + if (e.err) { + dev->log_seq--; + goto warn; + } + + ret = lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); + if (ret) + goto warn; + + ret = lbd_log_buf_append(dev, comp_buf, comp_len); + if (ret) + goto warn; + + dev->log_has_entries = true; + lbd_maybe_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + kvfree(comp_buf); + kvfree(data_buf); + return; + +warn: + mutex_unlock(&dev->log_mutex); + kvfree(comp_buf); + kvfree(data_buf); + pr_warn_ratelimited("lbd%d: log write failed (%d)\n", dev->index, ret); +} + +/* ---------------------------------------------------------------- + * I/O: write path + * ---------------------------------------------------------------- */ + +static int lbd_do_write(struct lbd_device *dev, struct request *rq) +{ + struct req_iterator iter; + struct bio_vec bvec; + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + ssize_t ret; + + if (dev->is_qcow2) { + ret = lbd_qcow2_write(dev, rq); + if (ret) + return ret; + lbd_log_write(dev, rq); + atomic64_inc(&dev->stat_writes); + atomic64_add(blk_rq_bytes(rq), &dev->stat_write_bytes); + return 0; + } + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + + ret = kernel_write(dev->backing_file, mapped + bvec.bv_offset, + bvec.bv_len, &pos); + kunmap_local(mapped); + + if (ret != bvec.bv_len) { + if (ret >= 0) + ret = -EIO; + return ret; + } + } + + /* Log the write - failure is non-fatal */ + lbd_log_write(dev, rq); + + atomic64_inc(&dev->stat_writes); + atomic64_add(blk_rq_bytes(rq), &dev->stat_write_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: discard (TRIM) path + * ---------------------------------------------------------------- */ + +static void lbd_log_discard(struct lbd_device *dev, struct request *rq) +{ + u8 tmp[64]; + struct cbor_enc e; + u64 ts = ktime_get_real_ns(); + u64 block = (loff_t)blk_rq_pos(rq) * 512 / LBD_BLOCK_SIZE; + u32 length = blk_rq_bytes(rq); + int ret; + + mutex_lock(&dev->log_mutex); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 5); + + cbor_enc_uint(&e, LBD_CBOR_KEY_OP); + cbor_enc_text(&e, "T", 1); + + cbor_enc_uint(&e, LBD_CBOR_KEY_TIMESTAMP); + cbor_enc_uint(&e, ts); + + cbor_enc_uint(&e, LBD_CBOR_KEY_SEQUENCE); + cbor_enc_uint(&e, dev->log_seq++); + + cbor_enc_uint(&e, LBD_CBOR_KEY_BLOCK); + cbor_enc_uint(&e, block); + + cbor_enc_uint(&e, LBD_CBOR_KEY_LENGTH); + cbor_enc_uint(&e, length); + + if (e.err) { + dev->log_seq--; + goto warn; + } + + ret = lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); + if (ret) + goto warn; + + dev->log_has_entries = true; + lbd_maybe_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + return; + +warn: + mutex_unlock(&dev->log_mutex); + pr_warn_ratelimited("lbd%d: log discard failed (%d)\n", dev->index, ret); +} + +static int lbd_do_discard(struct lbd_device *dev, struct request *rq) +{ + loff_t pos = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + unsigned int len = blk_rq_bytes(rq); + int ret; + + if (dev->is_qcow2) { + ret = lbd_qcow2_discard(dev, rq); + if (ret) + return ret; + lbd_log_discard(dev, rq); + atomic64_inc(&dev->stat_trims); + atomic64_add(len, &dev->stat_trim_bytes); + return 0; + } + + if (dev->backing_file->f_op->fallocate) { + ret = dev->backing_file->f_op->fallocate(dev->backing_file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, pos, len); + if (ret && ret != -EINVAL && ret != -EOPNOTSUPP) + return ret; + } + + lbd_log_discard(dev, rq); + + atomic64_inc(&dev->stat_trims); + atomic64_add(len, &dev->stat_trim_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * I/O: request dispatch + * ---------------------------------------------------------------- */ + +static void lbd_handle_request(struct lbd_device *dev, struct request *rq) +{ + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + int ret; + + switch (req_op(rq)) { + case REQ_OP_READ: + ret = lbd_do_read(dev, rq); + break; + case REQ_OP_WRITE: + ret = lbd_do_write(dev, rq); + break; + case REQ_OP_DISCARD: + ret = lbd_do_discard(dev, rq); + break; + case REQ_OP_FLUSH: + ret = vfs_fsync(dev->backing_file, 0); + if (!ret) { + mutex_lock(&dev->log_mutex); + if (dev->log_has_entries) + lbd_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); + } + break; + default: + ret = -EIO; + break; + } + + cmd->ret = ret; +} + +static void lbd_work_fn(struct work_struct *work) +{ + struct lbd_device *dev = container_of(work, struct lbd_device, work); + struct lbd_cmd *cmd; + LIST_HEAD(local_list); + unsigned int saved_flags = current->flags; + + /* Prevent writeback deadlock (same technique as loop driver) */ + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + + spin_lock_irq(&dev->cmd_lock); + list_splice_init(&dev->cmd_list, &local_list); + spin_unlock_irq(&dev->cmd_lock); + + while (!list_empty(&local_list)) { + cmd = list_first_entry(&local_list, struct lbd_cmd, list_entry); + list_del(&cmd->list_entry); + + lbd_handle_request(dev, + blk_mq_rq_from_pdu(cmd)); + + blk_mq_complete_request(blk_mq_rq_from_pdu(cmd)); + } + + current->flags = (current->flags & ~(PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO)) | + (saved_flags & (PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO)); +} + +static void lbd_complete_rq(struct request *rq) +{ + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + + blk_mq_end_request(rq, cmd->ret ? BLK_STS_IOERR : BLK_STS_OK); +} + +static blk_status_t lbd_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) +{ + struct lbd_device *dev = hctx->queue->queuedata; + struct request *rq = bd->rq; + struct lbd_cmd *cmd = blk_mq_rq_to_pdu(rq); + + blk_mq_start_request(rq); + + if (dev->state != LBD_STATE_BOUND) + return BLK_STS_IOERR; + + INIT_LIST_HEAD(&cmd->list_entry); + cmd->ret = 0; + + spin_lock_irq(&dev->cmd_lock); + list_add_tail(&cmd->list_entry, &dev->cmd_list); + spin_unlock_irq(&dev->cmd_lock); + + queue_work(dev->wq, &dev->work); + return BLK_STS_OK; +} + +static const struct blk_mq_ops lbd_mq_ops = { + .queue_rq = lbd_queue_rq, + .complete = lbd_complete_rq, +}; + +/* ---------------------------------------------------------------- + * Device lifecycle + * ---------------------------------------------------------------- */ + +static int lbd_write_log_header(struct lbd_device *dev) +{ + u8 tmp[512]; + struct cbor_enc e; + size_t path_len = strlen(dev->backing_path); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + + cbor_enc_map(&e, 5); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_VERSION); + cbor_enc_uint(&e, LBD_LOG_VERSION); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_BLOCK_SIZE); + cbor_enc_uint(&e, LBD_BLOCK_SIZE); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_SEGMENT_LABEL); + cbor_enc_text(&e, dev->log_segment_label, + strlen(dev->log_segment_label)); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_DEVICE_SIZE); + cbor_enc_uint(&e, dev->size); + + cbor_enc_uint(&e, LBD_CBOR_KEY_HDR_BACKING_PATH); + cbor_enc_text(&e, dev->backing_path, path_len); + + if (e.err) + return e.err; + + return lbd_log_buf_append(dev, tmp, cbor_enc_len(&e)); +} + +/* ---------------------------------------------------------------- + * Log segmentation + * ---------------------------------------------------------------- */ + +static int lbd_log_name_tmp(const char *label, char *buf, size_t sz) +{ + return snprintf(buf, sz, "disk.%s.log.tmp", label); +} + +static int lbd_log_name_final(const char *label, char *buf, size_t sz) +{ + return snprintf(buf, sz, "disk.%s.log", label); +} + +/* + * Generate a TAI64N label from the current wall-clock time. + * Format: 16 hex digits (TAI seconds) + 8 hex digits (nanoseconds). + * buf must be at least 25 bytes (24 chars + NUL). + */ +static void lbd_tai64n_label(char *buf) +{ + struct timespec64 ts; + u64 tai_secs; + + ktime_get_real_ts64(&ts); + tai_secs = (u64)ts.tv_sec + 0x4000000000000000ULL; + snprintf(buf, 25, "%016llx%08lx", tai_secs, ts.tv_nsec); +} + +static int lbd_rename_file(struct file *old_file, const char *new_basename) +{ + struct dentry *old_dentry = old_file->f_path.dentry; + struct dentry *parent = old_dentry->d_parent; + struct dentry *new_dentry; + struct renamedata rd; + int ret; + + lock_rename(parent, parent); + +#if LBD_HAS_LOOKUP_ONE_QSTR + { + struct qstr qname = QSTR_INIT(new_basename, + strlen(new_basename)); + new_dentry = lookup_one(&nop_mnt_idmap, &qname, parent); + } +#elif LBD_HAS_RENAME_NO_DIR + new_dentry = lookup_one(&nop_mnt_idmap, new_basename, parent, + strlen(new_basename)); +#else + new_dentry = lookup_one_len(new_basename, parent, strlen(new_basename)); +#endif + if (IS_ERR(new_dentry)) { + ret = PTR_ERR(new_dentry); + goto out_unlock; + } + + memset(&rd, 0, sizeof(rd)); +#if LBD_HAS_RENAME_PARENT + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_parent = parent; + rd.old_dentry = old_dentry; + rd.new_parent = parent; + rd.new_dentry = new_dentry; +#elif LBD_HAS_RENAME_SINGLE_IDMAP + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_dentry = old_dentry; + rd.new_dentry = new_dentry; +#elif LBD_HAS_RENAME_NO_DIR + rd.old_mnt_idmap = &nop_mnt_idmap; + rd.old_dentry = old_dentry; + rd.new_mnt_idmap = &nop_mnt_idmap; + rd.new_dentry = new_dentry; +#elif LBD_HAS_MNT_IDMAP + rd.old_mnt_idmap = &nop_mnt_idmap; + rd.old_dir = d_inode(parent); + rd.old_dentry = old_dentry; + rd.new_mnt_idmap = &nop_mnt_idmap; + rd.new_dir = d_inode(parent); + rd.new_dentry = new_dentry; +#else + rd.old_mnt_userns = &init_user_ns; + rd.old_dir = d_inode(parent); + rd.old_dentry = old_dentry; + rd.new_mnt_userns = &init_user_ns; + rd.new_dir = d_inode(parent); + rd.new_dentry = new_dentry; +#endif + + ret = vfs_rename(&rd); + dput(new_dentry); + +out_unlock: + unlock_rename(parent, parent); + return ret; +} + +/* + * Lazy-open the .log.tmp file for the current segment. Called from + * lbd_log_flush() when buffered data first needs to hit disk. + */ +static int lbd_open_log_file(struct lbd_device *dev) +{ + char name[48]; + struct file *f; + + lbd_log_name_tmp(dev->log_segment_label, name, sizeof(name)); + + /* + * Open relative to the log directory rather than using an + * absolute path. This avoids path resolution failures when + * called from a kworker whose mount namespace differs from + * the process that created the device. + */ + f = file_open_root(&dev->log_dir_path, name, + O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, 0600); + if (IS_ERR(f)) { + pr_warn("lbd%d: cannot open %s: %ld\n", + dev->index, name, PTR_ERR(f)); + return PTR_ERR(f); + } + + dev->log_file = f; + return 0; +} + +/* + * Begin a new log segment. Resets the buffer, generates a TAI64N + * label, and writes the CBOR header into the buffer. No file is + * created on disk — that happens lazily in lbd_log_flush(). + */ +static void lbd_begin_segment(struct lbd_device *dev) +{ + dev->log_file = NULL; + dev->log_buf_used = 0; + dev->log_has_entries = false; + dev->segment_start_time = ktime_get(); + lbd_tai64n_label(dev->log_segment_label); + lbd_write_log_header(dev); +} + +static void lbd_rotate_segment(struct lbd_device *dev) +{ + char final_name[48]; + int ret; + + if (!dev->log_has_entries) + return; + + /* Flush buffer (lazy-opens .log.tmp) then fsync */ + lbd_log_flush(dev); + if (!dev->log_file) + goto next; + + vfs_fsync(dev->log_file, 0); + + /* Rename disk..log.tmp -> disk..log */ + lbd_log_name_final(dev->log_segment_label, final_name, + sizeof(final_name)); + + ret = lbd_rename_file(dev->log_file, final_name); + if (ret) + pr_warn("lbd%d: rename to %s failed: %d\n", + dev->index, final_name, ret); + + fput(dev->log_file); + dev->log_file = NULL; + +next: + atomic64_inc(&dev->stat_log_rotations); + + /* Notify watchers before lbd_begin_segment() overwrites the label */ + lbd_notify_watchers(dev); + + /* Begin fresh segment (buffer only, no file yet) */ + lbd_begin_segment(dev); + + /* Reschedule age timer */ + if (dev->log_max_age_secs > 0) + mod_delayed_work(system_wq, &dev->log_rotate_dwork, + msecs_to_jiffies(dev->log_max_age_secs * 1000)); +} + +static void lbd_log_rotate_work_fn(struct work_struct *work) +{ + struct lbd_device *dev = container_of(work, struct lbd_device, + log_rotate_dwork.work); + + mutex_lock(&dev->log_mutex); + if (dev->log_has_entries) + lbd_rotate_segment(dev); + mutex_unlock(&dev->log_mutex); +} + +static void lbd_maybe_rotate_segment(struct lbd_device *dev) +{ + loff_t file_size = dev->log_file ? + i_size_read(file_inode(dev->log_file)) : 0; + + if (file_size + dev->log_buf_used >= (loff_t)dev->log_max_size) + lbd_rotate_segment(dev); +} + +static void lbd_finalize_log(struct lbd_device *dev) +{ + char final_name[48]; + + if (dev->log_has_entries) + lbd_log_flush(dev); + + if (!dev->log_file) + return; + + vfs_fsync(dev->log_file, 0); + + if (dev->log_has_entries) { + lbd_log_name_final(dev->log_segment_label, final_name, + sizeof(final_name)); + lbd_rename_file(dev->log_file, final_name); + lbd_notify_watchers(dev); + } + + fput(dev->log_file); + dev->log_file = NULL; +} + +static void lbd_destroy_device(struct lbd_device *dev) +{ + dev->state = LBD_STATE_REMOVING; + + cancel_delayed_work_sync(&dev->log_rotate_dwork); + + device_remove_group(disk_to_dev(dev->gd), &lbd_attr_group); + del_gendisk(dev->gd); + flush_workqueue(dev->wq); + destroy_workqueue(dev->wq); + + lbd_finalize_log(dev); + + if (dev->base) + lbd_qcow2_base_destroy(dev->base); + + if (dev->is_qcow2) + lbd_qcow2_destroy(dev); + + kvfree(dev->lz4_state); + kvfree(dev->log_buf); + path_put(&dev->log_dir_path); + if (dev->backing_file) { + vfs_fsync(dev->backing_file, 0); + fput(dev->backing_file); + } + + put_disk(dev->gd); + blk_mq_free_tag_set(&dev->tag_set); + kfree(dev); +} + +static int lbd_add_device(struct lbd_ctl_add __user *uarg) +{ + struct lbd_ctl_add arg; + struct lbd_device *dev; + struct inode *inode; + int ret, idx; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + arg.path[LBD_LOG_PATH_MAX - 1] = '\0'; + arg.log_dir[LBD_LOG_PATH_MAX - 1] = '\0'; + arg.base_path[LBD_LOG_PATH_MAX - 1] = '\0'; + + if (arg.log_dir[0] == '\0') { + pr_err("lbd: log_dir is required\n"); + return -EINVAL; + } + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + strscpy(dev->backing_path, arg.path, sizeof(dev->backing_path)); + strscpy(dev->log_dir, arg.log_dir, sizeof(dev->log_dir)); + dev->state = LBD_STATE_UNBOUND; + spin_lock_init(&dev->cmd_lock); +#ifdef CONFIG_LBD_MISS_HANDLER + spin_lock_init(&dev->miss_handler_lock); +#endif + INIT_LIST_HEAD(&dev->cmd_list); + INIT_WORK(&dev->work, lbd_work_fn); + mutex_init(&dev->log_mutex); + dev->log_seq = 0; + dev->log_has_entries = false; + dev->log_max_size = arg.log_max_size ? arg.log_max_size + : LBD_LOG_MAX_SIZE_DEFAULT; + dev->log_max_age_secs = arg.log_max_age_secs ? arg.log_max_age_secs + : LBD_LOG_MAX_AGE_DEFAULT; + INIT_DELAYED_WORK(&dev->log_rotate_dwork, lbd_log_rotate_work_fn); + + dev->log_buf = kvmalloc(LBD_LOG_BUF_SIZE, GFP_KERNEL); + if (!dev->log_buf) { + ret = -ENOMEM; + goto err_free; + } + dev->log_buf_used = 0; + + dev->lz4_state = kvmalloc(LZ4_sizeofState(), GFP_KERNEL); + if (!dev->lz4_state) { + ret = -ENOMEM; + goto err_free; + } + + /* Allocate device index */ + mutex_lock(&lbd_devices_mutex); + idx = idr_alloc(&lbd_devices, dev, 0, 256, GFP_KERNEL); + mutex_unlock(&lbd_devices_mutex); + if (idx < 0) { + ret = idx; + goto err_free; + } + dev->index = idx; + + /* Setup blk-mq tag set */ + memset(&dev->tag_set, 0, sizeof(dev->tag_set)); + dev->tag_set.ops = &lbd_mq_ops; + dev->tag_set.nr_hw_queues = 1; + dev->tag_set.queue_depth = LBD_QUEUE_DEPTH; + dev->tag_set.numa_node = NUMA_NO_NODE; + dev->tag_set.cmd_size = sizeof(struct lbd_cmd); +#if LBD_HAS_QUEUE_LIMITS_API + dev->tag_set.flags = 0; +#else + dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; +#endif + + ret = blk_mq_alloc_tag_set(&dev->tag_set); + if (ret) + goto err_idr; + + /* Allocate gendisk - version dependent */ +#if LBD_HAS_QUEUE_LIMITS_API + { + struct queue_limits lim = { + .logical_block_size = LBD_BLOCK_SIZE, + .physical_block_size = LBD_BLOCK_SIZE, + .max_hw_sectors = 256, + .features = BLK_FEAT_WRITE_CACHE, + .max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT, + .discard_granularity = LBD_BLOCK_SIZE, + }; + dev->gd = blk_mq_alloc_disk(&dev->tag_set, &lim, dev); + } + if (IS_ERR(dev->gd)) { + ret = PTR_ERR(dev->gd); + dev->gd = NULL; + goto err_tagset; + } +#elif LBD_HAS_BLK_MQ_ALLOC_DISK + dev->gd = blk_mq_alloc_disk(&dev->tag_set, dev); + if (IS_ERR(dev->gd)) { + ret = PTR_ERR(dev->gd); + dev->gd = NULL; + goto err_tagset; + } +#else + { + struct request_queue *q; + q = blk_mq_init_queue(&dev->tag_set); + if (IS_ERR(q)) { + ret = PTR_ERR(q); + goto err_tagset; + } + dev->gd = alloc_disk(1); + if (!dev->gd) { + blk_cleanup_queue(q); + ret = -ENOMEM; + goto err_tagset; + } + dev->gd->queue = q; + } +#endif + + dev->gd->major = lbd_major; + dev->gd->first_minor = dev->index; + dev->gd->minors = 1; + dev->gd->fops = &lbd_fops; + dev->gd->private_data = dev; + snprintf(dev->gd->disk_name, DISK_NAME_LEN, "lbd%d", dev->index); + +#if !LBD_HAS_BLK_MQ_ALLOC_DISK + dev->gd->queue->queuedata = dev; +#endif + + /* Create workqueue */ + dev->wq = alloc_workqueue("lbd%d", WQ_UNBOUND | WQ_MEM_RECLAIM, 0, + dev->index); + if (!dev->wq) { + ret = -ENOMEM; + goto err_disk; + } + + /* Open backing file */ + dev->backing_file = filp_open(arg.path, O_RDWR | O_LARGEFILE, 0); + if (IS_ERR(dev->backing_file)) { + ret = PTR_ERR(dev->backing_file); + dev->backing_file = NULL; + pr_err("lbd: cannot open backing file '%s': %d\n", arg.path, ret); + goto err_wq; + } + + inode = file_inode(dev->backing_file); + if (!S_ISREG(inode->i_mode)) { + ret = -EINVAL; + pr_err("lbd: backing path must be a regular file\n"); + goto err_backing; + } + + dev->size = i_size_read(inode); + if (dev->size == 0) { + ret = -EINVAL; + pr_err("lbd: backing file is empty\n"); + goto err_backing; + } + + /* Detect qcow2-lz4 format */ + { + u64 magic; + loff_t magic_pos = 0; + ssize_t mret; + + mret = kernel_read(dev->backing_file, &magic, 8, &magic_pos); + if (mret == 8 && be64_to_cpu(magic) == LBD_QCOW2_MAGIC) { + ret = lbd_qcow2_init(dev); + if (ret) + goto err_backing; + dev->is_qcow2 = true; + dev->size = dev->qcow2.virtual_size; + } else { + dev->is_qcow2 = false; + } + } + + /* Initialize base layer if requested */ + if (arg.base_path[0] != '\0') { + if (!dev->is_qcow2) { + pr_err("lbd: base layer requires qcow2-lz4 primary\n"); + ret = -EINVAL; + goto err_backing; + } + strscpy(dev->base_path, arg.base_path, + sizeof(dev->base_path)); + ret = lbd_qcow2_base_init(dev, arg.base_path); + if (ret) + goto err_backing; + } + + /* Resolve log directory */ + ret = kern_path(dev->log_dir, LOOKUP_DIRECTORY, &dev->log_dir_path); + if (ret) { + pr_err("lbd: cannot resolve log directory '%s': %d\n", + dev->log_dir, ret); + goto err_base; + } + + /* Begin initial log segment (file created lazily on first flush) */ + lbd_begin_segment(dev); + + /* Set capacity and activate */ + set_capacity(dev->gd, dev->size >> SECTOR_SHIFT); + + /* Set queue limits (on 6.9+ these are set via struct queue_limits above) */ +#if !LBD_HAS_QUEUE_LIMITS_API + blk_queue_logical_block_size(dev->gd->queue, LBD_BLOCK_SIZE); + blk_queue_physical_block_size(dev->gd->queue, LBD_BLOCK_SIZE); + blk_queue_max_hw_sectors(dev->gd->queue, 256); /* 128K max per request */ + blk_queue_write_cache(dev->gd->queue, true, false); + + blk_queue_max_discard_sectors(dev->gd->queue, UINT_MAX >> SECTOR_SHIFT); + dev->gd->queue->limits.discard_granularity = LBD_BLOCK_SIZE; +#endif + + dev->state = LBD_STATE_BOUND; + + ret = add_disk(dev->gd); + if (ret) + goto err_logdir; + + ret = device_add_group(disk_to_dev(dev->gd), &lbd_attr_group); + if (ret) + pr_warn("lbd%d: failed to create sysfs group: %d\n", + dev->index, ret); + + /* Start age-based rotation timer */ + if (dev->log_max_age_secs > 0) + schedule_delayed_work(&dev->log_rotate_dwork, + msecs_to_jiffies(dev->log_max_age_secs * 1000)); + + /* Return the assigned index to userspace */ + arg.index = dev->index; + if (copy_to_user(uarg, &arg, sizeof(arg))) { + /* Device is live - must tear down */ + lbd_destroy_device(dev); + mutex_lock(&lbd_devices_mutex); + idr_remove(&lbd_devices, idx); + mutex_unlock(&lbd_devices_mutex); + return -EFAULT; + } + + pr_info("lbd%d: attached to %s (%lld bytes)\n", + dev->index, dev->backing_path, dev->size); + return 0; + +err_logdir: + path_put(&dev->log_dir_path); +err_base: + if (dev->base) + lbd_qcow2_base_destroy(dev->base); +err_backing: + if (dev->is_qcow2) + lbd_qcow2_destroy(dev); + fput(dev->backing_file); +err_wq: + destroy_workqueue(dev->wq); +err_disk: +#if !LBD_HAS_BLK_MQ_ALLOC_DISK + blk_cleanup_queue(dev->gd->queue); +#endif + put_disk(dev->gd); +err_tagset: + blk_mq_free_tag_set(&dev->tag_set); +err_idr: + mutex_lock(&lbd_devices_mutex); + idr_remove(&lbd_devices, idx); + mutex_unlock(&lbd_devices_mutex); +err_free: + kvfree(dev->lz4_state); + kvfree(dev->log_buf); + kfree(dev); + return ret; +} + +static int lbd_remove_device(struct lbd_ctl_remove __user *uarg) +{ + struct lbd_ctl_remove arg; + struct lbd_device *dev; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, arg.index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + if (dev->state != LBD_STATE_BOUND) { + mutex_unlock(&lbd_devices_mutex); + return -EBUSY; + } + if (atomic_read(&dev->open_count) > 0) { + mutex_unlock(&lbd_devices_mutex); + return -EBUSY; + } + idr_remove(&lbd_devices, arg.index); + mutex_unlock(&lbd_devices_mutex); + + pr_info("lbd%d: detaching\n", dev->index); + lbd_destroy_device(dev); + return 0; +} + +static int lbd_info_device(struct lbd_ctl_info __user *uarg) +{ + struct lbd_ctl_info info; + struct lbd_device *dev; + + if (copy_from_user(&info, uarg, sizeof(info))) + return -EFAULT; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, info.index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + + info.state = dev->state; + info.size = dev->size; + strscpy(info.path, dev->backing_path, sizeof(info.path)); + mutex_unlock(&lbd_devices_mutex); + + if (copy_to_user(uarg, &info, sizeof(info))) + return -EFAULT; + return 0; +} + +/* ---------------------------------------------------------------- + * Sysfs stats + * ---------------------------------------------------------------- */ + +static ssize_t backing_path_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->backing_path); +} +static DEVICE_ATTR_RO(backing_path); + +static ssize_t base_path_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->base ? dev->base_path : "(none)"); +} +static DEVICE_ATTR_RO(base_path); + +static ssize_t log_dir_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->log_dir); +} +static DEVICE_ATTR_RO(log_dir); + +static ssize_t state_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + const char *s; + + switch (dev->state) { + case LBD_STATE_UNBOUND: s = "unbound"; break; + case LBD_STATE_BOUND: s = "bound"; break; + case LBD_STATE_REMOVING: s = "removing"; break; + default: s = "unknown"; break; + } + return sysfs_emit(buf, "%s\n", s); +} +static DEVICE_ATTR_RO(state); + +static ssize_t device_size_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", dev->size); +} +static DEVICE_ATTR_RO(device_size); + +static ssize_t reads_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_reads)); +} +static DEVICE_ATTR_RO(reads); + +static ssize_t writes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_writes)); +} +static DEVICE_ATTR_RO(writes); + +static ssize_t trims_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_trims)); +} +static DEVICE_ATTR_RO(trims); + +static ssize_t read_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_read_bytes)); +} +static DEVICE_ATTR_RO(read_bytes); + +static ssize_t write_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_write_bytes)); +} +static DEVICE_ATTR_RO(write_bytes); + +static ssize_t trim_bytes_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_trim_bytes)); +} +static DEVICE_ATTR_RO(trim_bytes); + +static ssize_t alloc_reused_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_reused)); +} +static DEVICE_ATTR_RO(alloc_reused); + +static ssize_t alloc_new_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_new)); +} +static DEVICE_ATTR_RO(alloc_new); + +static ssize_t alloc_freed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_alloc_freed)); +} +static DEVICE_ATTR_RO(alloc_freed); + +static ssize_t compressed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_compressed)); +} +static DEVICE_ATTR_RO(compressed); + +static ssize_t uncompressed_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_uncompressed)); +} +static DEVICE_ATTR_RO(uncompressed); + +static ssize_t log_seq_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%llu\n", READ_ONCE(dev->log_seq)); +} +static DEVICE_ATTR_RO(log_seq); + +static ssize_t log_segment_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%s\n", dev->log_segment_label); +} +static DEVICE_ATTR_RO(log_segment); + +static ssize_t log_rotations_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%lld\n", atomic64_read(&dev->stat_log_rotations)); +} +static DEVICE_ATTR_RO(log_rotations); + +static ssize_t log_buf_used_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + + return sysfs_emit(buf, "%zu\n", READ_ONCE(dev->log_buf_used)); +} +static DEVICE_ATTR_RO(log_buf_used); + +static ssize_t segment_age_secs_show(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct lbd_device *dev = dev_to_disk(d)->private_data; + s64 age = ktime_to_ms(ktime_sub(ktime_get(), dev->segment_start_time)); + + return sysfs_emit(buf, "%lld\n", age / 1000); +} +static DEVICE_ATTR_RO(segment_age_secs); + +static struct attribute *lbd_attrs[] = { + &dev_attr_backing_path.attr, + &dev_attr_base_path.attr, + &dev_attr_log_dir.attr, + &dev_attr_state.attr, + &dev_attr_device_size.attr, + &dev_attr_reads.attr, + &dev_attr_writes.attr, + &dev_attr_trims.attr, + &dev_attr_read_bytes.attr, + &dev_attr_write_bytes.attr, + &dev_attr_trim_bytes.attr, + &dev_attr_alloc_reused.attr, + &dev_attr_alloc_new.attr, + &dev_attr_alloc_freed.attr, + &dev_attr_compressed.attr, + &dev_attr_uncompressed.attr, + &dev_attr_log_seq.attr, + &dev_attr_log_segment.attr, + &dev_attr_log_rotations.attr, + &dev_attr_log_buf_used.attr, + &dev_attr_segment_age_secs.attr, + NULL, +}; + +static const struct attribute_group lbd_attr_group = { + .name = "lbd", + .attrs = lbd_attrs, +}; + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Miss handler helpers + * ---------------------------------------------------------------- */ + +static struct lbd_miss_handler *lbd_miss_handler_alloc(int dev_index) +{ + struct lbd_miss_handler *mh; + + mh = kzalloc(sizeof(*mh), GFP_KERNEL); + if (!mh) + return NULL; + + spin_lock_init(&mh->lock); + init_waitqueue_head(&mh->wq); + mh->dev_index = dev_index; + mh->pending = NULL; + mh->has_event = false; + return mh; +} + +/* + * Called from I/O workqueue when a cluster miss is detected. + * Posts the miss event to the handler and blocks until userspace responds. + * rwsem must be RELEASED before calling this. + */ +enum lbd_miss_action lbd_qcow2_handle_miss(struct lbd_device *dev, + u64 cluster_index) +{ + struct lbd_miss_handler *mh; + struct lbd_miss_pending pending; + + init_completion(&pending.done); + pending.action = LBD_MISS_CONTINUE; + + /* + * Hold miss_handler_lock while setting mh->pending to prevent + * lbd_ctl_release() from freeing mh between our read of the + * pointer and the registration of the pending completion. + * Once pending is registered, release will complete() it + * before kfree(mh), so mh stays alive until we wake up. + */ + spin_lock(&dev->miss_handler_lock); + mh = dev->miss_handler; + if (!mh) { + spin_unlock(&dev->miss_handler_lock); + return LBD_MISS_CONTINUE; + } + + spin_lock(&mh->lock); + mh->pending = &pending; + mh->miss_cluster = cluster_index; + mh->has_event = true; + spin_unlock(&mh->lock); + spin_unlock(&dev->miss_handler_lock); + + wake_up_interruptible(&mh->wq); + + /* Block I/O thread until userspace responds */ + wait_for_completion(&pending.done); + + return pending.action; +} +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* ---------------------------------------------------------------- + * Watcher helpers + * ---------------------------------------------------------------- */ + +static struct lbd_watcher *lbd_watcher_alloc(int filter_dev) +{ + struct lbd_watcher *w; + + w = kzalloc(sizeof(*w), GFP_KERNEL); + if (!w) + return NULL; + + spin_lock_init(&w->lock); + init_waitqueue_head(&w->wq); + w->filter_dev = filter_dev; + w->head = 0; + w->tail = 0; + w->count = 0; + return w; +} + +static void lbd_watcher_enqueue(struct lbd_watcher *w, + const struct lbd_watcher_event *ev) +{ + spin_lock(&w->lock); + if (w->count == LBD_WATCHER_QUEUE_SIZE) { + /* Overflow: drop oldest */ + w->head = (w->head + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count--; + } + w->queue[w->tail] = *ev; + w->tail = (w->tail + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count++; + spin_unlock(&w->lock); + wake_up_interruptible(&w->wq); +} + +/* Copy front event without removing (for encode-then-dequeue pattern) */ +static bool lbd_watcher_peek(struct lbd_watcher *w, + struct lbd_watcher_event *ev) +{ + bool got; + + spin_lock(&w->lock); + got = w->count > 0; + if (got) + *ev = w->queue[w->head]; + spin_unlock(&w->lock); + return got; +} + +/* Remove front event after successful copy_to_user */ +static void lbd_watcher_pop(struct lbd_watcher *w) +{ + spin_lock(&w->lock); + if (w->count > 0) { + w->head = (w->head + 1) % LBD_WATCHER_QUEUE_SIZE; + w->count--; + } + spin_unlock(&w->lock); +} + +/* + * Notify all watchers of a log rotation event. + * Called with dev->log_mutex held (sleepable). + * Acquires lbd_watchers_lock (spin), then w->lock (spin). + */ +static void lbd_notify_watchers(struct lbd_device *dev) +{ + struct lbd_watcher_event ev; + struct lbd_watcher *w; + ev.dev_index = dev->index; + ev.log_seq = dev->log_seq; + ev.device_size = dev->size; + memcpy(ev.segment_label, dev->log_segment_label, + sizeof(ev.segment_label)); + strscpy(ev.log_dir, dev->log_dir, sizeof(ev.log_dir)); + + spin_lock(&lbd_watchers_lock); + list_for_each_entry(w, &lbd_watchers, list) { + if (w->filter_dev >= 0 && w->filter_dev != dev->index) + continue; + lbd_watcher_enqueue(w, &ev); + } + spin_unlock(&lbd_watchers_lock); +} + +/* ---------------------------------------------------------------- + * Control device: watcher fops + * ---------------------------------------------------------------- */ + +static int lbd_ctl_open(struct inode *inode, struct file *file) +{ + struct lbd_ctl_state *state; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + file->private_data = state; + return 0; +} + +static int lbd_ctl_release(struct inode *inode, struct file *file) +{ + struct lbd_ctl_state *state = file->private_data; + + if (!state) + return 0; + + if (state->watcher) { + spin_lock(&lbd_watchers_lock); + list_del(&state->watcher->list); + spin_unlock(&lbd_watchers_lock); + kfree(state->watcher); + } + +#ifdef CONFIG_LBD_MISS_HANDLER + if (state->miss) { + struct lbd_miss_handler *mh = state->miss; + struct lbd_device *dev; + + /* Find the device and clear its handler pointer */ + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, mh->dev_index); + if (dev) { + spin_lock(&dev->miss_handler_lock); + if (dev->miss_handler == mh) + dev->miss_handler = NULL; + spin_unlock(&dev->miss_handler_lock); + } + mutex_unlock(&lbd_devices_mutex); + + /* Complete any pending miss with CONTINUE */ + spin_lock(&mh->lock); + if (mh->pending) { + mh->pending->action = LBD_MISS_CONTINUE; + complete(&mh->pending->done); + mh->pending = NULL; + } + spin_unlock(&mh->lock); + + kfree(mh); + } +#endif + + kfree(state); + file->private_data = NULL; + return 0; +} + +static ssize_t lbd_ctl_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct lbd_ctl_state *state = file->private_data; + u8 kbuf[512]; + struct cbor_dec d; + u64 map_count, key; + char cmd[16]; + int filter_dev = -1; + bool have_cmd = false; + char path[LBD_LOG_PATH_MAX]; + bool have_path = false; + u64 i; + + if (!state) + return -EINVAL; + + if (count > sizeof(kbuf)) + return -EINVAL; + + if (copy_from_user(kbuf, ubuf, count)) + return -EFAULT; + + cbor_dec_init(&d, kbuf, count); + + if (cbor_dec_map(&d, &map_count)) + return -EINVAL; + + path[0] = '\0'; + + for (i = 0; i < map_count; i++) { + if (cbor_dec_uint(&d, &key)) + return -EINVAL; + + switch (key) { + case LBD_WATCH_KEY_CMD: + if (cbor_dec_text(&d, cmd, sizeof(cmd), NULL)) + return -EINVAL; + have_cmd = true; + break; + case LBD_WATCH_KEY_DEV: { + u64 dev_val; + if (cbor_dec_uint(&d, &dev_val)) + return -EINVAL; + filter_dev = (int)dev_val; + break; + } + case LBD_WATCH_KEY_PATH: + if (cbor_dec_text(&d, path, sizeof(path), NULL)) + return -EINVAL; + have_path = true; + break; + default: + return -EINVAL; + } + } + + if (!have_cmd) + return -EINVAL; + + if (strcmp(cmd, "watch") == 0) { + struct lbd_watcher *w; + + if (state->watcher) + return -EBUSY; + + w = lbd_watcher_alloc(filter_dev); + if (!w) + return -ENOMEM; + + spin_lock(&lbd_watchers_lock); + list_add_tail(&w->list, &lbd_watchers); + spin_unlock(&lbd_watchers_lock); + + state->watcher = w; + return count; + } + +#ifdef CONFIG_LBD_MISS_HANDLER + if (strcmp(cmd, "manage_misses") == 0) { + struct lbd_device *dev; + struct lbd_miss_handler *mh; + + if (state->miss) + return -EBUSY; + if (filter_dev < 0) + return -EINVAL; + + mh = lbd_miss_handler_alloc(filter_dev); + if (!mh) + return -ENOMEM; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, filter_dev); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -ENODEV; + } + if (!dev->is_qcow2) { + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -EINVAL; + } + + spin_lock(&dev->miss_handler_lock); + if (dev->miss_handler) { + spin_unlock(&dev->miss_handler_lock); + mutex_unlock(&lbd_devices_mutex); + kfree(mh); + return -EBUSY; + } + dev->miss_handler = mh; + spin_unlock(&dev->miss_handler_lock); + mutex_unlock(&lbd_devices_mutex); + + state->miss = mh; + return count; + } + + if (strcmp(cmd, "continue") == 0) { + struct lbd_miss_handler *mh = state->miss; + + if (!mh) + return -EINVAL; + + spin_lock(&mh->lock); + if (!mh->pending) { + spin_unlock(&mh->lock); + return -EINVAL; + } + mh->pending->action = LBD_MISS_CONTINUE; + complete(&mh->pending->done); + mh->pending = NULL; + mh->has_event = false; + spin_unlock(&mh->lock); + + return count; + } + + if (strcmp(cmd, "retry") == 0) { + struct lbd_miss_handler *mh = state->miss; + + if (!mh) + return -EINVAL; + + spin_lock(&mh->lock); + if (!mh->pending) { + spin_unlock(&mh->lock); + return -EINVAL; + } + mh->pending->action = LBD_MISS_RETRY; + complete(&mh->pending->done); + mh->pending = NULL; + mh->has_event = false; + spin_unlock(&mh->lock); + + return count; + } + + if (strcmp(cmd, "swap") == 0) { + struct lbd_device *dev; + struct lbd_miss_handler *mh = state->miss; + int ret; + + if (!mh) + return -EINVAL; + if (!have_path || path[0] == '\0') + return -EINVAL; + + mutex_lock(&lbd_devices_mutex); + dev = idr_find(&lbd_devices, mh->dev_index); + if (!dev) { + mutex_unlock(&lbd_devices_mutex); + return -ENODEV; + } + mutex_unlock(&lbd_devices_mutex); + + ret = lbd_qcow2_swap_base(dev, path); + if (ret) + return ret; + + return count; + } +#endif /* CONFIG_LBD_MISS_HANDLER */ + + return -EINVAL; +} + +static bool lbd_ctl_has_event(struct lbd_ctl_state *state) +{ + struct lbd_watcher *w = state->watcher; + struct lbd_watcher_event ev; + +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh = state->miss; + + if (mh) { + bool has; + + spin_lock(&mh->lock); + has = mh->has_event; + spin_unlock(&mh->lock); + if (has) + return true; + } +#endif + if (w && lbd_watcher_peek(w, &ev)) + return true; + return false; +} + +static ssize_t lbd_ctl_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct lbd_ctl_state *state = file->private_data; + struct lbd_watcher *w; + u8 tmp[512]; + struct cbor_enc e; + size_t encoded_len; +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh; +#endif + + if (!state) + return -EINVAL; + + w = state->watcher; + +#ifdef CONFIG_LBD_MISS_HANDLER + mh = state->miss; + + if (!mh && !w) + return -EINVAL; +#else + if (!w) + return -EINVAL; +#endif + + /* Block until an event is available */ + if (!lbd_ctl_has_event(state)) { + int ret; + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + +#ifdef CONFIG_LBD_MISS_HANDLER + if (mh) { + ret = wait_event_interruptible(mh->wq, + lbd_ctl_has_event(state)); + } else { + ret = wait_event_interruptible(w->wq, + lbd_ctl_has_event(state)); + } +#else + ret = wait_event_interruptible(w->wq, + lbd_ctl_has_event(state)); +#endif + if (ret) + return -ERESTARTSYS; + } + +#ifdef CONFIG_LBD_MISS_HANDLER + /* Miss events take priority (time-critical, I/O thread blocked) */ + if (mh) { + bool has; + + spin_lock(&mh->lock); + has = mh->has_event; + spin_unlock(&mh->lock); + + if (has) { + u64 cluster; + + spin_lock(&mh->lock); + cluster = mh->miss_cluster; + spin_unlock(&mh->lock); + + cbor_enc_init(&e, tmp, sizeof(tmp)); + cbor_enc_map(&e, 3); + + cbor_enc_uint(&e, LBD_MISS_KEY_TYPE); + cbor_enc_text(&e, "block_miss", 10); + + cbor_enc_uint(&e, LBD_MISS_KEY_DEV); + cbor_enc_uint(&e, (u64)mh->dev_index); + + cbor_enc_uint(&e, LBD_MISS_KEY_CLUSTER); + cbor_enc_uint(&e, cluster); + + if (e.err) + return -EINVAL; + + encoded_len = cbor_enc_len(&e); + if (count < encoded_len) + return -EINVAL; + + if (copy_to_user(ubuf, tmp, encoded_len)) + return -EFAULT; + + return encoded_len; + } + } +#endif /* CONFIG_LBD_MISS_HANDLER */ + + /* Fall through to watcher events */ + if (w) { + struct lbd_watcher_event ev; + size_t label_len, dir_len; + + if (!lbd_watcher_peek(w, &ev)) { + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(w->wq, + lbd_watcher_peek(w, &ev))) + return -ERESTARTSYS; + } + + cbor_enc_init(&e, tmp, sizeof(tmp)); + cbor_enc_map(&e, 6); + + cbor_enc_uint(&e, LBD_EVENT_KEY_TYPE); + cbor_enc_text(&e, "log_rotated", 11); + + cbor_enc_uint(&e, LBD_EVENT_KEY_DEV); + cbor_enc_uint(&e, (u64)ev.dev_index); + + cbor_enc_uint(&e, LBD_EVENT_KEY_LABEL); + label_len = strlen(ev.segment_label); + cbor_enc_text(&e, ev.segment_label, label_len); + + cbor_enc_uint(&e, LBD_EVENT_KEY_DIR); + dir_len = strlen(ev.log_dir); + cbor_enc_text(&e, ev.log_dir, dir_len); + + cbor_enc_uint(&e, LBD_EVENT_KEY_SEQ); + cbor_enc_uint(&e, ev.log_seq); + + cbor_enc_uint(&e, LBD_EVENT_KEY_SIZE); + cbor_enc_uint(&e, ev.device_size); + + if (e.err) + return -EINVAL; + + encoded_len = cbor_enc_len(&e); + if (count < encoded_len) + return -EINVAL; + + if (copy_to_user(ubuf, tmp, encoded_len)) + return -EFAULT; + + lbd_watcher_pop(w); + return encoded_len; + } + + return -EAGAIN; +} + +static __poll_t lbd_ctl_poll(struct file *file, + struct poll_table_struct *wait) +{ + struct lbd_ctl_state *state = file->private_data; + struct lbd_watcher *w; + __poll_t mask = 0; +#ifdef CONFIG_LBD_MISS_HANDLER + struct lbd_miss_handler *mh; +#endif + + if (!state) + return 0; + + w = state->watcher; + +#ifdef CONFIG_LBD_MISS_HANDLER + mh = state->miss; + + if (!mh && !w) + return 0; + + if (mh) + poll_wait(file, &mh->wq, wait); +#else + if (!w) + return 0; +#endif + if (w) + poll_wait(file, &w->wq, wait); + +#ifdef CONFIG_LBD_MISS_HANDLER + if (mh) { + spin_lock(&mh->lock); + if (mh->has_event) + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock(&mh->lock); + } +#endif + + if (w) { + spin_lock(&w->lock); + if (w->count > 0) + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock(&w->lock); + } + + return mask; +} + +/* ---------------------------------------------------------------- + * Control device (misc device) + * ---------------------------------------------------------------- */ + +static long lbd_ctl_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case LBD_CTL_ADD: + return lbd_add_device((struct lbd_ctl_add __user *)arg); + case LBD_CTL_REMOVE: + return lbd_remove_device((struct lbd_ctl_remove __user *)arg); + case LBD_CTL_INFO: + return lbd_info_device((struct lbd_ctl_info __user *)arg); + default: + return -ENOTTY; + } +} + +static const struct file_operations lbd_ctl_fops = { + .owner = THIS_MODULE, + .open = lbd_ctl_open, + .release = lbd_ctl_release, + .read = lbd_ctl_read, + .write = lbd_ctl_write, + .poll = lbd_ctl_poll, + .unlocked_ioctl = lbd_ctl_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice lbd_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = LBD_CTL_NAME, + .fops = &lbd_ctl_fops, +}; + +/* ---------------------------------------------------------------- + * Module init / exit + * ---------------------------------------------------------------- */ + +static int __init lbd_init(void) +{ + int ret; + + lbd_major = register_blkdev(0, LBD_NAME); + if (lbd_major < 0) { + pr_err("lbd: failed to register block device\n"); + return lbd_major; + } + + idr_init(&lbd_devices); + + ret = misc_register(&lbd_misc); + if (ret) { + pr_err("lbd: failed to register control device\n"); + idr_destroy(&lbd_devices); + unregister_blkdev(lbd_major, LBD_NAME); + return ret; + } + + pr_info("lbd: module loaded (major=%d)\n", lbd_major); + return 0; +} + +static void __exit lbd_exit(void) +{ + struct lbd_device *dev; + int id; + + misc_deregister(&lbd_misc); + + mutex_lock(&lbd_devices_mutex); + idr_for_each_entry(&lbd_devices, dev, id) { + idr_remove(&lbd_devices, id); + mutex_unlock(&lbd_devices_mutex); + lbd_destroy_device(dev); + mutex_lock(&lbd_devices_mutex); + } + mutex_unlock(&lbd_devices_mutex); + + idr_destroy(&lbd_devices); + unregister_blkdev(lbd_major, LBD_NAME); + + pr_info("lbd: module unloaded\n"); +} + +module_init(lbd_init); +module_exit(lbd_exit); diff --git a/third_party/lbd/src/lbd_qcow2.c b/third_party/lbd/src/lbd_qcow2.c new file mode 100644 index 000000000..d3e81388a --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2.c @@ -0,0 +1,1921 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LBD qcow2-lz4 backing store + * + * Implements a qcow2-inspired format with LZ4-compressed clusters + * as the backing store for LBD block devices. Data is organized in + * 64 KiB clusters with two-level (L1/L2) address translation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lbd.h" +#include "lbd_qcow2.h" +#include "lz4_kcompat.h" + +/* Forward declaration for base layer read (used by lbd_qcow2_cl_load) */ +static int lbd_qcow2_base_read_cluster(struct lbd_device *dev, + u64 cluster_index, void *buf); + +/* ---------------------------------------------------------------- + * Helpers + * ---------------------------------------------------------------- */ + +static inline u64 lbd_qcow2_lru_tick(struct lbd_qcow2 *q) +{ + return q->lru_tick++; +} + +/* Write header's alloc_offset field to disk */ +static int lbd_qcow2_write_alloc_offset(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[8]; + loff_t pos = LBD_QCOW2_OFF_ALLOC_OFFSET; + ssize_t ret; + + _qcow2_put64(buf, 0, q->alloc_offset); + ret = kernel_write(dev->backing_file, buf, 8, &pos); + if (ret != 8) + return ret < 0 ? ret : -EIO; + return 0; +} + +/* Write header's free_list_head field to disk */ +static int lbd_qcow2_write_free_list_head(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[8]; + loff_t pos = LBD_QCOW2_OFF_FREE_LIST; + ssize_t ret; + + _qcow2_put64(buf, 0, q->free_list_head); + ret = kernel_write(dev->backing_file, buf, 8, &pos); + if (ret != 8) + return ret < 0 ? ret : -EIO; + return 0; +} + +/* + * Compute the on-disk allocation size for an existing L2 entry. + * Returns 0 on error. + */ +static u64 lbd_qcow2_read_old_alloc_size(struct lbd_device *dev, u64 l2_entry) +{ + struct lbd_qcow2 *q = &dev->qcow2; + + if (l2_entry == 0) + return 0; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + u64 phys = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + __be32 comp_size_be; + u32 comp_size; + loff_t pos = phys; + ssize_t ret; + + ret = kernel_read(dev->backing_file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) + return 0; + + comp_size = be32_to_cpu(comp_size_be); + return ALIGN(sizeof(__be32) + comp_size, 4096); + } + + return q->cluster_size; +} + +/* + * Write a tombstone at a physical offset, prepending to the free list. + * extent_size is the total usable space at that location. + */ +static int lbd_qcow2_free_extent(struct lbd_device *dev, loff_t phys_offset, + u64 extent_size) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 buf[16]; + loff_t pos = phys_offset; + ssize_t ret; + + /* Tombstone marker */ + _qcow2_put32(buf, 0, LBD_QCOW2_FREE_TOMBSTONE); + /* Extent size */ + _qcow2_put32(buf, 4, (u32)extent_size); + /* Next free pointer */ + _qcow2_put64(buf, 8, q->free_list_head); + + ret = kernel_write(dev->backing_file, buf, 16, &pos); + if (ret != 16) + return ret < 0 ? ret : -EIO; + + q->free_list_head = phys_offset; + atomic64_inc(&dev->stat_alloc_freed); + return lbd_qcow2_write_free_list_head(dev); +} + +/* Maximum number of free list entries to scan when allocating */ +#define LBD_QCOW2_FREE_SCAN_LIMIT 8 + +/* Minimum free entry size (tombstone header) */ +#define LBD_QCOW2_FREE_ENTRY_MIN 16 + +/* + * Try to allocate space from the free list or append. + * needed: bytes required for the new on-disk data. + * Returns the physical offset to write at via *out_phys. + */ +static int lbd_qcow2_alloc_space(struct lbd_device *dev, u64 needed, + loff_t *out_phys) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t prev_phys = 0; + loff_t cur = q->free_list_head; + int scanned = 0; + int is_first = 1; + + while (cur != 0 && scanned < LBD_QCOW2_FREE_SCAN_LIMIT) { + u8 buf[16]; + loff_t pos = cur; + ssize_t ret; + u32 tombstone, extent_size; + u64 next_free; + + ret = kernel_read(dev->backing_file, buf, 16, &pos); + if (ret != 16) + break; + + tombstone = _qcow2_get32(buf, 0); + if (tombstone != LBD_QCOW2_FREE_TOMBSTONE) + break; + + extent_size = _qcow2_get32(buf, 4); + next_free = _qcow2_get64(buf, 8); + + if (extent_size >= needed) { + u64 remainder = extent_size - needed; + + /* Unlink this entry from the free list */ + if (is_first) { + q->free_list_head = next_free; + } else { + /* Update previous entry's next pointer */ + u8 nbuf[8]; + loff_t npos = prev_phys + 8; + + _qcow2_put64(nbuf, 0, next_free); + ret = kernel_write(dev->backing_file, nbuf, 8, + &npos); + if (ret != 8) + goto append; + } + + /* Split if remainder is large enough */ + if (remainder >= LBD_QCOW2_FREE_ENTRY_MIN) { + loff_t split_phys = cur + needed; + int err; + + /* + * Add the remainder back as a new free entry + * at the head of the free list. + */ + err = lbd_qcow2_free_extent(dev, split_phys, + remainder); + if (err) { + /* + * Non-fatal: we waste the remainder + * but the allocation itself is fine. + */ + lbd_qcow2_write_free_list_head(dev); + } + } else { + lbd_qcow2_write_free_list_head(dev); + } + + *out_phys = cur; + atomic64_inc(&dev->stat_alloc_reused); + return 0; + } + + prev_phys = cur; + cur = next_free; + is_first = 0; + scanned++; + } + +append: + *out_phys = q->alloc_offset; + q->alloc_offset += needed; + atomic64_inc(&dev->stat_alloc_new); + return 0; +} + +/* ---------------------------------------------------------------- + * L2 cache + * ---------------------------------------------------------------- */ + +/* Find or load an L2 table into cache, return pointer to cache entry */ +static struct lbd_l2_cache_entry * +lbd_qcow2_l2_get(struct lbd_device *dev, u32 l1_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_l2_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + /* Check cache for hit */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (e->valid && e->l1_index == l1_index) { + e->lru = lbd_qcow2_lru_tick(q); + return e; + } + } + + /* Cache miss: find LRU entry to evict */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* Flush dirty entry before evicting */ + if (best->valid && best->dirty) { + u64 l2_phys = q->l1_table[best->l1_index]; + if (l2_phys) { + loff_t pos = l2_phys; + __be64 *disk_l2; + int j; + ssize_t ret; + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (disk_l2) { + u8 *raw = (u8 *)disk_l2; + u32 crc; + __be32 crc_be; + + memset(disk_l2, 0, q->cluster_size); + for (j = 0; j < q->l2_entries; j++) + disk_l2[j] = cpu_to_be64(best->table[j]); + + /* CRC32C trailer */ + crc = ~crc32c(~0, raw, q->cluster_size - 4); + crc_be = cpu_to_be32(crc); + memcpy(raw + q->cluster_size - 4, &crc_be, 4); + + ret = kernel_write(dev->backing_file, disk_l2, + q->cluster_size, &pos); + if (ret != q->cluster_size) + pr_warn("lbd%d: L2 flush failed\n", + dev->index); + kvfree(disk_l2); + } + } + best->dirty = false; + } + + /* Load L2 table from disk */ + best->l1_index = l1_index; + best->dirty = false; + best->lru = lbd_qcow2_lru_tick(q); + + if (l1_index < q->l1_size && q->l1_table[l1_index] != 0) { + loff_t pos = q->l1_table[l1_index]; + __be64 *disk_l2; + ssize_t ret; + int j; + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (!disk_l2) { + best->valid = false; + return NULL; + } + + ret = kernel_read(dev->backing_file, disk_l2, + q->cluster_size, &pos); + if (ret != q->cluster_size) { + pr_warn("lbd%d: L2 read failed for l1[%u]\n", + dev->index, l1_index); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + + /* Verify CRC32C trailer */ + { + u8 *raw = (u8 *)disk_l2; + u32 stored_crc = _qcow2_get32(raw, q->cluster_size - 4); + u32 calc_crc = ~crc32c(~0, raw, q->cluster_size - 4); + + if (stored_crc != calc_crc) { + pr_warn("lbd%d: L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + dev->index, l1_index, + stored_crc, calc_crc); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + } + + for (j = 0; j < q->l2_entries; j++) + best->table[j] = be64_to_cpu(disk_l2[j]); + + kvfree(disk_l2); + } else { + /* Unallocated L2: all zeros */ + memset(best->table, 0, q->cluster_size); + } + + best->valid = true; + return best; +} + +/* Flush a specific dirty L2 entry to disk */ +static int lbd_qcow2_l2_flush(struct lbd_device *dev, + struct lbd_l2_cache_entry *e) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u64 l2_phys; + __be64 *disk_l2; + loff_t pos; + ssize_t ret; + int j; + + if (!e->valid || !e->dirty) + return 0; + + l2_phys = q->l1_table[e->l1_index]; + if (!l2_phys) + return -EIO; /* should not happen */ + + disk_l2 = kvmalloc(q->cluster_size, GFP_NOIO); + if (!disk_l2) + return -ENOMEM; + + memset(disk_l2, 0, q->cluster_size); + for (j = 0; j < q->l2_entries; j++) + disk_l2[j] = cpu_to_be64(e->table[j]); + + /* Compute and store CRC32C trailer at end of cluster */ + { + u8 *raw = (u8 *)disk_l2; + u32 crc = ~crc32c(~0, raw, q->cluster_size - 4); + __be32 crc_be = cpu_to_be32(crc); + + memcpy(raw + q->cluster_size - 4, &crc_be, 4); + } + + pos = l2_phys; + ret = kernel_write(dev->backing_file, disk_l2, q->cluster_size, &pos); + kvfree(disk_l2); + + if (ret != q->cluster_size) + return ret < 0 ? ret : -EIO; + + e->dirty = false; + return 0; +} + +/* Allocate a new L2 table on disk if needed */ +static int lbd_qcow2_l2_alloc(struct lbd_device *dev, u32 l1_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t pos; + __be64 val; + ssize_t ret; + + if (l1_index >= q->l1_size) + return -ENOSPC; + + if (q->l1_table[l1_index] != 0) + return 0; /* already allocated */ + + /* Allocate cluster for L2 table at append point */ + q->l1_table[l1_index] = q->alloc_offset; + q->alloc_offset += q->cluster_size; + + /* Write zeroed L2 table to disk (with CRC32C trailer) */ + { + u8 *zeros = kvmalloc(q->cluster_size, GFP_NOIO); + u32 crc; + __be32 crc_be; + + if (!zeros) + return -ENOMEM; + memset(zeros, 0, q->cluster_size); + + /* CRC32C of all-zero data (covers bytes [0, cluster_size-4)) */ + crc = ~crc32c(~0, zeros, q->cluster_size - 4); + crc_be = cpu_to_be32(crc); + memcpy(zeros + q->cluster_size - 4, &crc_be, 4); + + pos = q->l1_table[l1_index]; + ret = kernel_write(dev->backing_file, zeros, + q->cluster_size, &pos); + kvfree(zeros); + if (ret != q->cluster_size) + return ret < 0 ? ret : -EIO; + } + + /* Write L1 entry to disk */ + val = cpu_to_be64(q->l1_table[l1_index]); + pos = q->l1_offset + (loff_t)l1_index * sizeof(__be64); + ret = kernel_write(dev->backing_file, &val, sizeof(val), &pos); + if (ret != sizeof(val)) + return ret < 0 ? ret : -EIO; + + return 0; +} + +/* ---------------------------------------------------------------- + * Cluster cache + * ---------------------------------------------------------------- */ + +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_find(struct lbd_qcow2 *q, u64 cluster_index) +{ + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) { + e->lru = lbd_qcow2_lru_tick(q); + return e; + } + } + return NULL; +} + +static void lbd_qcow2_cl_invalidate(struct lbd_qcow2 *q, u64 cluster_index) +{ + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) { + e->valid = false; + return; + } + } +} + +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_alloc_entry(struct lbd_qcow2 *q) +{ + struct lbd_cl_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* Evict LRU (cluster cache is read-only cache, no flush needed) */ + best->valid = false; + return best; +} + +/* + * Return a zeroed cache entry for write path when cluster is unallocated. + * Caller holds q->rwsem for write. + */ +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_get_zero(struct lbd_device *dev, u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_cl_cache_entry *ce; + + ce = lbd_qcow2_cl_find(q, cluster_index); + if (ce) + return ce; + + ce = lbd_qcow2_cl_alloc_entry(q); + ce->cluster_index = cluster_index; + ce->lru = lbd_qcow2_lru_tick(q); + ce->dirty = false; + memset(ce->data, 0, q->cluster_size); + ce->valid = true; + return ce; +} + +/* Load a cluster into cache from disk via L2 lookup */ +static struct lbd_cl_cache_entry * +lbd_qcow2_cl_load(struct lbd_device *dev, u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_cl_cache_entry *ce; + struct lbd_l2_cache_entry *l2e; + u32 l1_idx, l2_idx; + u64 l2_entry, phys_offset; + ssize_t ret; + + /* Check cache first */ + ce = lbd_qcow2_cl_find(q, cluster_index); + if (ce) + return ce; + + /* L2 lookup */ + l1_idx = cluster_index / q->l2_entries; + l2_idx = cluster_index % q->l2_entries; + + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) + return NULL; + + l2_entry = l2e->table[l2_idx]; + + /* Get a cache entry */ + ce = lbd_qcow2_cl_alloc_entry(q); + ce->cluster_index = cluster_index; + ce->lru = lbd_qcow2_lru_tick(q); + ce->dirty = false; + + if (l2_entry == 0) { + if (dev->base) { + int err = lbd_qcow2_base_read_cluster(dev, + cluster_index, ce->data); + if (err == 1) { + /* Unallocated in both primary and base */ +#ifdef CONFIG_LBD_MISS_HANDLER + if (dev->miss_handler) + return ERR_PTR(-ENODATA); +#endif + /* No handler: zero-fill (backward compatible) */ + memset(ce->data, 0, q->cluster_size); + } else if (err < 0) { + return NULL; /* I/O error */ + } + /* err == 0: data was read successfully */ +#ifdef CONFIG_LBD_MISS_HANDLER + } else if (dev->miss_handler) { + /* No base, but miss handler registered */ + return ERR_PTR(-ENODATA); +#endif + } else { + memset(ce->data, 0, q->cluster_size); + } + ce->valid = true; + return ce; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + /* Compressed cluster: read size header + compressed data */ + __be32 comp_size_be; + u32 comp_size; + int dec_len; + loff_t pos = phys_offset; + + ret = kernel_read(dev->backing_file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + pr_warn("lbd%d: failed to read compressed size\n", + dev->index); + return NULL; + } + + comp_size = be32_to_cpu(comp_size_be); + if (comp_size > LZ4_compressBound(q->cluster_size)) { + pr_warn("lbd%d: invalid compressed size %u\n", + dev->index, comp_size); + return NULL; + } + + ret = kernel_read(dev->backing_file, q->read_buf, + comp_size, &pos); + if (ret != comp_size) { + pr_warn("lbd%d: failed to read compressed data\n", + dev->index); + return NULL; + } + + dec_len = LZ4_decompress_safe(q->read_buf, ce->data, + comp_size, q->cluster_size); + if (dec_len != q->cluster_size) { + pr_warn("lbd%d: LZ4 decompress failed (%d)\n", + dev->index, dec_len); + return NULL; + } + } else { + /* Uncompressed cluster */ + loff_t pos = phys_offset; + + ret = kernel_read(dev->backing_file, ce->data, + q->cluster_size, &pos); + if (ret != q->cluster_size) { + pr_warn("lbd%d: failed to read cluster\n", + dev->index); + return NULL; + } + } + + ce->valid = true; + return ce; +} + +/* + * Flush all dirty L2 cache entries to disk. + */ +static void lbd_qcow2_flush_all_l2(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int i; + + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + if (e->valid && e->dirty) + lbd_qcow2_l2_flush(dev, e); + } +} + +/* ---------------------------------------------------------------- + * Init / Destroy + * ---------------------------------------------------------------- */ + +int lbd_qcow2_init(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + u8 *hdr; + loff_t pos = 0; + ssize_t ret; + __be64 *disk_l1; + int i; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_KERNEL); + if (!hdr) + return -ENOMEM; + + /* Read header */ + ret = kernel_read(dev->backing_file, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + pr_err("lbd%d: qcow2 header read failed\n", dev->index); + kvfree(hdr); + return ret < 0 ? ret : -EIO; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + pr_err("lbd%d: invalid qcow2 magic\n", dev->index); + kvfree(hdr); + return -EINVAL; + } + + if (lbd_qcow2_hdr_version(hdr) != LBD_QCOW2_VERSION) { + pr_err("lbd%d: unsupported qcow2 version %u\n", + dev->index, lbd_qcow2_hdr_version(hdr)); + kvfree(hdr); + return -EINVAL; + } + + q->cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + if (q->cluster_bits < 12 || q->cluster_bits > 24) { + pr_err("lbd%d: invalid cluster_bits %u\n", + dev->index, q->cluster_bits); + kvfree(hdr); + return -EINVAL; + } + + q->cluster_size = 1U << q->cluster_bits; + q->l2_entries = (q->cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / sizeof(u64); + q->virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + q->l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + q->l1_size = lbd_qcow2_hdr_l1_size(hdr); + q->alloc_offset = lbd_qcow2_hdr_alloc_offset(hdr); + q->free_list_head = lbd_qcow2_hdr_free_list(hdr); + q->lru_tick = 0; + + kvfree(hdr); + + init_rwsem(&q->rwsem); + + /* Allocate and read L1 table */ + q->l1_table = kvmalloc_array(q->l1_size, sizeof(u64), GFP_KERNEL); + if (!q->l1_table) + return -ENOMEM; + + disk_l1 = kvmalloc_array(q->l1_size, sizeof(__be64), GFP_KERNEL); + if (!disk_l1) { + kvfree(q->l1_table); + q->l1_table = NULL; + return -ENOMEM; + } + + pos = q->l1_offset; + ret = kernel_read(dev->backing_file, disk_l1, + q->l1_size * sizeof(__be64), &pos); + if (ret != q->l1_size * sizeof(__be64)) { + pr_err("lbd%d: L1 table read failed\n", dev->index); + kvfree(disk_l1); + kvfree(q->l1_table); + q->l1_table = NULL; + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < q->l1_size; i++) + q->l1_table[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + /* Allocate L2 cache entries */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + e->table = kvmalloc(q->cluster_size, GFP_KERNEL); + if (!e->table) + goto err_l2_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate cluster cache entries */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + e->data = kvmalloc(q->cluster_size, GFP_KERNEL); + if (!e->data) + goto err_cl_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate compression buffer */ + q->comp_buf = kvmalloc(LZ4_compressBound(q->cluster_size), GFP_KERNEL); + if (!q->comp_buf) + goto err_cl_cache; + + /* Allocate read buffer for compressed data */ + q->read_buf = kvmalloc(LZ4_compressBound(q->cluster_size), GFP_KERNEL); + if (!q->read_buf) { + kvfree(q->comp_buf); + q->comp_buf = NULL; + goto err_cl_cache; + } + + pr_info("lbd%d: qcow2-lz4 format detected, virtual_size=%llu, " + "cluster_size=%u, l1_size=%u\n", + dev->index, q->virtual_size, q->cluster_size, q->l1_size); + + return 0; + +err_cl_cache: + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + kvfree(q->cl_cache[i].data); +err_l2_cache: + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(q->l2_cache[i].table); + kvfree(q->l1_table); + q->l1_table = NULL; + return -ENOMEM; +} + +void lbd_qcow2_destroy(struct lbd_device *dev) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int i; + + /* Flush any dirty L2 entries */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &q->l2_cache[i]; + + if (e->valid && e->dirty) + lbd_qcow2_l2_flush(dev, e); + } + + /* Write final alloc_offset and free_list_head */ + lbd_qcow2_write_alloc_offset(dev); + lbd_qcow2_write_free_list_head(dev); + + kvfree(q->read_buf); + kvfree(q->comp_buf); + + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + kvfree(q->cl_cache[i].data); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(q->l2_cache[i].table); + + kvfree(q->l1_table); + q->l1_table = NULL; +} + +#ifdef CONFIG_LBD_MISS_HANDLER +/* ---------------------------------------------------------------- + * Miss handler support + * ---------------------------------------------------------------- */ + +/* Declared in lbd.c, defined there to access miss handler internals */ +enum lbd_miss_action lbd_qcow2_handle_miss(struct lbd_device *dev, + u64 cluster_index); + +/* + * Invalidate cached entries for a cluster before retry. + * Called with rwsem released. + */ +static void lbd_qcow2_invalidate_for_retry(struct lbd_device *dev, + u64 cluster_index) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_qcow2_base *base = dev->base; + int i; + + /* Invalidate primary cluster cache */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) { + struct lbd_cl_cache_entry *e = &q->cl_cache[i]; + + if (e->valid && e->cluster_index == cluster_index) + e->valid = false; + } + + /* Invalidate base L2 cache entry covering this cluster */ + if (base && base->is_qcow2) { + u32 l1_idx = cluster_index / base->l2_entries; + + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (e->valid && e->l1_index == l1_idx) + e->valid = false; + } + } +} +#endif /* CONFIG_LBD_MISS_HANDLER */ + +/* ---------------------------------------------------------------- + * Read path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_read(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct req_iterator iter; + struct bio_vec bvec; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + + rq_for_each_segment(bvec, rq, iter) { + void *mapped; + unsigned int remaining = bvec.bv_len; + unsigned int bv_off = bvec.bv_offset; + + mapped = kmap_local_page(bvec.bv_page); + + while (remaining > 0) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, remaining, + q->cluster_size - off_in_cluster); + struct lbd_cl_cache_entry *ce; +#ifdef CONFIG_LBD_MISS_HANDLER + int retries = 0; + +retry_cluster: +#endif + down_read(&q->rwsem); + + ce = lbd_qcow2_cl_load(dev, cluster_idx); +#ifdef CONFIG_LBD_MISS_HANDLER + if (IS_ERR(ce)) { + enum lbd_miss_action action; + + up_read(&q->rwsem); + /* -ENODATA = miss, ask userspace */ + action = lbd_qcow2_handle_miss(dev, + cluster_idx); + if (action == LBD_MISS_RETRY && + retries++ < 3) { + lbd_qcow2_invalidate_for_retry(dev, + cluster_idx); + goto retry_cluster; + } + /* CONTINUE or retry exhausted: zero-fill */ + memset(mapped + bv_off, 0, bytes); + goto next_chunk; + } +#endif + if (!ce) { + /* Real I/O error */ + up_read(&q->rwsem); + kunmap_local(mapped); + return -EIO; + } + + memcpy(mapped + bv_off, ce->data + off_in_cluster, + bytes); + + up_read(&q->rwsem); + +#ifdef CONFIG_LBD_MISS_HANDLER +next_chunk: +#endif + bv_off += bytes; + guest_offset += bytes; + remaining -= bytes; + } + + kunmap_local(mapped); + } + + atomic64_inc(&dev->stat_reads); + atomic64_add(blk_rq_bytes(rq), &dev->stat_read_bytes); + return 0; +} + +/* ---------------------------------------------------------------- + * Write path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_write(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct req_iterator iter; + struct bio_vec bvec; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + u32 total_len = blk_rq_bytes(rq); + void *write_data; + size_t offset = 0; + + /* Gather all write data into contiguous buffer */ + write_data = kvmalloc(total_len, GFP_NOIO); + if (!write_data) + return -ENOMEM; + + rq_for_each_segment(bvec, rq, iter) { + void *mapped = kmap_local_page(bvec.bv_page); + memcpy(write_data + offset, mapped + bvec.bv_offset, + bvec.bv_len); + kunmap_local(mapped); + offset += bvec.bv_len; + } + + down_write(&q->rwsem); + + offset = 0; + while (offset < total_len) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, total_len - offset, + q->cluster_size - off_in_cluster); + struct lbd_cl_cache_entry *ce; + struct lbd_l2_cache_entry *l2e; + u32 l1_idx = cluster_idx / q->l2_entries; + u32 l2_idx = cluster_idx % q->l2_entries; + int comp_len; + loff_t phys; + ssize_t ret; + int err; + u64 old_l2_entry; + u64 old_alloc, new_alloc; + + /* Ensure L2 table is allocated */ + err = lbd_qcow2_l2_alloc(dev, l1_idx); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + /* Load existing cluster into cache (or zeros if new) */ + ce = lbd_qcow2_cl_load(dev, cluster_idx); + if (IS_ERR(ce)) { + /* + * -ENODATA: cluster unallocated (miss). For writes we + * don't need the old data — just grab a zeroed cache + * slot and let the write overwrite it. + */ + ce = lbd_qcow2_cl_get_zero(dev, cluster_idx); + if (!ce) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + } else if (!ce) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + + /* Read old L2 entry before modifying */ + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) { + up_write(&q->rwsem); + kvfree(write_data); + return -EIO; + } + old_l2_entry = l2e->table[l2_idx]; + old_alloc = lbd_qcow2_read_old_alloc_size(dev, old_l2_entry); + + /* Apply write data to cluster */ + memcpy(ce->data + off_in_cluster, write_data + offset, bytes); + + /* Compress the full cluster */ + comp_len = LZ4_compress_fast_extState( + dev->lz4_state, ce->data, q->comp_buf, + q->cluster_size, + LZ4_compressBound(q->cluster_size), 1); + + if (comp_len > 0 && + (u32)comp_len < q->cluster_size - sizeof(__be32)) { + /* Store compressed */ + __be32 comp_size_be = cpu_to_be32(comp_len); + new_alloc = ALIGN(sizeof(__be32) + comp_len, 4096); + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2_entry & + LBD_QCOW2_L2_OFFSET_MASK; + err = lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + err = lbd_qcow2_alloc_space(dev, new_alloc, + &phys); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + + /* Write size header + compressed data */ + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + + ret = kernel_write(dev->backing_file, + q->comp_buf, comp_len, + &pos); + if (ret != comp_len) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + l2e->dirty = true; + atomic64_inc(&dev->stat_compressed); + } else { + /* Store uncompressed */ + new_alloc = q->cluster_size; + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2_entry & + LBD_QCOW2_L2_OFFSET_MASK; + err = lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + err = lbd_qcow2_alloc_space(dev, new_alloc, + &phys); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + ce->data, q->cluster_size, + &pos); + if (ret != q->cluster_size) { + up_write(&q->rwsem); + kvfree(write_data); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = phys; + l2e->dirty = true; + atomic64_inc(&dev->stat_uncompressed); + } + + /* Flush L2 to disk */ + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + /* Update alloc_offset on disk */ + err = lbd_qcow2_write_alloc_offset(dev); + if (err) { + up_write(&q->rwsem); + kvfree(write_data); + return err; + } + + offset += bytes; + guest_offset += bytes; + } + + up_write(&q->rwsem); + kvfree(write_data); + return 0; +} + +/* ---------------------------------------------------------------- + * TRIM path + * ---------------------------------------------------------------- */ + +int lbd_qcow2_discard(struct lbd_device *dev, struct request *rq) +{ + struct lbd_qcow2 *q = &dev->qcow2; + loff_t guest_offset = (loff_t)blk_rq_pos(rq) << SECTOR_SHIFT; + u32 remaining = blk_rq_bytes(rq); + int err; + + down_write(&q->rwsem); + + while (remaining > 0) { + u64 cluster_idx = guest_offset >> q->cluster_bits; + u32 off_in_cluster = guest_offset & (q->cluster_size - 1); + u32 bytes = min_t(u32, remaining, + q->cluster_size - off_in_cluster); + u32 l1_idx = cluster_idx / q->l2_entries; + u32 l2_idx = cluster_idx % q->l2_entries; + struct lbd_l2_cache_entry *l2e; + + if (off_in_cluster == 0 && bytes == q->cluster_size) { + /* Full-cluster trim */ + if (l1_idx < q->l1_size && + q->l1_table[l1_idx] != 0) { + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (l2e && l2e->table[l2_idx] != 0) { + u64 old_l2 = l2e->table[l2_idx]; + u64 old_sz = lbd_qcow2_read_old_alloc_size( + dev, old_l2); + if (old_sz > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_sz); + } + l2e->table[l2_idx] = 0; + l2e->dirty = true; + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + return err; + } + } + } + lbd_qcow2_cl_invalidate(q, cluster_idx); + } else { + /* Partial-cluster trim: read-modify-write */ + struct lbd_cl_cache_entry *ce; + + err = lbd_qcow2_l2_alloc(dev, l1_idx); + if (err) { + up_write(&q->rwsem); + return err; + } + + ce = lbd_qcow2_cl_load(dev, cluster_idx); + if (IS_ERR(ce)) { + /* + * -ENODATA: cluster unallocated. Trimming + * an unallocated cluster is a no-op. + */ + goto next; + } + if (!ce) { + up_write(&q->rwsem); + return -EIO; + } + + l2e = lbd_qcow2_l2_get(dev, l1_idx); + if (!l2e) { + up_write(&q->rwsem); + return -EIO; + } + + if (l2e->table[l2_idx] == 0) { + goto next; + } + + /* Zero the trimmed portion */ + memset(ce->data + off_in_cluster, 0, bytes); + + /* Recompress and write back */ + { + int comp_len; + loff_t phys; + ssize_t ret; + u64 old_l2 = l2e->table[l2_idx]; + u64 old_alloc = lbd_qcow2_read_old_alloc_size( + dev, old_l2); + u64 new_alloc; + + comp_len = LZ4_compress_fast_extState( + dev->lz4_state, ce->data, q->comp_buf, + q->cluster_size, + LZ4_compressBound(q->cluster_size), 1); + + if (comp_len > 0 && + (u32)comp_len < q->cluster_size - sizeof(__be32)) { + __be32 comp_size_be = cpu_to_be32(comp_len); + new_alloc = ALIGN(sizeof(__be32) + comp_len, 4096); + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2 & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + } + err = lbd_qcow2_alloc_space(dev, + new_alloc, &phys); + if (err) { + up_write(&q->rwsem); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + ret = kernel_write(dev->backing_file, + q->comp_buf, comp_len, + &pos); + if (ret != comp_len) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + atomic64_inc(&dev->stat_compressed); + } else { + new_alloc = q->cluster_size; + + if (old_alloc > 0 && new_alloc <= old_alloc) { + phys = old_l2 & LBD_QCOW2_L2_OFFSET_MASK; + atomic64_inc(&dev->stat_alloc_reused); + } else { + if (old_alloc > 0) { + loff_t old_phys = old_l2 & + LBD_QCOW2_L2_OFFSET_MASK; + lbd_qcow2_free_extent(dev, + old_phys, old_alloc); + } + err = lbd_qcow2_alloc_space(dev, + new_alloc, &phys); + if (err) { + up_write(&q->rwsem); + return err; + } + } + + { + loff_t pos = phys; + ret = kernel_write(dev->backing_file, + ce->data, q->cluster_size, + &pos); + if (ret != q->cluster_size) { + up_write(&q->rwsem); + return ret < 0 ? ret : -EIO; + } + } + + l2e->table[l2_idx] = phys; + atomic64_inc(&dev->stat_uncompressed); + } + + l2e->dirty = true; + err = lbd_qcow2_l2_flush(dev, l2e); + if (err) { + up_write(&q->rwsem); + return err; + } + + err = lbd_qcow2_write_alloc_offset(dev); + if (err) { + up_write(&q->rwsem); + return err; + } + } + } + +next: + guest_offset += bytes; + remaining -= bytes; + } + + up_write(&q->rwsem); + return 0; +} + +/* ---------------------------------------------------------------- + * Base layer (thin snapshot) support + * ---------------------------------------------------------------- */ + +static inline u64 lbd_qcow2_base_lru_tick(struct lbd_qcow2_base *base) +{ + return base->lru_tick++; +} + +/* Find or load an L2 table from the base layer into its cache */ +static struct lbd_l2_cache_entry * +lbd_qcow2_base_l2_get(struct lbd_qcow2_base *base, u32 l1_index) +{ + struct lbd_l2_cache_entry *best = NULL; + u64 oldest = U64_MAX; + int i; + + /* Check cache for hit */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (e->valid && e->l1_index == l1_index) { + e->lru = lbd_qcow2_base_lru_tick(base); + return e; + } + } + + /* Cache miss: find LRU entry to evict */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + if (!e->valid) { + best = e; + break; + } + if (e->lru < oldest) { + oldest = e->lru; + best = e; + } + } + + /* No dirty tracking — just overwrite on eviction */ + best->l1_index = l1_index; + best->dirty = false; + best->lru = lbd_qcow2_base_lru_tick(base); + + if (l1_index < base->l1_size && base->l1_table[l1_index] != 0) { + loff_t pos = base->l1_table[l1_index]; + __be64 *disk_l2; + ssize_t ret; + int j; + + disk_l2 = kvmalloc(base->cluster_size, GFP_NOIO); + if (!disk_l2) { + best->valid = false; + return NULL; + } + + ret = kernel_read(base->file, disk_l2, + base->cluster_size, &pos); + if (ret != base->cluster_size) { + pr_warn("lbd: base L2 read failed for l1[%u]\n", + l1_index); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + + /* Verify CRC32C trailer */ + { + u8 *raw = (u8 *)disk_l2; + u32 stored_crc = _qcow2_get32(raw, base->cluster_size - 4); + u32 calc_crc = ~crc32c(~0, raw, base->cluster_size - 4); + + if (stored_crc != calc_crc) { + pr_warn("lbd: base L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + l1_index, stored_crc, calc_crc); + kvfree(disk_l2); + best->valid = false; + return NULL; + } + } + + for (j = 0; j < base->l2_entries; j++) + best->table[j] = be64_to_cpu(disk_l2[j]); + + kvfree(disk_l2); + } else { + /* Unallocated L2: all zeros */ + memset(best->table, 0, base->cluster_size); + } + + best->valid = true; + return best; +} + +/* + * Read a full cluster from the base layer into buf. + * Returns 0 on success, negative errno on failure. + */ +static int lbd_qcow2_base_read_cluster(struct lbd_device *dev, + u64 cluster_index, void *buf) +{ + struct lbd_qcow2_base *base = dev->base; + u32 cluster_size = dev->qcow2.cluster_size; + + if (!base->is_qcow2) { + /* Raw base: direct read */ + loff_t pos = cluster_index * cluster_size; + ssize_t ret; + + if (pos + cluster_size > base->size) { + /* Unallocated: signal with return 1 */ + return 1; + } + + ret = kernel_read(base->file, buf, cluster_size, &pos); + if (ret != cluster_size) { + pr_warn("lbd%d: base raw read failed at cluster %llu\n", + dev->index, cluster_index); + return ret < 0 ? ret : -EIO; + } + return 0; + } + + /* qcow2 base: L1/L2 lookup */ + { + u32 l1_idx = cluster_index / base->l2_entries; + u32 l2_idx = cluster_index % base->l2_entries; + struct lbd_l2_cache_entry *l2e; + u64 l2_entry, phys_offset; + ssize_t ret; + + l2e = lbd_qcow2_base_l2_get(base, l1_idx); + if (!l2e) + return -EIO; + + l2_entry = l2e->table[l2_idx]; + + if (l2_entry == 0) { + /* Unallocated in base: signal with return 1 */ + return 1; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + __be32 comp_size_be; + u32 comp_size; + int dec_len; + loff_t pos = phys_offset; + + ret = kernel_read(base->file, &comp_size_be, + sizeof(comp_size_be), &pos); + if (ret != sizeof(comp_size_be)) { + pr_warn("lbd%d: base compressed size read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + + comp_size = be32_to_cpu(comp_size_be); + if (comp_size > LZ4_compressBound(cluster_size)) { + pr_warn("lbd%d: base invalid compressed size %u\n", + dev->index, comp_size); + return -EIO; + } + + ret = kernel_read(base->file, base->read_buf, + comp_size, &pos); + if (ret != comp_size) { + pr_warn("lbd%d: base compressed data read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + + dec_len = LZ4_decompress_safe(base->read_buf, buf, + comp_size, cluster_size); + if (dec_len != cluster_size) { + pr_warn("lbd%d: base LZ4 decompress failed (%d)\n", + dev->index, dec_len); + return -EIO; + } + } else { + /* Uncompressed cluster */ + loff_t pos = phys_offset; + + ret = kernel_read(base->file, buf, + cluster_size, &pos); + if (ret != cluster_size) { + pr_warn("lbd%d: base cluster read failed\n", + dev->index); + return ret < 0 ? ret : -EIO; + } + } + } + + return 0; +} + +/* ---------------------------------------------------------------- + * Swap base layer and reload L1 + * ---------------------------------------------------------------- */ + +/* + * Re-read the primary's L1 table from disk. + * Called with rwsem held for write. + */ +static int lbd_qcow2_reload_l1(struct lbd_qcow2 *q, struct lbd_device *dev) +{ + u8 *hdr; + loff_t pos = 0; + ssize_t ret; + __be64 *disk_l1; + u64 *new_l1; + u32 new_l1_size; + loff_t new_l1_offset; + int i; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_NOIO); + if (!hdr) + return -ENOMEM; + + ret = kernel_read(dev->backing_file, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + kvfree(hdr); + return ret < 0 ? ret : -EIO; + } + + new_l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + new_l1_size = lbd_qcow2_hdr_l1_size(hdr); + q->alloc_offset = lbd_qcow2_hdr_alloc_offset(hdr); + q->free_list_head = lbd_qcow2_hdr_free_list(hdr); + kvfree(hdr); + + new_l1 = kvmalloc_array(new_l1_size, sizeof(u64), GFP_NOIO); + if (!new_l1) + return -ENOMEM; + + disk_l1 = kvmalloc_array(new_l1_size, sizeof(__be64), GFP_NOIO); + if (!disk_l1) { + kvfree(new_l1); + return -ENOMEM; + } + + pos = new_l1_offset; + ret = kernel_read(dev->backing_file, disk_l1, + new_l1_size * sizeof(__be64), &pos); + if (ret != new_l1_size * sizeof(__be64)) { + kvfree(disk_l1); + kvfree(new_l1); + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < new_l1_size; i++) + new_l1[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + kvfree(q->l1_table); + q->l1_table = new_l1; + q->l1_size = new_l1_size; + q->l1_offset = new_l1_offset; + + return 0; +} + +/* + * Invalidate all caches (primary + base). + * Called with rwsem held for write. + */ +static void lbd_qcow2_invalidate_all_caches(struct lbd_qcow2 *q, + struct lbd_qcow2_base *base) +{ + int i; + + /* Primary cluster cache */ + for (i = 0; i < LBD_QCOW2_CL_CACHE_SIZE; i++) + q->cl_cache[i].valid = false; + + /* Primary L2 cache */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + q->l2_cache[i].valid = false; + + /* Base layer caches */ + if (base && base->is_qcow2) { + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + base->l2_cache[i].valid = false; + } +} + +int lbd_qcow2_swap_base(struct lbd_device *dev, const char *new_path) +{ + struct lbd_qcow2 *q = &dev->qcow2; + int ret; + + down_write(&q->rwsem); + + /* Flush all dirty L2 tables */ + lbd_qcow2_flush_all_l2(dev); + + /* Close old base */ + if (dev->base) { + lbd_qcow2_base_destroy(dev->base); + dev->base = NULL; + } + + /* Open new base */ + ret = lbd_qcow2_base_init(dev, new_path); + if (ret) { + pr_warn("lbd%d: swap_base failed to open '%s': %d\n", + dev->index, new_path, ret); + up_write(&q->rwsem); + return ret; + } + + /* Reload primary L1 table from disk */ + ret = lbd_qcow2_reload_l1(q, dev); + if (ret) { + pr_warn("lbd%d: swap_base failed to reload L1: %d\n", + dev->index, ret); + up_write(&q->rwsem); + return ret; + } + + /* Invalidate all cached data */ + lbd_qcow2_invalidate_all_caches(q, dev->base); + + /* Update stored path */ + strscpy(dev->base_path, new_path, sizeof(dev->base_path)); + + up_write(&q->rwsem); + + pr_info("lbd%d: base layer swapped to '%s'\n", dev->index, new_path); + return 0; +} + +int lbd_qcow2_base_init(struct lbd_device *dev, const char *path) +{ + struct lbd_qcow2 *q = &dev->qcow2; + struct lbd_qcow2_base *base; + struct file *f; + struct inode *inode, *primary_inode; + u64 magic; + loff_t pos; + ssize_t ret; + int i; + + f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(f)) { + pr_err("lbd%d: cannot open base file '%s': %ld\n", + dev->index, path, PTR_ERR(f)); + return PTR_ERR(f); + } + + inode = file_inode(f); + if (!S_ISREG(inode->i_mode)) { + pr_err("lbd%d: base path must be a regular file\n", + dev->index); + fput(f); + return -EINVAL; + } + + if (i_size_read(inode) == 0) { + pr_err("lbd%d: base file is empty\n", dev->index); + fput(f); + return -EINVAL; + } + + /* Base and primary must be different files */ + primary_inode = file_inode(dev->backing_file); + if (inode->i_sb == primary_inode->i_sb && + inode->i_ino == primary_inode->i_ino) { + pr_err("lbd%d: base and primary must be different files\n", + dev->index); + fput(f); + return -EINVAL; + } + + base = kzalloc(sizeof(*base), GFP_KERNEL); + if (!base) { + fput(f); + return -ENOMEM; + } + + base->file = f; + base->lru_tick = 0; + + /* Detect raw vs qcow2 via magic */ + pos = 0; + ret = kernel_read(f, &magic, 8, &pos); + if (ret != 8) { + pr_err("lbd%d: cannot read base file header\n", dev->index); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + if (be64_to_cpu(magic) == LBD_QCOW2_MAGIC) { + /* qcow2 base */ + u8 *hdr; + __be64 *disk_l1; + + base->is_qcow2 = true; + + hdr = kvmalloc(LBD_QCOW2_HEADER_SIZE, GFP_KERNEL); + if (!hdr) { + kfree(base); + fput(f); + return -ENOMEM; + } + + pos = 0; + ret = kernel_read(f, hdr, LBD_QCOW2_HEADER_SIZE, &pos); + if (ret != LBD_QCOW2_HEADER_SIZE) { + pr_err("lbd%d: base qcow2 header read failed\n", + dev->index); + kvfree(hdr); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + if (lbd_qcow2_hdr_version(hdr) != LBD_QCOW2_VERSION) { + pr_err("lbd%d: base qcow2 version mismatch (%u)\n", + dev->index, lbd_qcow2_hdr_version(hdr)); + kvfree(hdr); + kfree(base); + fput(f); + return -EINVAL; + } + + base->cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + if (base->cluster_bits != q->cluster_bits) { + pr_err("lbd%d: base cluster_bits %u != primary %u\n", + dev->index, base->cluster_bits, q->cluster_bits); + kvfree(hdr); + kfree(base); + fput(f); + return -EINVAL; + } + + base->cluster_size = 1U << base->cluster_bits; + base->l2_entries = (base->cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / sizeof(u64); + base->size = lbd_qcow2_hdr_virtual_size(hdr); + base->l1_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + base->l1_size = lbd_qcow2_hdr_l1_size(hdr); + + kvfree(hdr); + + /* Validate virtual size matches primary */ + if (base->size != q->virtual_size) { + pr_err("lbd%d: base virtual_size %llu != primary %llu\n", + dev->index, base->size, q->virtual_size); + kfree(base); + fput(f); + return -EINVAL; + } + + /* Load L1 table */ + base->l1_table = kvmalloc_array(base->l1_size, sizeof(u64), + GFP_KERNEL); + if (!base->l1_table) { + kfree(base); + fput(f); + return -ENOMEM; + } + + disk_l1 = kvmalloc_array(base->l1_size, sizeof(__be64), + GFP_KERNEL); + if (!disk_l1) { + kvfree(base->l1_table); + kfree(base); + fput(f); + return -ENOMEM; + } + + pos = base->l1_offset; + ret = kernel_read(f, disk_l1, + base->l1_size * sizeof(__be64), &pos); + if (ret != base->l1_size * sizeof(__be64)) { + pr_err("lbd%d: base L1 table read failed\n", + dev->index); + kvfree(disk_l1); + kvfree(base->l1_table); + kfree(base); + fput(f); + return ret < 0 ? ret : -EIO; + } + + for (i = 0; i < base->l1_size; i++) + base->l1_table[i] = be64_to_cpu(disk_l1[i]); + kvfree(disk_l1); + + /* Allocate L2 cache tables */ + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) { + struct lbd_l2_cache_entry *e = &base->l2_cache[i]; + + e->table = kvmalloc(base->cluster_size, GFP_KERNEL); + if (!e->table) + goto err_l2_cache; + e->valid = false; + e->dirty = false; + } + + /* Allocate read buffer for decompression */ + base->read_buf = kvmalloc(LZ4_compressBound(base->cluster_size), + GFP_KERNEL); + if (!base->read_buf) + goto err_l2_cache; + } else { + /* Raw base */ + base->is_qcow2 = false; + base->size = i_size_read(inode); + + /* Validate size matches primary */ + if (base->size != q->virtual_size) { + pr_err("lbd%d: base size %llu != primary virtual_size %llu\n", + dev->index, base->size, q->virtual_size); + kfree(base); + fput(f); + return -EINVAL; + } + } + + dev->base = base; + pr_info("lbd%d: base layer attached (%s, %llu bytes)\n", + dev->index, base->is_qcow2 ? "qcow2-lz4" : "raw", + base->size); + return 0; + +err_l2_cache: + if (base->is_qcow2) { + kvfree(base->read_buf); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(base->l2_cache[i].table); + kvfree(base->l1_table); + } + kfree(base); + fput(f); + return -ENOMEM; +} + +void lbd_qcow2_base_destroy(struct lbd_qcow2_base *base) +{ + int i; + + if (base->is_qcow2) { + kvfree(base->read_buf); + for (i = 0; i < LBD_QCOW2_L2_CACHE_SIZE; i++) + kvfree(base->l2_cache[i].table); + kvfree(base->l1_table); + } + + fput(base->file); + kfree(base); +} diff --git a/third_party/lbd/src/lbd_qcow2.h b/third_party/lbd/src/lbd_qcow2.h new file mode 100644 index 000000000..0d743cba8 --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LBD_QCOW2_H +#define _LBD_QCOW2_H + +#include +#include + +#include "lbd_qcow2_format.h" + +/* Cache sizes */ +#define LBD_QCOW2_L2_CACHE_SIZE 16 +#define LBD_QCOW2_CL_CACHE_SIZE 16 +/* ---------------------------------------------------------------- + * In-memory cache structures + * ---------------------------------------------------------------- */ + +struct lbd_l2_cache_entry { + u32 l1_index; + bool valid; + bool dirty; + u64 lru; + u64 *table; /* host-endian L2 entries, kvmalloc(cluster_size) */ +}; + +struct lbd_cl_cache_entry { + u64 cluster_index; + bool valid; + bool dirty; + u64 lru; + void *data; /* kvmalloc(cluster_size) */ +}; + +/* ---------------------------------------------------------------- + * Read-only base layer state (for thin snapshots) + * ---------------------------------------------------------------- */ + +struct lbd_qcow2_base { + struct file *file; + bool is_qcow2; + u64 size; /* raw: file size, qcow2: virtual_size */ + + /* Only valid when is_qcow2 == true */ + u32 cluster_bits; + u32 cluster_size; + u32 l2_entries; + u64 *l1_table; + u32 l1_size; + loff_t l1_offset; + + struct lbd_l2_cache_entry l2_cache[LBD_QCOW2_L2_CACHE_SIZE]; + u64 lru_tick; + + void *read_buf; /* decompression buffer */ +}; + +/* ---------------------------------------------------------------- + * Per-device qcow2 state (embedded in struct lbd_device) + * ---------------------------------------------------------------- */ + +struct lbd_qcow2 { + u32 cluster_bits; + u32 cluster_size; /* 1 << cluster_bits */ + u32 l2_entries; /* (cluster_size - L2_TRAILER_SIZE) / 8 */ + u64 virtual_size; + + u64 *l1_table; /* host-endian, always resident */ + u32 l1_size; + loff_t l1_offset; + + loff_t alloc_offset; /* append-only allocation cursor */ + loff_t free_list_head; /* head of on-disk free list, 0 if empty */ + + struct lbd_l2_cache_entry l2_cache[LBD_QCOW2_L2_CACHE_SIZE]; + struct lbd_cl_cache_entry cl_cache[LBD_QCOW2_CL_CACHE_SIZE]; + u64 lru_tick; /* monotonic counter for LRU */ + + struct rw_semaphore rwsem; /* read-shared, write-exclusive */ + + void *comp_buf; /* LZ4_compressBound(cluster_size) */ + void *read_buf; /* cluster_size, for reading compressed */ +}; + +/* ---------------------------------------------------------------- + * Function declarations + * ---------------------------------------------------------------- */ + +struct lbd_device; +struct request; + +int lbd_qcow2_init(struct lbd_device *dev); +void lbd_qcow2_destroy(struct lbd_device *dev); +int lbd_qcow2_read(struct lbd_device *dev, struct request *rq); +int lbd_qcow2_write(struct lbd_device *dev, struct request *rq); +int lbd_qcow2_discard(struct lbd_device *dev, struct request *rq); + +/* Base layer (thin snapshot) */ +int lbd_qcow2_base_init(struct lbd_device *dev, const char *path); +void lbd_qcow2_base_destroy(struct lbd_qcow2_base *base); +int lbd_qcow2_swap_base(struct lbd_device *dev, const char *new_path); + +#endif /* _LBD_QCOW2_H */ diff --git a/third_party/lbd/src/lbd_qcow2_format.h b/third_party/lbd/src/lbd_qcow2_format.h new file mode 100644 index 000000000..a310c7ab8 --- /dev/null +++ b/third_party/lbd/src/lbd_qcow2_format.h @@ -0,0 +1,254 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * LBD qcow2-lz4 on-disk format definitions. + * + * Shared between kernel module and userspace tools (lbdctl). + * All header fields are big-endian on disk and accessed at explicit + * byte offsets — no struct casting. + */ +#ifndef _LBD_QCOW2_FORMAT_H +#define _LBD_QCOW2_FORMAT_H + +/* ---------------------------------------------------------------- + * Constants + * ---------------------------------------------------------------- */ + +#define LBD_QCOW2_MAGIC 0x4C42444351573200ULL /* "LBDQCW2\0" */ +#define LBD_QCOW2_VERSION 2 +#define LBD_QCOW2_COMP_LZ4 1 +#define LBD_QCOW2_HEADER_SIZE 4096 +#define LBD_QCOW2_L2_TRAILER_SIZE 8 /* 4 bytes reserved + 4 bytes CRC32C */ + +/* Header field byte offsets */ +#define LBD_QCOW2_OFF_MAGIC 0 /* u64 */ +#define LBD_QCOW2_OFF_VERSION 8 /* u32 */ +#define LBD_QCOW2_OFF_CLUSTER_BITS 12 /* u32 */ +#define LBD_QCOW2_OFF_VIRTUAL_SIZE 16 /* u64 */ +#define LBD_QCOW2_OFF_L1_TABLE_OFFSET 24 /* u64 */ +#define LBD_QCOW2_OFF_L1_SIZE 32 /* u32 */ +#define LBD_QCOW2_OFF_ALLOC_OFFSET 36 /* u64 */ +#define LBD_QCOW2_OFF_COMP_TYPE 44 /* u32 */ +#define LBD_QCOW2_OFF_FREE_LIST 48 /* u64 */ + +/* Free list tombstone marker */ +#define LBD_QCOW2_FREE_TOMBSTONE 0xDEADF4EEU + +/* L2 entry flags */ +#define LBD_QCOW2_L2_COMPRESSED (1ULL << 63) +#define LBD_QCOW2_L2_OFFSET_MASK 0x3FFFFFFFFFFFFFFFULL + +/* Default cluster parameters */ +#define LBD_QCOW2_CLUSTER_BITS_DEFAULT 16 /* 64 KiB clusters */ +#define LBD_QCOW2_CLUSTER_SIZE_DEFAULT (1U << LBD_QCOW2_CLUSTER_BITS_DEFAULT) + +/* ---------------------------------------------------------------- + * Byte-order helpers (kernel vs userspace) + * ---------------------------------------------------------------- */ + +#ifdef __KERNEL__ + +#include +#include +#include + +typedef u8 _qcow2_u8; +typedef u32 _qcow2_u32; +typedef u64 _qcow2_u64; + +static inline _qcow2_u64 _qcow2_get64(const void *buf, int off) +{ + __be64 v; + memcpy(&v, (const u8 *)buf + off, 8); + return be64_to_cpu(v); +} + +static inline _qcow2_u32 _qcow2_get32(const void *buf, int off) +{ + __be32 v; + memcpy(&v, (const u8 *)buf + off, 4); + return be32_to_cpu(v); +} + +static inline void _qcow2_put64(void *buf, int off, _qcow2_u64 val) +{ + __be64 v = cpu_to_be64(val); + memcpy((u8 *)buf + off, &v, 8); +} + +static inline void _qcow2_put32(void *buf, int off, _qcow2_u32 val) +{ + __be32 v = cpu_to_be32(val); + memcpy((u8 *)buf + off, &v, 4); +} + +typedef u16 _qcow2_u16; + +static inline _qcow2_u16 _qcow2_get16(const void *buf, int off) +{ + __be16 v; + memcpy(&v, (const u8 *)buf + off, 2); + return be16_to_cpu(v); +} + +static inline void _qcow2_put16(void *buf, int off, _qcow2_u16 val) +{ + __be16 v = cpu_to_be16(val); + memcpy((u8 *)buf + off, &v, 2); +} + +#else /* userspace */ + +#include +#include + +typedef uint8_t _qcow2_u8; +typedef uint32_t _qcow2_u32; +typedef uint64_t _qcow2_u64; + +static inline _qcow2_u64 _qcow2_get64(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint64_t)p[0] << 56) | ((uint64_t)p[1] << 48) | + ((uint64_t)p[2] << 40) | ((uint64_t)p[3] << 32) | + ((uint64_t)p[4] << 24) | ((uint64_t)p[5] << 16) | + ((uint64_t)p[6] << 8) | (uint64_t)p[7]; +} + +static inline _qcow2_u32 _qcow2_get32(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) | + ((uint32_t)p[2] << 8) | (uint32_t)p[3]; +} + +static inline void _qcow2_put64(void *buf, int off, _qcow2_u64 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 56); p[1] = (uint8_t)(val >> 48); + p[2] = (uint8_t)(val >> 40); p[3] = (uint8_t)(val >> 32); + p[4] = (uint8_t)(val >> 24); p[5] = (uint8_t)(val >> 16); + p[6] = (uint8_t)(val >> 8); p[7] = (uint8_t)val; +} + +static inline void _qcow2_put32(void *buf, int off, _qcow2_u32 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 24); p[1] = (uint8_t)(val >> 16); + p[2] = (uint8_t)(val >> 8); p[3] = (uint8_t)val; +} + +typedef uint16_t _qcow2_u16; + +static inline _qcow2_u16 _qcow2_get16(const void *buf, int off) +{ + const uint8_t *p = (const uint8_t *)buf + off; + return ((uint16_t)p[0] << 8) | (uint16_t)p[1]; +} + +static inline void _qcow2_put16(void *buf, int off, _qcow2_u16 val) +{ + uint8_t *p = (uint8_t *)buf + off; + p[0] = (uint8_t)(val >> 8); p[1] = (uint8_t)val; +} + +#endif /* __KERNEL__ */ + +/* ---------------------------------------------------------------- + * Per-field accessors: getters + * ---------------------------------------------------------------- */ + +static inline _qcow2_u64 lbd_qcow2_hdr_magic(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_MAGIC); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_version(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_VERSION); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_cluster_bits(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_CLUSTER_BITS); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_virtual_size(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_VIRTUAL_SIZE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_l1_table_offset(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_L1_TABLE_OFFSET); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_l1_size(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_L1_SIZE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_alloc_offset(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_ALLOC_OFFSET); +} + +static inline _qcow2_u32 lbd_qcow2_hdr_comp_type(const void *h) +{ + return _qcow2_get32(h, LBD_QCOW2_OFF_COMP_TYPE); +} + +static inline _qcow2_u64 lbd_qcow2_hdr_free_list(const void *h) +{ + return _qcow2_get64(h, LBD_QCOW2_OFF_FREE_LIST); +} + +/* ---------------------------------------------------------------- + * Per-field accessors: setters + * ---------------------------------------------------------------- */ + +static inline void lbd_qcow2_hdr_set_magic(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_MAGIC, v); +} + +static inline void lbd_qcow2_hdr_set_version(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_VERSION, v); +} + +static inline void lbd_qcow2_hdr_set_cluster_bits(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_CLUSTER_BITS, v); +} + +static inline void lbd_qcow2_hdr_set_virtual_size(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_VIRTUAL_SIZE, v); +} + +static inline void lbd_qcow2_hdr_set_l1_table_offset(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_L1_TABLE_OFFSET, v); +} + +static inline void lbd_qcow2_hdr_set_l1_size(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_L1_SIZE, v); +} + +static inline void lbd_qcow2_hdr_set_alloc_offset(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_ALLOC_OFFSET, v); +} + +static inline void lbd_qcow2_hdr_set_comp_type(void *h, _qcow2_u32 v) +{ + _qcow2_put32(h, LBD_QCOW2_OFF_COMP_TYPE, v); +} + +static inline void lbd_qcow2_hdr_set_free_list(void *h, _qcow2_u64 v) +{ + _qcow2_put64(h, LBD_QCOW2_OFF_FREE_LIST, v); +} + +#endif /* _LBD_QCOW2_FORMAT_H */ diff --git a/third_party/lbd/src/lbdctl.c b/third_party/lbd/src/lbdctl.c new file mode 100644 index 000000000..6a10c642c --- /dev/null +++ b/third_party/lbd/src/lbdctl.c @@ -0,0 +1,2949 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * lbdctl - userspace control tool for LBD (Logging Block Device) + * + * Usage: + * lbdctl add [--json] --log-dir /path/to/logs /path/to/file.img + * lbdctl remove [--json] N - destroy /dev/lbdN + * lbdctl list [--json] - show all active devices + * lbdctl log [--json] /path/to/file.img.log - dump log + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lz4/lz4.h" + +/* Mirror the kernel ioctl structures from lbd.h */ +#define LBD_LOG_PATH_MAX 256 + +/* CBOR log numeric map keys */ +#define LBD_CBOR_KEY_HDR_VERSION 1 +#define LBD_CBOR_KEY_HDR_BLOCK_SIZE 2 +#define LBD_CBOR_KEY_HDR_SEGMENT_LABEL 3 +#define LBD_CBOR_KEY_HDR_DEVICE_SIZE 4 +#define LBD_CBOR_KEY_HDR_BACKING_PATH 5 + +#define LBD_CBOR_KEY_OP 1 +#define LBD_CBOR_KEY_TIMESTAMP 2 +#define LBD_CBOR_KEY_SEQUENCE 3 +#define LBD_CBOR_KEY_BLOCK 4 +#define LBD_CBOR_KEY_LENGTH 5 +#define LBD_CBOR_KEY_CHECKSUM 6 +#define LBD_CBOR_KEY_DATA 7 + +/* Watch command keys (write path) */ +#define LBD_WATCH_KEY_CMD 1 +#define LBD_WATCH_KEY_DEV 2 +#define LBD_WATCH_KEY_PATH 3 + +/* Miss event keys */ +#define LBD_MISS_KEY_TYPE 1 +#define LBD_MISS_KEY_DEV 2 +#define LBD_MISS_KEY_CLUSTER 3 + +/* Event keys (read path) */ +#define LBD_EVENT_KEY_TYPE 1 +#define LBD_EVENT_KEY_DEV 2 +#define LBD_EVENT_KEY_LABEL 3 +#define LBD_EVENT_KEY_DIR 4 +#define LBD_EVENT_KEY_SEQ 5 +#define LBD_EVENT_KEY_SIZE 6 + +#define LBD_CTL_MAGIC 'L' + +struct lbd_ctl_add { + char path[LBD_LOG_PATH_MAX]; + char log_dir[LBD_LOG_PATH_MAX]; + char base_path[LBD_LOG_PATH_MAX]; /* empty string = no base */ + __s32 index; + __u64 log_max_size; + __u32 log_max_age_secs; +}; + +struct lbd_ctl_remove { + __s32 index; +}; + +struct lbd_ctl_info { + __s32 index; + __u32 state; + __u64 size; + char path[LBD_LOG_PATH_MAX]; +}; + +#define LBD_CTL_ADD _IOWR(LBD_CTL_MAGIC, 0, struct lbd_ctl_add) +#define LBD_CTL_REMOVE _IOW(LBD_CTL_MAGIC, 1, struct lbd_ctl_remove) +#define LBD_CTL_INFO _IOWR(LBD_CTL_MAGIC, 2, struct lbd_ctl_info) + +#define LBD_CTL_PATH "/dev/lbd-control" + +/* ---------------------------------------------------------------- + * CRC32 (IEEE 802.3 polynomial, matches Linux kernel crc32()) + * ---------------------------------------------------------------- */ + +static uint32_t crc32_table[256]; +static int crc32_table_ready; + +static void crc32_init(void) +{ + uint32_t poly = 0xEDB88320; + for (int i = 0; i < 256; i++) { + uint32_t c = i; + for (int j = 0; j < 8; j++) + c = (c >> 1) ^ (poly & (-(c & 1))); + crc32_table[i] = c; + } + crc32_table_ready = 1; +} + +static uint32_t crc32_calc(const void *buf, size_t len) +{ + if (!crc32_table_ready) + crc32_init(); + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFF; + for (size_t i = 0; i < len; i++) + crc = crc32_table[(crc ^ p[i]) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFF; +} + +/* ---------------------------------------------------------------- + * CRC32C (Castagnoli polynomial 0x82F63B78, matches Linux crc32c()) + * ---------------------------------------------------------------- */ + +static uint32_t crc32c_table[256]; +static int crc32c_table_ready; + +static void crc32c_init(void) +{ + uint32_t poly = 0x82F63B78; + for (int i = 0; i < 256; i++) { + uint32_t c = i; + for (int j = 0; j < 8; j++) + c = (c >> 1) ^ (poly & (-(c & 1))); + crc32c_table[i] = c; + } + crc32c_table_ready = 1; +} + +static uint32_t crc32c_calc(const void *buf, size_t len) +{ + if (!crc32c_table_ready) + crc32c_init(); + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFF; + for (size_t i = 0; i < len; i++) + crc = crc32c_table[(crc ^ p[i]) & 0xFF] ^ (crc >> 8); + return crc ^ 0xFFFFFFFF; +} + +/* ---------------------------------------------------------------- + * Helpers + * ---------------------------------------------------------------- */ + +static const char *state_str(unsigned int state) +{ + switch (state) { + case 0: return "unbound"; + case 1: return "bound"; + case 2: return "removing"; + default: return "unknown"; + } +} + +static int open_ctl(void) +{ + int fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + } + return fd; +} + +static int read_exact(int fd, void *buf, size_t len) +{ + size_t done = 0; + while (done < len) { + ssize_t n = read(fd, (char *)buf + done, len - done); + if (n < 0) { + if (errno == EINTR) + continue; + return -1; + } + if (n == 0) + return done > 0 ? -1 : 0; /* EOF */ + done += n; + } + return 1; /* success */ +} + +/* Escape a string for JSON output (handles \, ", and control chars) */ +static void json_print_string(FILE *out, const char *s) +{ + fputc('"', out); + for (; *s; s++) { + unsigned char c = *s; + switch (c) { + case '"': fputs("\\\"", out); break; + case '\\': fputs("\\\\", out); break; + case '\b': fputs("\\b", out); break; + case '\f': fputs("\\f", out); break; + case '\n': fputs("\\n", out); break; + case '\r': fputs("\\r", out); break; + case '\t': fputs("\\t", out); break; + default: + if (c < 0x20) + fprintf(out, "\\u%04x", c); + else + fputc(c, out); + } + } + fputc('"', out); +} + +/* Format size with human-readable units */ +static const char *fmt_size(uint64_t bytes, char *buf, size_t bufsz) +{ + if (bytes >= (1ULL << 30)) + snprintf(buf, bufsz, "%.1f GiB", (double)bytes / (1ULL << 30)); + else if (bytes >= (1ULL << 20)) + snprintf(buf, bufsz, "%.1f MiB", (double)bytes / (1ULL << 20)); + else if (bytes >= (1ULL << 10)) + snprintf(buf, bufsz, "%.1f KiB", (double)bytes / (1ULL << 10)); + else + snprintf(buf, bufsz, "%llu B", (unsigned long long)bytes); + return buf; +} + +/* Print hex dump of data (16 bytes per line) */ +static void hex_dump(FILE *out, const uint8_t *data, size_t len, + const char *indent) +{ + for (size_t off = 0; off < len; off += 16) { + fprintf(out, "%s%08zx ", indent, off); + /* hex */ + for (size_t i = 0; i < 16; i++) { + if (off + i < len) + fprintf(out, "%02x ", data[off + i]); + else + fputs(" ", out); + if (i == 7) + fputc(' ', out); + } + fputs(" |", out); + /* ascii */ + for (size_t i = 0; i < 16 && off + i < len; i++) { + uint8_t c = data[off + i]; + fputc((c >= 0x20 && c < 0x7f) ? c : '.', out); + } + fputs("|\n", out); + } +} + +/* ---------------------------------------------------------------- + * Device control commands + * ---------------------------------------------------------------- */ + +static int cmd_add(int argc, char **argv) +{ + struct lbd_ctl_add arg; + char resolved[PATH_MAX]; + const char *path = NULL; + const char *log_dir = NULL; + const char *base = NULL; + int fd, ret; + int json = 0; + + memset(&arg, 0, sizeof(arg)); + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else if (strcmp(argv[i], "--log-max-size") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-max-size requires a value\n"); + return 1; + } + arg.log_max_size = strtoull(argv[i], NULL, 0); + } else if (strcmp(argv[i], "--log-max-age") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-max-age requires a value\n"); + return 1; + } + arg.log_max_age_secs = strtoul(argv[i], NULL, 0); + } else if (strcmp(argv[i], "--log-dir") == 0) { + if (++i >= argc) { + fprintf(stderr, "--log-dir requires a value\n"); + return 1; + } + log_dir = argv[i]; + } else if (strcmp(argv[i], "--base") == 0) { + if (++i >= argc) { + fprintf(stderr, "--base requires a value\n"); + return 1; + } + base = argv[i]; + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "add requires a path argument\n"); + return 1; + } + + if (!log_dir) { + fprintf(stderr, "add requires --log-dir \n"); + return 1; + } + + if (!realpath(path, resolved)) { + fprintf(stderr, "Cannot resolve path '%s': %s\n", + path, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Path too long (max %d)\n", LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.path, LBD_LOG_PATH_MAX, "%s", resolved); + + if (!realpath(log_dir, resolved)) { + fprintf(stderr, "Cannot resolve log directory '%s': %s\n", + log_dir, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Log directory path too long (max %d)\n", + LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.log_dir, LBD_LOG_PATH_MAX, "%s", resolved); + + if (base) { + if (!realpath(base, resolved)) { + fprintf(stderr, "Cannot resolve base path '%s': %s\n", + base, strerror(errno)); + return 1; + } + + if (strlen(resolved) >= LBD_LOG_PATH_MAX) { + fprintf(stderr, "Base path too long (max %d)\n", + LBD_LOG_PATH_MAX - 1); + return 1; + } + + snprintf(arg.base_path, LBD_LOG_PATH_MAX, "%s", resolved); + } + + fd = open_ctl(); + if (fd < 0) + return 1; + + ret = ioctl(fd, LBD_CTL_ADD, &arg); + if (ret < 0) { + fprintf(stderr, "LBD_CTL_ADD failed: %s\n", strerror(errno)); + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d}\n", + arg.index, arg.index); + } else { + printf("Created /dev/lbd%d\n", arg.index); + } + close(fd); + return 0; +} + +static int cmd_remove(int argc, char **argv) +{ + struct lbd_ctl_remove arg; + int fd, ret; + int json = 0; + const char *index_str = NULL; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else if (!index_str) + index_str = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!index_str) { + fprintf(stderr, "remove requires an index argument\n"); + return 1; + } + + arg.index = atoi(index_str); + + fd = open_ctl(); + if (fd < 0) + return 1; + + ret = ioctl(fd, LBD_CTL_REMOVE, &arg); + if (ret < 0) { + fprintf(stderr, "LBD_CTL_REMOVE failed: %s\n", strerror(errno)); + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d}\n", + arg.index, arg.index); + } else { + printf("Removed /dev/lbd%d\n", arg.index); + } + close(fd); + return 0; +} + +static int cmd_list(int argc, char **argv) +{ + struct lbd_ctl_info info; + int fd, i, found = 0; + int json = 0; + + for (i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + fd = open_ctl(); + if (fd < 0) + return 1; + + if (json) { + printf("["); + } else { + printf("%-8s %-10s %-12s %s\n", "DEVICE", "STATE", "SIZE", "BACKING"); + printf("%-8s %-10s %-12s %s\n", "------", "-----", "----", "-------"); + } + + for (i = 0; i < 256; i++) { + memset(&info, 0, sizeof(info)); + info.index = i; + + if (ioctl(fd, LBD_CTL_INFO, &info) < 0) + continue; + + if (json) { + if (found > 0) + printf(","); + printf("\n {\"device\": \"/dev/lbd%d\", \"index\": %d, \"state\": \"%s\", \"size\": %llu, \"backing\": ", + info.index, info.index, state_str(info.state), + (unsigned long long)info.size); + json_print_string(stdout, info.path); + printf("}"); + } else { + printf("lbd%-5d %-10s %-12llu %s\n", + info.index, state_str(info.state), + (unsigned long long)info.size, info.path); + } + found++; + } + + if (json) { + printf("\n]\n"); + } else if (!found) { + printf("(no devices)\n"); + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * CBOR decoder (streaming, reads from fd) + * ---------------------------------------------------------------- */ + +/* + * Read a CBOR head: returns major type (0-7) and argument value. + * Returns 0 on success, -1 on EOF, -2 on error. + */ +static int cbor_read_head(int fd, uint8_t *major_out, uint64_t *val_out) +{ + uint8_t ib; + int ret = read_exact(fd, &ib, 1); + if (ret == 0) + return -1; /* EOF */ + if (ret < 0) + return -2; + + *major_out = ib >> 5; + uint8_t ai = ib & 0x1F; + + if (ai < 24) { + *val_out = ai; + } else if (ai == 24) { + uint8_t b; + if (read_exact(fd, &b, 1) <= 0) return -2; + *val_out = b; + } else if (ai == 25) { + uint8_t b[2]; + if (read_exact(fd, b, 2) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 8) | b[1]; + } else if (ai == 26) { + uint8_t b[4]; + if (read_exact(fd, b, 4) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 24) | ((uint64_t)b[1] << 16) | + ((uint64_t)b[2] << 8) | b[3]; + } else if (ai == 27) { + uint8_t b[8]; + if (read_exact(fd, b, 8) <= 0) return -2; + *val_out = ((uint64_t)b[0] << 56) | ((uint64_t)b[1] << 48) | + ((uint64_t)b[2] << 40) | ((uint64_t)b[3] << 32) | + ((uint64_t)b[4] << 24) | ((uint64_t)b[5] << 16) | + ((uint64_t)b[6] << 8) | b[7]; + } else { + return -2; /* indefinite / reserved */ + } + + return 0; +} + +/* Expect a uint (major 0), returns 0 on success */ +static int cbor_read_uint(int fd, uint64_t *val_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, val_out); + if (ret) return ret; + if (major != 0) return -2; + return 0; +} + +/* Read a text string (major 3) into buf, NUL-terminated. Returns 0 on success */ +static int cbor_read_text(int fd, char *buf, size_t cap, uint64_t *len_out) +{ + uint8_t major; + uint64_t len; + int ret = cbor_read_head(fd, &major, &len); + if (ret) return ret; + if (major != 3) return -2; + if (len >= cap) return -2; /* too long */ + if (len > 0 && read_exact(fd, buf, len) <= 0) return -2; + buf[len] = '\0'; + if (len_out) *len_out = len; + return 0; +} + +/* Read a byte string header (major 2), returns length */ +static int cbor_read_bytes_hdr(int fd, uint64_t *len_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, len_out); + if (ret) return ret; + if (major != 2) return -2; + return 0; +} + +/* Read a map header (major 5), returns count */ +static int cbor_read_map(int fd, uint64_t *count_out) +{ + uint8_t major; + int ret = cbor_read_head(fd, &major, count_out); + if (ret) return ret; + if (major != 5) return -2; + return 0; +} + +/* Skip one CBOR data item (recursively handles maps, arrays, etc.) */ +static int cbor_skip(int fd) +{ + uint8_t major; + uint64_t val; + int ret = cbor_read_head(fd, &major, &val); + if (ret) return ret; + + switch (major) { + case 0: /* uint */ + case 1: /* negint */ + case 7: /* simple/float */ + return 0; + case 2: /* byte string */ + case 3: /* text string */ + if (val > 0) { + /* Skip val bytes */ + while (val > 0) { + uint8_t tmp[256]; + size_t chunk = val < sizeof(tmp) ? (size_t)val : sizeof(tmp); + if (read_exact(fd, tmp, chunk) <= 0) return -2; + val -= chunk; + } + } + return 0; + case 4: /* array */ + for (uint64_t i = 0; i < val; i++) { + if (cbor_skip(fd)) return -2; + } + return 0; + case 5: /* map */ + for (uint64_t i = 0; i < val; i++) { + if (cbor_skip(fd)) return -2; /* key */ + if (cbor_skip(fd)) return -2; /* value */ + } + return 0; + default: + return -2; + } +} + +/* ---------------------------------------------------------------- + * Log reading (CBOR format) + * ---------------------------------------------------------------- */ + +static int cmd_log_cbor(int fd, const char *path, int json, int show_data) +{ + uint64_t map_count, key, val; + uint32_t version = 0, block_size = 4096; + char segment_label[32]; + uint64_t device_size = 0; + char backing_path[LBD_LOG_PATH_MAX]; + int entry_count = 0; + char sizebuf[32]; + uint8_t *data = NULL; + size_t data_cap = 0; + + memset(segment_label, 0, sizeof(segment_label)); + memset(backing_path, 0, sizeof(backing_path)); + + /* Read header map */ + if (cbor_read_map(fd, &map_count)) { + fprintf(stderr, "Failed to read CBOR header map\n"); + return 1; + } + + for (uint64_t i = 0; i < map_count; i++) { + if (cbor_read_uint(fd, &key)) { + fprintf(stderr, "Failed to read header key\n"); + return 1; + } + switch (key) { + case LBD_CBOR_KEY_HDR_VERSION: + if (cbor_read_uint(fd, &val)) return 1; + version = (uint32_t)val; + break; + case LBD_CBOR_KEY_HDR_BLOCK_SIZE: + if (cbor_read_uint(fd, &val)) return 1; + block_size = (uint32_t)val; + break; + case LBD_CBOR_KEY_HDR_SEGMENT_LABEL: + if (cbor_read_text(fd, segment_label, + sizeof(segment_label), NULL)) + return 1; + break; + case LBD_CBOR_KEY_HDR_DEVICE_SIZE: + if (cbor_read_uint(fd, &val)) return 1; + device_size = val; + break; + case LBD_CBOR_KEY_HDR_BACKING_PATH: + if (cbor_read_text(fd, backing_path, sizeof(backing_path), NULL)) + return 1; + break; + default: + if (cbor_skip(fd)) return 1; + break; + } + } + + if (json) { + printf("{\n"); + printf(" \"header\": {\n"); + printf(" \"version\": %u,\n", version); + printf(" \"block_size\": %u,\n", block_size); + printf(" \"segment_label\": "); + json_print_string(stdout, segment_label); + printf(",\n"); + printf(" \"device_size\": %llu,\n", + (unsigned long long)device_size); + printf(" \"backing_path\": "); + json_print_string(stdout, backing_path); + printf("\n },\n"); + printf(" \"entries\": [\n"); + } else { + printf("=== LBD Log: %s ===\n", path); + printf("Version: %u\n", version); + printf("Block size: %u\n", block_size); + printf("Segment: %s\n", segment_label); + printf("Device size: %s (%llu bytes)\n", + fmt_size(device_size, sizebuf, sizeof(sizebuf)), + (unsigned long long)device_size); + printf("Backing file: %s\n", backing_path); + printf("\n"); + } + + /* Read entry maps until EOF */ + while (1) { + uint8_t peek_major; + uint64_t entry_map_count; + int ret; + + /* Try to read next map header; EOF is normal end */ + ret = cbor_read_head(fd, &peek_major, &entry_map_count); + if (ret == -1) + break; /* clean EOF */ + if (ret < 0) { + fprintf(stderr, "Error reading entry %d\n", entry_count); + break; + } + if (peek_major != 5) { + fprintf(stderr, "Expected CBOR map at entry %d, got major %u\n", + entry_count, peek_major); + break; + } + + /* Parse entry fields */ + char op = 0; + uint64_t timestamp_ns = 0, sequence = 0, block = 0; + uint32_t length = 0, checksum = 0; + int has_checksum = 0, has_data = 0; + uint64_t data_len = 0; + uint64_t comp_size = 0; /* compressed size from CBOR */ + + for (uint64_t i = 0; i < entry_map_count; i++) { + if (cbor_read_uint(fd, &key)) { + fprintf(stderr, "Failed to read entry key at entry %d\n", + entry_count); + goto done; + } + switch (key) { + case LBD_CBOR_KEY_OP: { + char tbuf[4]; + if (cbor_read_text(fd, tbuf, sizeof(tbuf), NULL)) + goto done; + op = tbuf[0]; + break; + } + case LBD_CBOR_KEY_TIMESTAMP: + if (cbor_read_uint(fd, ×tamp_ns)) goto done; + break; + case LBD_CBOR_KEY_SEQUENCE: + if (cbor_read_uint(fd, &sequence)) goto done; + break; + case LBD_CBOR_KEY_BLOCK: + if (cbor_read_uint(fd, &block)) goto done; + break; + case LBD_CBOR_KEY_LENGTH: + if (cbor_read_uint(fd, &val)) goto done; + length = (uint32_t)val; + break; + case LBD_CBOR_KEY_CHECKSUM: + if (cbor_read_uint(fd, &val)) goto done; + checksum = (uint32_t)val; + has_checksum = 1; + break; + case LBD_CBOR_KEY_DATA: + if (cbor_read_bytes_hdr(fd, &data_len)) goto done; + has_data = 1; + if (data_len > data_cap) { + free(data); + data_cap = (size_t)data_len; + data = malloc(data_cap); + if (!data) { + fprintf(stderr, + "Out of memory for %llu byte payload\n", + (unsigned long long)data_len); + goto done; + } + } + if (data_len > 0 && + read_exact(fd, data, (size_t)data_len) <= 0) { + fprintf(stderr, + "Truncated data at entry %d\n", + entry_count); + goto done; + } + break; + default: + if (cbor_skip(fd)) goto done; + break; + } + } + + int is_trim = (op == 'T'); + + /* Decompress LZ4 data */ + comp_size = data_len; + if (has_data && length > 0 && data_len > 0) { + uint8_t *decompressed = malloc(length); + if (!decompressed) { + fprintf(stderr, + "Out of memory for %u byte decompression buffer\n", + length); + goto done; + } + int dec_len = LZ4_decompress_safe( + (const char *)data, (char *)decompressed, + (int)data_len, (int)length); + if (dec_len < 0) { + fprintf(stderr, + "LZ4 decompression failed at entry %d\n", + entry_count); + free(decompressed); + goto done; + } + /* Replace compressed data with decompressed */ + if ((size_t)length > data_cap) { + free(data); + data_cap = length; + data = malloc(data_cap); + if (!data) { + free(decompressed); + data = NULL; + data_cap = 0; + goto done; + } + } + memcpy(data, decompressed, dec_len); + data_len = dec_len; + free(decompressed); + } + + /* Validate CRC on decompressed data */ + uint32_t computed_crc = 0; + int crc_ok = 1; + if (has_data && has_checksum) { + computed_crc = crc32_calc(data, (size_t)data_len); + crc_ok = (computed_crc == checksum); + } + + if (json) { + if (entry_count > 0) + printf(",\n"); + printf(" {\n"); + printf(" \"type\": \"%s\",\n", + is_trim ? "trim" : "write"); + printf(" \"sequence\": %llu,\n", + (unsigned long long)sequence); + printf(" \"timestamp_ns\": %llu,\n", + (unsigned long long)timestamp_ns); + + time_t secs = timestamp_ns / 1000000000ULL; + unsigned long ns = timestamp_ns % 1000000000ULL; + struct tm tm; + gmtime_r(&secs, &tm); + char tbuf[64]; + strftime(tbuf, sizeof(tbuf), "%Y-%m-%dT%H:%M:%S", &tm); + printf(" \"timestamp\": \"%s.%09luZ\",\n", tbuf, ns); + + printf(" \"block\": %llu,\n", + (unsigned long long)block); + printf(" \"block_end\": %llu,\n", + (unsigned long long)(block + length / block_size - 1)); + printf(" \"extent\": \"%llu-%llu\",\n", + (unsigned long long)block, + (unsigned long long)(block + length / block_size - 1)); + printf(" \"offset_bytes\": %llu,\n", + (unsigned long long)block * block_size); + printf(" \"length\": %u", length); + if (has_data && comp_size > 0) { + printf(",\n \"compressed_size\": %llu", + (unsigned long long)comp_size); + printf(",\n \"compression_ratio\": %.1f", + length > 0 ? (1.0 - (double)comp_size / length) * 100.0 : 0.0); + } + if (has_checksum) { + printf(",\n \"checksum\": \"0x%08x\",\n", + checksum); + printf(" \"checksum_valid\": %s", + crc_ok ? "true" : "false"); + } + if (show_data && has_data) { + printf(",\n \"data_hex\": \""); + for (uint64_t i = 0; i < data_len; i++) + printf("%02x", data[i]); + printf("\""); + } + printf("\n }"); + } else { + time_t secs = timestamp_ns / 1000000000ULL; + unsigned long ns = timestamp_ns % 1000000000ULL; + struct tm tm; + gmtime_r(&secs, &tm); + char tbuf[64]; + strftime(tbuf, sizeof(tbuf), "%Y-%m-%d %H:%M:%S", &tm); + + printf("--- Entry #%llu [%s] ---\n", + (unsigned long long)sequence, + is_trim ? "TRIM" : "WRITE"); + printf(" Time: %s.%09lu UTC\n", tbuf, ns); + printf(" Extent: %llu-%llu (%u blocks)\n", + (unsigned long long)block, + (unsigned long long)(block + length / block_size - 1), + length / block_size); + printf(" Offset: 0x%llx-0x%llx\n", + (unsigned long long)block * block_size, + (unsigned long long)(block * block_size + length - 1)); + printf(" Length: %s (%u bytes)\n", + fmt_size(length, sizebuf, sizeof(sizebuf)), + length); + if (has_data && comp_size > 0) { + printf(" LZ4: %s compressed (%.1f%% reduction)\n", + fmt_size(comp_size, sizebuf, sizeof(sizebuf)), + length > 0 ? (1.0 - (double)comp_size / length) * 100.0 : 0.0); + } + if (has_checksum) { + printf(" CRC32: 0x%08x %s\n", checksum, + crc_ok ? "(OK)" : "(MISMATCH - computed 0x%08x)"); + if (!crc_ok) + printf(" computed: 0x%08x\n", + computed_crc); + } + if (show_data && has_data) { + printf(" Data:\n"); + hex_dump(stdout, data, (size_t)data_len, " "); + } + printf("\n"); + } + + entry_count++; + } + +done: + if (json) { + printf("\n ],\n"); + printf(" \"entry_count\": %d\n", entry_count); + printf("}\n"); + } else { + printf("Total entries: %d\n", entry_count); + } + + free(data); + return 0; +} + +static int cmd_log(const char *path, int json, int show_data) +{ + int fd, ret; + + fd = open(path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open log file '%s': %s\n", + path, strerror(errno)); + return 1; + } + + ret = cmd_log_cbor(fd, path, json, show_data); + close(fd); + return ret; +} + +/* ---------------------------------------------------------------- + * qcow2-lz4 format support + * ---------------------------------------------------------------- */ + +#include "lbd_qcow2_format.h" + +static inline uint64_t htobe64_val(uint64_t x) +{ + uint8_t buf[8]; + _qcow2_put64(buf, 0, x); + uint64_t v; + memcpy(&v, buf, 8); + return v; +} + +static inline uint64_t be64toh_val(uint64_t x) +{ + uint8_t buf[8]; + memcpy(buf, &x, 8); + return _qcow2_get64(buf, 0); +} + +static inline uint32_t htobe32_val(uint32_t x) +{ + uint8_t buf[4]; + _qcow2_put32(buf, 0, x); + uint32_t v; + memcpy(&v, buf, 4); + return v; +} + +static inline uint32_t be32toh_val(uint32_t x) +{ + uint8_t buf[4]; + memcpy(buf, &x, 4); + return _qcow2_get32(buf, 0); +} + +static int is_all_zero(const void *buf, size_t len) +{ + const uint8_t *p = buf; + for (size_t i = 0; i < len; i++) + if (p[i] != 0) + return 0; + return 1; +} + +static uint64_t parse_size(const char *s) +{ + char *end; + uint64_t val = strtoull(s, &end, 0); + + switch (*end) { + case 'k': case 'K': val *= 1024; break; + case 'm': case 'M': val *= 1024 * 1024; break; + case 'g': case 'G': val *= 1024ULL * 1024 * 1024; break; + case 't': case 'T': val *= 1024ULL * 1024 * 1024 * 1024; break; + } + return val; +} + +/* + * lbdctl create --size [--cluster-bits 16] + * Create an empty qcow2-lz4 image file. + */ +static int cmd_create(int argc, char **argv) +{ + const char *path = NULL; + uint64_t virtual_size = 0; + uint32_t cluster_bits = 16; + uint32_t cluster_size, l2_entries, l1_size; + uint64_t l1_table_offset, alloc_offset; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + int fd; + ssize_t n; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--size") == 0) { + if (++i >= argc) { + fprintf(stderr, "--size requires a value\n"); + return 1; + } + virtual_size = parse_size(argv[i]); + } else if (strcmp(argv[i], "--cluster-bits") == 0) { + if (++i >= argc) { + fprintf(stderr, "--cluster-bits requires a value\n"); + return 1; + } + cluster_bits = atoi(argv[i]); + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "create requires an output path\n"); + return 1; + } + + if (virtual_size == 0) { + fprintf(stderr, "create requires --size \n"); + return 1; + } + + if (cluster_bits < 12 || cluster_bits > 24) { + fprintf(stderr, "cluster_bits must be between 12 and 24\n"); + return 1; + } + + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + + /* Compute L1 size: one entry per L2 table needed */ + l1_size = (virtual_size + (uint64_t)l2_entries * cluster_size - 1) / + ((uint64_t)l2_entries * cluster_size); + if (l1_size == 0) + l1_size = 1; + + l1_table_offset = LBD_QCOW2_HEADER_SIZE; + alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + /* Align alloc_offset to cluster boundary */ + alloc_offset = (alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Build header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", path, strerror(errno)); + return 1; + } + + /* Write header */ + n = write(fd, hdr, LBD_QCOW2_HEADER_SIZE); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to write header: %s\n", strerror(errno)); + close(fd); + return 1; + } + + /* Write zeroed L1 table */ + { + size_t l1_bytes = l1_size * sizeof(uint64_t); + void *zeros = calloc(1, l1_bytes); + if (!zeros) { + fprintf(stderr, "Out of memory\n"); + close(fd); + return 1; + } + n = write(fd, zeros, l1_bytes); + free(zeros); + if (n != (ssize_t)l1_bytes) { + fprintf(stderr, "Failed to write L1 table: %s\n", + strerror(errno)); + close(fd); + return 1; + } + } + + /* Extend file to alloc_offset */ + if (ftruncate(fd, alloc_offset) < 0) { + fprintf(stderr, "Failed to extend file: %s\n", strerror(errno)); + close(fd); + return 1; + } + + close(fd); + + printf("Created qcow2-lz4 image: %s\n", path); + printf(" Virtual size: %llu bytes (%s)\n", + (unsigned long long)virtual_size, + fmt_size(virtual_size, (char[32]){0}, 32)); + printf(" Cluster size: %u bytes (bits=%u)\n", cluster_size, cluster_bits); + printf(" L1 entries: %u\n", l1_size); + printf(" L2 entries: %u per table\n", l2_entries); + printf(" Alloc offset: %llu\n", (unsigned long long)alloc_offset); + + return 0; +} + +/* + * lbdctl convert + * Convert a flat (raw) image file to qcow2-lz4 format. + */ +static int cmd_convert(const char *flat_path, const char *qcow2_path) +{ + int in_fd, out_fd; + struct stat st; + uint64_t virtual_size; + uint32_t cluster_bits = 16; + uint32_t cluster_size = 1U << cluster_bits; + uint32_t l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + uint32_t l1_size; + uint64_t l1_table_offset, alloc_offset; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t *l1_table; + uint64_t **l2_tables; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + uint64_t total_clusters, clusters_written = 0, clusters_zero = 0; + ssize_t n; + + in_fd = open(flat_path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", flat_path, strerror(errno)); + return 1; + } + + if (fstat(in_fd, &st) < 0) { + fprintf(stderr, "Cannot stat '%s': %s\n", flat_path, strerror(errno)); + close(in_fd); + return 1; + } + + virtual_size = st.st_size; + if (virtual_size == 0) { + fprintf(stderr, "Input file is empty\n"); + close(in_fd); + return 1; + } + + total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + l1_size = (virtual_size + (uint64_t)l2_entries * cluster_size - 1) / + ((uint64_t)l2_entries * cluster_size); + if (l1_size == 0) + l1_size = 1; + + l1_table_offset = LBD_QCOW2_HEADER_SIZE; + alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + alloc_offset = (alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Allocate tables */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + l2_tables = calloc(l1_size, sizeof(uint64_t *)); + cluster_buf = malloc(cluster_size); + comp_cap = LZ4_compressBound(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !l2_tables || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + for (uint32_t i = 0; i < l1_size; i++) { + l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + } + + out_fd = open(qcow2_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", qcow2_path, strerror(errno)); + close(in_fd); + return 1; + } + + /* Reserve space for header + L1 table (written at end) */ + if (ftruncate(out_fd, alloc_offset) < 0) { + fprintf(stderr, "Failed to extend output: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + if (lseek(out_fd, alloc_offset, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Process each cluster */ + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + size_t to_read = cluster_size; + ssize_t rd; + + /* Handle last partial cluster */ + if ((ci + 1) * cluster_size > virtual_size) + to_read = virtual_size - ci * cluster_size; + + memset(cluster_buf, 0, cluster_size); + rd = pread(in_fd, cluster_buf, to_read, ci * cluster_size); + if (rd < 0) { + fprintf(stderr, "Read error at cluster %llu: %s\n", + (unsigned long long)ci, strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Skip all-zero clusters (leave L2 = 0 for sparse) */ + if (is_all_zero(cluster_buf, cluster_size)) { + clusters_zero++; + continue; + } + + /* Allocate L2 table if needed */ + if (l1_table[l1_idx] == 0) { + l1_table[l1_idx] = alloc_offset; + alloc_offset += cluster_size; + } + + /* Compress */ + int comp_len = LZ4_compress_default( + (const char *)cluster_buf, (char *)comp_buf, + cluster_size, comp_cap); + + if (comp_len > 0 && + (uint32_t)comp_len < cluster_size - sizeof(uint32_t)) { + /* Store compressed */ + uint32_t total_on_disk = ((sizeof(uint32_t) + comp_len) + 4095) & ~4095U; + uint64_t phys = alloc_offset; + uint32_t comp_size_be = htobe32_val(comp_len); + + /* Write size header + compressed data */ + if (pwrite(out_fd, &comp_size_be, sizeof(comp_size_be), phys) != + sizeof(comp_size_be)) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + if (pwrite(out_fd, comp_buf, comp_len, + phys + sizeof(comp_size_be)) != comp_len) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + + l2_tables[l1_idx][l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + alloc_offset += total_on_disk; + } else { + /* Store uncompressed */ + uint64_t phys = alloc_offset; + + if (pwrite(out_fd, cluster_buf, cluster_size, phys) != + cluster_size) { + fprintf(stderr, "Write error\n"); + close(in_fd); + close(out_fd); + return 1; + } + + l2_tables[l1_idx][l2_idx] = phys; + alloc_offset += cluster_size; + } + + clusters_written++; + } + + /* Write L2 tables to their allocated positions */ + for (uint32_t i = 0; i < l1_size; i++) { + if (l1_table[i] == 0) + continue; + + /* Convert L2 entries to big-endian on disk */ + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + + memset(disk_l2, 0, cluster_size); + for (uint32_t j = 0; j < l2_entries; j++) + disk_l2[j] = htobe64_val(l2_tables[i][j]); + + /* Compute and store CRC32C trailer */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t crc = crc32c_calc(raw, cluster_size - 4); + _qcow2_put32(raw, cluster_size - 4, crc); + } + + n = pwrite(out_fd, disk_l2, cluster_size, l1_table[i]); + free(disk_l2); + if (n != cluster_size) { + fprintf(stderr, "Failed to write L2 table %u\n", i); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Write header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + n = pwrite(out_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to write header\n"); + close(in_fd); + close(out_fd); + return 1; + } + + /* Write L1 table (big-endian) */ + { + uint64_t *disk_l1 = calloc(l1_size, sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + disk_l1[i] = htobe64_val(l1_table[i]); + + n = pwrite(out_fd, disk_l1, l1_size * sizeof(uint64_t), + l1_table_offset); + free(disk_l1); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to write L1 table\n"); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Truncate file to final size */ + if (ftruncate(out_fd, alloc_offset) < 0) + fprintf(stderr, "Warning: ftruncate failed: %s\n", strerror(errno)); + + close(in_fd); + close(out_fd); + + /* Cleanup */ + for (uint32_t i = 0; i < l1_size; i++) + free(l2_tables[i]); + free(l2_tables); + free(l1_table); + free(cluster_buf); + free(comp_buf); + + printf("Converted %s -> %s\n", flat_path, qcow2_path); + printf(" Virtual size: %llu bytes\n", (unsigned long long)virtual_size); + printf(" Total clusters: %llu\n", (unsigned long long)total_clusters); + printf(" Written: %llu (data)\n", (unsigned long long)clusters_written); + printf(" Zero (sparse): %llu\n", (unsigned long long)clusters_zero); + + { + struct stat out_st; + if (stat(qcow2_path, &out_st) == 0) { + printf(" File size: %llu bytes (%s)\n", + (unsigned long long)out_st.st_size, + fmt_size(out_st.st_size, (char[32]){0}, 32)); + if (virtual_size > 0) + printf(" Compression: %.1f%%\n", + (1.0 - (double)out_st.st_size / virtual_size) * 100.0); + } + } + + return 0; +} + +/* + * lbdctl extract + * Decompress all clusters from a qcow2-lz4 image to a flat file. + */ +static int cmd_extract(const char *qcow2_path, const char *flat_path) +{ + int in_fd, out_fd; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t virtual_size, l1_table_offset, alloc_off; + uint32_t cluster_bits, cluster_size, l2_entries, l1_size; + uint64_t *l1_table; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + ssize_t n; + + in_fd = open(qcow2_path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", qcow2_path, strerror(errno)); + return 1; + } + + /* Read header */ + n = pread(in_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to read header\n"); + close(in_fd); + return 1; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + fprintf(stderr, "Not a qcow2-lz4 image\n"); + close(in_fd); + return 1; + } + + cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + l1_table_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + l1_size = lbd_qcow2_hdr_l1_size(hdr); + alloc_off = lbd_qcow2_hdr_alloc_offset(hdr); + (void)alloc_off; + + /* Read L1 table */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + comp_cap = LZ4_compressBound(cluster_size); + cluster_buf = malloc(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + { + uint64_t *disk_l1 = malloc(l1_size * sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to read L1 table\n"); + free(disk_l1); + close(in_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + l1_table[i] = be64toh_val(disk_l1[i]); + free(disk_l1); + } + + /* Create output file */ + out_fd = open(flat_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create '%s': %s\n", flat_path, strerror(errno)); + close(in_fd); + return 1; + } + + /* Pre-allocate the output */ + if (ftruncate(out_fd, virtual_size) < 0) { + fprintf(stderr, "Failed to extend output: %s\n", strerror(errno)); + close(in_fd); + close(out_fd); + return 1; + } + + /* Read and verify all L2 tables */ + uint64_t **ext_l2_tables = calloc(l1_size, sizeof(uint64_t *)); + if (!ext_l2_tables) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) { + ext_l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!ext_l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + if (l1_table[i] == 0) + continue; + + uint8_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + n = pread(in_fd, disk_l2, cluster_size, l1_table[i]); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read L2 table %u\n", i); + free(disk_l2); + close(in_fd); + close(out_fd); + return 1; + } + + /* Verify CRC32C */ + { + uint32_t stored_crc = _qcow2_get32(disk_l2, cluster_size - 4); + uint32_t calc_crc = crc32c_calc(disk_l2, cluster_size - 4); + if (stored_crc != calc_crc) { + fprintf(stderr, "L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + i, stored_crc, calc_crc); + free(disk_l2); + close(in_fd); + close(out_fd); + return 1; + } + } + + for (uint32_t j = 0; j < l2_entries; j++) + ext_l2_tables[i][j] = be64toh_val(((uint64_t *)disk_l2)[j]); + free(disk_l2); + } + + /* Extract each cluster */ + uint64_t total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + uint64_t l2_entry; + uint64_t phys_offset; + size_t write_len = cluster_size; + + if (l1_idx >= l1_size || l1_table[l1_idx] == 0) { + /* Unallocated L2 table: output zeros (already zero from ftruncate) */ + continue; + } + + l2_entry = ext_l2_tables[l1_idx][l2_idx]; + + if (l2_entry == 0) { + /* Unallocated cluster: zeros */ + continue; + } + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + /* Handle last partial cluster */ + if ((ci + 1) * cluster_size > virtual_size) + write_len = virtual_size - ci * cluster_size; + + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + /* Compressed cluster */ + uint32_t comp_size_be, comp_size; + + n = pread(in_fd, &comp_size_be, sizeof(comp_size_be), + phys_offset); + if (n != sizeof(comp_size_be)) { + fprintf(stderr, "Failed to read compressed size at cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + + comp_size = be32toh_val(comp_size_be); + + n = pread(in_fd, comp_buf, comp_size, + phys_offset + sizeof(comp_size_be)); + if (n != (ssize_t)comp_size) { + fprintf(stderr, "Failed to read compressed data at cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + + int dec_len = LZ4_decompress_safe( + (const char *)comp_buf, (char *)cluster_buf, + comp_size, cluster_size); + if (dec_len != (int)cluster_size) { + fprintf(stderr, "LZ4 decompression failed at cluster %llu (got %d, expected %u)\n", + (unsigned long long)ci, dec_len, cluster_size); + close(in_fd); + close(out_fd); + return 1; + } + } else { + /* Uncompressed cluster */ + n = pread(in_fd, cluster_buf, cluster_size, phys_offset); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + } + + n = pwrite(out_fd, cluster_buf, write_len, ci * cluster_size); + if (n != (ssize_t)write_len) { + fprintf(stderr, "Failed to write cluster %llu\n", + (unsigned long long)ci); + close(in_fd); + close(out_fd); + return 1; + } + } + + close(in_fd); + close(out_fd); + for (uint32_t i = 0; i < l1_size; i++) + free(ext_l2_tables[i]); + free(ext_l2_tables); + free(l1_table); + free(cluster_buf); + free(comp_buf); + + printf("Extracted %s -> %s (%llu bytes)\n", + qcow2_path, flat_path, (unsigned long long)virtual_size); + + return 0; +} + +/* + * lbdctl compact [--inplace] + * Rewrite the image, eliminating dead space from overwrites. + */ +static int cmd_compact(int argc, char **argv) +{ + const char *path = NULL; + int in_fd, out_fd; + uint8_t hdr[LBD_QCOW2_HEADER_SIZE]; + uint64_t virtual_size, l1_table_offset; + uint32_t cluster_bits, cluster_size, l2_entries, l1_size; + uint64_t *l1_table; + uint64_t **l2_tables; + uint8_t *cluster_buf, *comp_buf; + int comp_cap; + uint64_t new_alloc_offset; + ssize_t n; + char tmp_path[PATH_MAX]; + + for (int i = 0; i < argc; i++) { + if (!path) + path = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (!path) { + fprintf(stderr, "compact requires a qcow2 file path\n"); + return 1; + } + + in_fd = open(path, O_RDONLY); + if (in_fd < 0) { + fprintf(stderr, "Cannot open '%s': %s\n", path, strerror(errno)); + return 1; + } + + /* Read header */ + n = pread(in_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0); + if (n != LBD_QCOW2_HEADER_SIZE) { + fprintf(stderr, "Failed to read header\n"); + close(in_fd); + return 1; + } + + if (lbd_qcow2_hdr_magic(hdr) != LBD_QCOW2_MAGIC) { + fprintf(stderr, "Not a qcow2-lz4 image\n"); + close(in_fd); + return 1; + } + + cluster_bits = lbd_qcow2_hdr_cluster_bits(hdr); + cluster_size = 1U << cluster_bits; + l2_entries = (cluster_size - LBD_QCOW2_L2_TRAILER_SIZE) / 8; + virtual_size = lbd_qcow2_hdr_virtual_size(hdr); + l1_table_offset = lbd_qcow2_hdr_l1_table_offset(hdr); + l1_size = lbd_qcow2_hdr_l1_size(hdr); + + /* Read L1 */ + l1_table = calloc(l1_size, sizeof(uint64_t)); + l2_tables = calloc(l1_size, sizeof(uint64_t *)); + cluster_buf = malloc(cluster_size); + comp_cap = LZ4_compressBound(cluster_size); + comp_buf = malloc(comp_cap); + + if (!l1_table || !l2_tables || !cluster_buf || !comp_buf) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + { + uint64_t *disk_l1 = malloc(l1_size * sizeof(uint64_t)); + if (!disk_l1) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) { + fprintf(stderr, "Failed to read L1 table\n"); + free(disk_l1); + close(in_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) + l1_table[i] = be64toh_val(disk_l1[i]); + free(disk_l1); + } + + /* Read all L2 tables */ + for (uint32_t i = 0; i < l1_size; i++) { + l2_tables[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!l2_tables[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + + if (l1_table[i] == 0) + continue; + + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + return 1; + } + n = pread(in_fd, disk_l2, cluster_size, l1_table[i]); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Failed to read L2 table %u\n", i); + free(disk_l2); + close(in_fd); + return 1; + } + + /* Verify CRC32C */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t stored_crc = _qcow2_get32(raw, cluster_size - 4); + uint32_t calc_crc = crc32c_calc(raw, cluster_size - 4); + if (stored_crc != calc_crc) { + fprintf(stderr, "L2 CRC32C mismatch for l1[%u]: " + "stored=0x%08x computed=0x%08x\n", + i, stored_crc, calc_crc); + free(disk_l2); + close(in_fd); + return 1; + } + } + + for (uint32_t j = 0; j < l2_entries; j++) + l2_tables[i][j] = be64toh_val(disk_l2[j]); + free(disk_l2); + } + + /* Create temp output file */ + snprintf(tmp_path, sizeof(tmp_path), "%s.compact.tmp", path); + out_fd = open(tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0644); + if (out_fd < 0) { + fprintf(stderr, "Cannot create temp file: %s\n", strerror(errno)); + close(in_fd); + return 1; + } + + /* New layout: header + L1 table, then data */ + new_alloc_offset = l1_table_offset + (uint64_t)l1_size * sizeof(uint64_t); + new_alloc_offset = (new_alloc_offset + cluster_size - 1) & ~((uint64_t)cluster_size - 1); + + /* Reserve space for header + L1 */ + if (ftruncate(out_fd, new_alloc_offset) < 0) { + fprintf(stderr, "Failed to extend temp file\n"); + close(in_fd); + close(out_fd); + return 1; + } + + /* New L1 and L2 tables */ + uint64_t *new_l1 = calloc(l1_size, sizeof(uint64_t)); + uint64_t **new_l2 = calloc(l1_size, sizeof(uint64_t *)); + if (!new_l1 || !new_l2) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + for (uint32_t i = 0; i < l1_size; i++) { + new_l2[i] = calloc(l2_entries, sizeof(uint64_t)); + if (!new_l2[i]) { + fprintf(stderr, "Out of memory\n"); + close(in_fd); + close(out_fd); + return 1; + } + } + + /* Walk all clusters and re-pack */ + uint64_t total_clusters = (virtual_size + cluster_size - 1) / cluster_size; + + for (uint64_t ci = 0; ci < total_clusters; ci++) { + uint32_t l1_idx = ci / l2_entries; + uint32_t l2_idx = ci % l2_entries; + uint64_t l2_entry; + uint64_t phys_offset; + + if (l1_idx >= l1_size || l1_table[l1_idx] == 0) + continue; + + l2_entry = l2_tables[l1_idx][l2_idx]; + if (l2_entry == 0) + continue; + + phys_offset = l2_entry & LBD_QCOW2_L2_OFFSET_MASK; + + /* Read cluster data (decompress if needed) */ + if (l2_entry & LBD_QCOW2_L2_COMPRESSED) { + uint32_t comp_size_be, comp_size; + + n = pread(in_fd, &comp_size_be, sizeof(comp_size_be), phys_offset); + if (n != sizeof(comp_size_be)) { + fprintf(stderr, "Read error\n"); + goto compact_err; + } + comp_size = be32toh_val(comp_size_be); + + n = pread(in_fd, comp_buf, comp_size, + phys_offset + sizeof(comp_size_be)); + if (n != (ssize_t)comp_size) { + fprintf(stderr, "Read error\n"); + goto compact_err; + } + + int dec_len = LZ4_decompress_safe( + (const char *)comp_buf, (char *)cluster_buf, + comp_size, cluster_size); + if (dec_len != (int)cluster_size) { + fprintf(stderr, "Decompression failed at cluster %llu\n", + (unsigned long long)ci); + goto compact_err; + } + } else { + n = pread(in_fd, cluster_buf, cluster_size, phys_offset); + if (n != (ssize_t)cluster_size) { + fprintf(stderr, "Read error at cluster %llu\n", + (unsigned long long)ci); + goto compact_err; + } + } + + /* Skip zero clusters */ + if (is_all_zero(cluster_buf, cluster_size)) + continue; + + /* Allocate L2 table for new image if needed */ + if (new_l1[l1_idx] == 0) { + new_l1[l1_idx] = new_alloc_offset; + new_alloc_offset += cluster_size; + } + + /* Recompress and write */ + int comp_len = LZ4_compress_default( + (const char *)cluster_buf, (char *)comp_buf, + cluster_size, comp_cap); + + if (comp_len > 0 && + (uint32_t)comp_len < cluster_size - sizeof(uint32_t)) { + uint32_t total_on_disk = ((sizeof(uint32_t) + comp_len) + 4095) & ~4095U; + uint64_t phys = new_alloc_offset; + uint32_t comp_size_be = htobe32_val(comp_len); + + if (pwrite(out_fd, &comp_size_be, sizeof(comp_size_be), phys) != + sizeof(comp_size_be)) + goto compact_err; + if (pwrite(out_fd, comp_buf, comp_len, + phys + sizeof(comp_size_be)) != comp_len) + goto compact_err; + + new_l2[l1_idx][l2_idx] = LBD_QCOW2_L2_COMPRESSED | phys; + new_alloc_offset += total_on_disk; + } else { + uint64_t phys = new_alloc_offset; + + if (pwrite(out_fd, cluster_buf, cluster_size, phys) != + (ssize_t)cluster_size) + goto compact_err; + + new_l2[l1_idx][l2_idx] = phys; + new_alloc_offset += cluster_size; + } + } + + /* Write L2 tables */ + for (uint32_t i = 0; i < l1_size; i++) { + if (new_l1[i] == 0) + continue; + + uint64_t *disk_l2 = malloc(cluster_size); + if (!disk_l2) + goto compact_err; + + memset(disk_l2, 0, cluster_size); + for (uint32_t j = 0; j < l2_entries; j++) + disk_l2[j] = htobe64_val(new_l2[i][j]); + + /* Compute and store CRC32C trailer */ + { + uint8_t *raw = (uint8_t *)disk_l2; + uint32_t crc = crc32c_calc(raw, cluster_size - 4); + _qcow2_put32(raw, cluster_size - 4, crc); + } + + n = pwrite(out_fd, disk_l2, cluster_size, new_l1[i]); + free(disk_l2); + if (n != (ssize_t)cluster_size) + goto compact_err; + } + + /* Write header */ + memset(hdr, 0, LBD_QCOW2_HEADER_SIZE); + lbd_qcow2_hdr_set_magic(hdr, LBD_QCOW2_MAGIC); + lbd_qcow2_hdr_set_version(hdr, LBD_QCOW2_VERSION); + lbd_qcow2_hdr_set_cluster_bits(hdr, cluster_bits); + lbd_qcow2_hdr_set_virtual_size(hdr, virtual_size); + lbd_qcow2_hdr_set_l1_table_offset(hdr, l1_table_offset); + lbd_qcow2_hdr_set_l1_size(hdr, l1_size); + lbd_qcow2_hdr_set_alloc_offset(hdr, new_alloc_offset); + lbd_qcow2_hdr_set_comp_type(hdr, LBD_QCOW2_COMP_LZ4); + lbd_qcow2_hdr_set_free_list(hdr, 0); + + if (pwrite(out_fd, hdr, LBD_QCOW2_HEADER_SIZE, 0) != LBD_QCOW2_HEADER_SIZE) + goto compact_err; + + /* Write L1 table */ + { + uint64_t *disk_l1 = calloc(l1_size, sizeof(uint64_t)); + if (!disk_l1) + goto compact_err; + for (uint32_t i = 0; i < l1_size; i++) + disk_l1[i] = htobe64_val(new_l1[i]); + n = pwrite(out_fd, disk_l1, l1_size * sizeof(uint64_t), l1_table_offset); + free(disk_l1); + if (n != (ssize_t)(l1_size * sizeof(uint64_t))) + goto compact_err; + } + + if (ftruncate(out_fd, new_alloc_offset) < 0) + fprintf(stderr, "Warning: ftruncate failed\n"); + + close(in_fd); + close(out_fd); + + /* Replace original with compacted */ + if (rename(tmp_path, path) < 0) { + fprintf(stderr, "Failed to rename %s -> %s: %s\n", + tmp_path, path, strerror(errno)); + return 1; + } + + printf("Compacted %s\n", path); + { + struct stat st; + if (stat(path, &st) == 0) { + printf(" File size: %llu bytes (%s)\n", + (unsigned long long)st.st_size, + fmt_size(st.st_size, (char[32]){0}, 32)); + } + } + + /* Cleanup */ + for (uint32_t i = 0; i < l1_size; i++) { + free(l2_tables[i]); + free(new_l2[i]); + } + free(l2_tables); + free(new_l2); + free(l1_table); + free(new_l1); + free(cluster_buf); + free(comp_buf); + return 0; + +compact_err: + fprintf(stderr, "Compaction failed\n"); + close(in_fd); + close(out_fd); + unlink(tmp_path); + for (uint32_t i = 0; i < l1_size; i++) { + free(l2_tables[i]); + free(new_l2[i]); + } + free(l2_tables); + free(new_l2); + free(l1_table); + free(new_l1); + free(cluster_buf); + free(comp_buf); + return 1; +} + +/* ---------------------------------------------------------------- + * Watch command (log rotation notifications) + * ---------------------------------------------------------------- */ + +static volatile sig_atomic_t watch_running = 1; + +static void watch_sigint(int sig) +{ + (void)sig; + watch_running = 0; +} + +/* + * Minimal CBOR encoder for userspace (write path). + * Encodes a CBOR head: major type (top 3 bits) + value. + */ +static size_t cbor_write_head(uint8_t *buf, uint8_t major, uint64_t val) +{ + uint8_t mt = major << 5; + + if (val < 24) { + buf[0] = mt | (uint8_t)val; + return 1; + } else if (val <= 0xFF) { + buf[0] = mt | 24; + buf[1] = (uint8_t)val; + return 2; + } else if (val <= 0xFFFF) { + buf[0] = mt | 25; + buf[1] = (uint8_t)(val >> 8); + buf[2] = (uint8_t)val; + return 3; + } else if (val <= 0xFFFFFFFF) { + buf[0] = mt | 26; + buf[1] = (uint8_t)(val >> 24); + buf[2] = (uint8_t)(val >> 16); + buf[3] = (uint8_t)(val >> 8); + buf[4] = (uint8_t)val; + return 5; + } else { + buf[0] = mt | 27; + buf[1] = (uint8_t)(val >> 56); + buf[2] = (uint8_t)(val >> 48); + buf[3] = (uint8_t)(val >> 40); + buf[4] = (uint8_t)(val >> 32); + buf[5] = (uint8_t)(val >> 24); + buf[6] = (uint8_t)(val >> 16); + buf[7] = (uint8_t)(val >> 8); + buf[8] = (uint8_t)val; + return 9; + } +} + +/* + * Encode and write a watch command to the control device fd. + * dev_index < 0 means watch all devices. + */ +static int cbor_write_watch_cmd(int fd, int dev_index) +{ + uint8_t buf[64]; + size_t pos = 0; + const char *cmd_str = "watch"; + size_t cmd_len = 5; + int map_items = (dev_index >= 0) ? 2 : 1; + ssize_t n; + + /* map(1 or 2) */ + pos += cbor_write_head(buf + pos, 5, map_items); + + /* key 1: "watch" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 2: dev_index (optional) */ + if (dev_index >= 0) { + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_DEV); + pos += cbor_write_head(buf + pos, 0, (uint64_t)dev_index); + } + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write watch command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on watch command\n"); + return -1; + } + return 0; +} + +/* + * Decode a CBOR-encoded event message from a buffer. + * Uses the existing fd-based cbor_read_* functions via a temporary + * approach: we decode inline from the buffer. + */ +static int decode_watch_event(const uint8_t *buf, size_t len, + int *dev_index, char *label, size_t label_cap, + char *dir, size_t dir_cap, + uint64_t *log_seq, uint64_t *device_size) +{ + size_t pos = 0; + uint8_t ib, major, ai; + uint64_t map_count, key, val; + uint64_t slen; + + /* Inline buffer-based CBOR decoder (mirrors cbor_dec.h logic) */ +#define BUF_HEAD(major_out, val_out) do { \ + if (pos >= len) return -1; \ + ib = buf[pos++]; \ + *(major_out) = ib >> 5; \ + ai = ib & 0x1F; \ + if (ai < 24) { *(val_out) = ai; } \ + else if (ai == 24) { \ + if (pos + 1 > len) return -1; \ + *(val_out) = buf[pos++]; \ + } else if (ai == 25) { \ + if (pos + 2 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 8) | buf[pos+1]; \ + pos += 2; \ + } else if (ai == 26) { \ + if (pos + 4 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 24) | \ + ((uint64_t)buf[pos+1] << 16) | \ + ((uint64_t)buf[pos+2] << 8) | \ + buf[pos+3]; \ + pos += 4; \ + } else if (ai == 27) { \ + if (pos + 8 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 56) | \ + ((uint64_t)buf[pos+1] << 48) | \ + ((uint64_t)buf[pos+2] << 40) | \ + ((uint64_t)buf[pos+3] << 32) | \ + ((uint64_t)buf[pos+4] << 24) | \ + ((uint64_t)buf[pos+5] << 16) | \ + ((uint64_t)buf[pos+6] << 8) | \ + buf[pos+7]; \ + pos += 8; \ + } else { return -1; } \ +} while (0) + + /* Read map header */ + BUF_HEAD(&major, &map_count); + if (major != 5) + return -1; + + *dev_index = -1; + *log_seq = 0; + *device_size = 0; + label[0] = '\0'; + dir[0] = '\0'; + + for (uint64_t i = 0; i < map_count; i++) { + /* Read key (uint) */ + BUF_HEAD(&major, &key); + if (major != 0) + return -1; + + switch (key) { + case LBD_EVENT_KEY_TYPE: + /* text string — skip it */ + BUF_HEAD(&major, &slen); + if (major != 3 || pos + slen > len) + return -1; + pos += slen; + break; + case LBD_EVENT_KEY_DEV: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *dev_index = (int)val; + break; + case LBD_EVENT_KEY_LABEL: + BUF_HEAD(&major, &slen); + if (major != 3 || slen >= label_cap || pos + slen > len) + return -1; + memcpy(label, buf + pos, slen); + label[slen] = '\0'; + pos += slen; + break; + case LBD_EVENT_KEY_DIR: + BUF_HEAD(&major, &slen); + if (major != 3 || slen >= dir_cap || pos + slen > len) + return -1; + memcpy(dir, buf + pos, slen); + dir[slen] = '\0'; + pos += slen; + break; + case LBD_EVENT_KEY_SEQ: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *log_seq = val; + break; + case LBD_EVENT_KEY_SIZE: + BUF_HEAD(&major, &val); + if (major != 0) + return -1; + *device_size = val; + break; + default: + return -1; + } + } + +#undef BUF_HEAD + return 0; +} + +static int cmd_watch(int argc, char **argv) +{ + int fd; + int dev_filter = -1; + int json = 0; + struct pollfd pfd; + struct sigaction sa; + uint8_t rbuf[512]; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_filter = atoi(argv[i]); + } else if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + /* Send watch command */ + if (cbor_write_watch_cmd(fd, dev_filter) < 0) { + close(fd); + return 1; + } + + if (!json) { + if (dev_filter >= 0) + fprintf(stderr, "Watching lbd%d for log rotations...\n", + dev_filter); + else + fprintf(stderr, "Watching all devices for log rotations...\n"); + } + + /* Setup SIGINT handler for clean exit */ + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = watch_sigint; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + pfd.fd = fd; + pfd.events = POLLIN; + + while (watch_running) { + int ret = poll(&pfd, 1, 1000); + + if (ret < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "poll error: %s\n", strerror(errno)); + break; + } + if (ret == 0) + continue; + + if (pfd.revents & POLLIN) { + ssize_t n = read(fd, rbuf, sizeof(rbuf)); + + if (n < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "read error: %s\n", + strerror(errno)); + break; + } + if (n == 0) + break; + + int dev_idx; + char label[32]; + char dir[LBD_LOG_PATH_MAX]; + uint64_t log_seq, device_size; + + if (decode_watch_event(rbuf, (size_t)n, + &dev_idx, label, sizeof(label), + dir, sizeof(dir), + &log_seq, &device_size) < 0) { + fprintf(stderr, "Failed to decode event\n"); + continue; + } + + if (json) { + printf("{\"type\":\"log_rotated\"," + "\"dev\":%d," + "\"segment_label\":\"%s\"," + "\"log_dir\":", + dev_idx, label); + json_print_string(stdout, dir); + printf(",\"log_seq\":%llu," + "\"device_size\":%llu}\n", + (unsigned long long)log_seq, + (unsigned long long)device_size); + } else { + printf("lbd%d: %s/disk.%s.log (seq=%llu)\n", + dev_idx, dir, label, + (unsigned long long)log_seq); + } + fflush(stdout); + } + + if (pfd.revents & (POLLERR | POLLHUP)) + break; + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Miss handler command + * ---------------------------------------------------------------- */ + +/* + * Encode and write a manage_misses command. + */ +static int cbor_write_manage_misses_cmd(int fd, int dev_index) +{ + uint8_t buf[64]; + size_t pos = 0; + const char *cmd_str = "manage_misses"; + size_t cmd_len = strlen(cmd_str); + ssize_t n; + + /* map(2) */ + pos += cbor_write_head(buf + pos, 5, 2); + + /* key 1: "manage_misses" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 2: dev_index */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_DEV); + pos += cbor_write_head(buf + pos, 0, (uint64_t)dev_index); + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write manage_misses command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on manage_misses command\n"); + return -1; + } + return 0; +} + +/* + * Encode and write a continue/retry command. + */ +static int cbor_write_miss_response(int fd, const char *action) +{ + uint8_t buf[64]; + size_t pos = 0; + size_t cmd_len = strlen(action); + ssize_t n; + + /* map(1) */ + pos += cbor_write_head(buf + pos, 5, 1); + + /* key 1: action */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, action, cmd_len); + pos += cmd_len; + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write %s command: %s\n", + action, strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on %s command\n", action); + return -1; + } + return 0; +} + +/* + * Decode a miss event from a CBOR buffer. + * Returns 0 on success, -1 on error. + */ +static int decode_miss_event(const uint8_t *buf, size_t len, + int *dev_index, uint64_t *cluster) +{ + size_t pos = 0; + uint8_t ib, major, ai; + uint64_t map_count, key, val; + uint64_t slen; + +#define BUF_HEAD2(major_out, val_out) do { \ + if (pos >= len) return -1; \ + ib = buf[pos++]; \ + *(major_out) = ib >> 5; \ + ai = ib & 0x1F; \ + if (ai < 24) { *(val_out) = ai; } \ + else if (ai == 24) { \ + if (pos + 1 > len) return -1; \ + *(val_out) = buf[pos++]; \ + } else if (ai == 25) { \ + if (pos + 2 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 8) | buf[pos+1]; \ + pos += 2; \ + } else if (ai == 26) { \ + if (pos + 4 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 24) | \ + ((uint64_t)buf[pos+1] << 16) | \ + ((uint64_t)buf[pos+2] << 8) | \ + buf[pos+3]; \ + pos += 4; \ + } else if (ai == 27) { \ + if (pos + 8 > len) return -1; \ + *(val_out) = ((uint64_t)buf[pos] << 56) | \ + ((uint64_t)buf[pos+1] << 48) | \ + ((uint64_t)buf[pos+2] << 40) | \ + ((uint64_t)buf[pos+3] << 32) | \ + ((uint64_t)buf[pos+4] << 24) | \ + ((uint64_t)buf[pos+5] << 16) | \ + ((uint64_t)buf[pos+6] << 8) | \ + buf[pos+7]; \ + pos += 8; \ + } else { return -1; } \ +} while (0) + + BUF_HEAD2(&major, &map_count); + if (major != 5) + return -1; + + *dev_index = -1; + *cluster = 0; + + for (uint64_t i = 0; i < map_count; i++) { + BUF_HEAD2(&major, &key); + if (major != 0) + return -1; + + switch (key) { + case LBD_MISS_KEY_TYPE: + BUF_HEAD2(&major, &slen); + if (major != 3 || pos + slen > len) + return -1; + pos += slen; + break; + case LBD_MISS_KEY_DEV: + BUF_HEAD2(&major, &val); + if (major != 0) + return -1; + *dev_index = (int)val; + break; + case LBD_MISS_KEY_CLUSTER: + BUF_HEAD2(&major, &val); + if (major != 0) + return -1; + *cluster = val; + break; + default: + return -1; + } + } + +#undef BUF_HEAD2 + return 0; +} + +static int cmd_miss_handler(int argc, char **argv) +{ + int fd; + int dev_index = -1; + int json = 0; + struct pollfd pfd; + struct sigaction sa; + uint8_t rbuf[512]; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_index = atoi(argv[i]); + } else if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (dev_index < 0) { + fprintf(stderr, "miss-handler requires --dev N\n"); + return 1; + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + if (cbor_write_manage_misses_cmd(fd, dev_index) < 0) { + close(fd); + return 1; + } + + if (!json) + fprintf(stderr, "Handling block misses for lbd%d...\n", + dev_index); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = watch_sigint; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + pfd.fd = fd; + pfd.events = POLLIN; + + while (watch_running) { + int ret = poll(&pfd, 1, 1000); + + if (ret < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "poll error: %s\n", strerror(errno)); + break; + } + if (ret == 0) + continue; + + if (pfd.revents & POLLIN) { + ssize_t n = read(fd, rbuf, sizeof(rbuf)); + + if (n < 0) { + if (errno == EINTR) + continue; + fprintf(stderr, "read error: %s\n", + strerror(errno)); + break; + } + if (n == 0) + break; + + int miss_dev; + uint64_t miss_cluster; + + if (decode_miss_event(rbuf, (size_t)n, + &miss_dev, &miss_cluster) < 0) { + fprintf(stderr, "Failed to decode miss event\n"); + continue; + } + + if (json) { + printf("{\"type\":\"block_miss\"," + "\"dev\":%d," + "\"cluster\":%llu}\n", + miss_dev, + (unsigned long long)miss_cluster); + } else { + printf("lbd%d: block miss at cluster %llu\n", + miss_dev, + (unsigned long long)miss_cluster); + } + fflush(stdout); + + /* Respond with continue */ + if (cbor_write_miss_response(fd, "continue") < 0) + break; + } + + if (pfd.revents & (POLLERR | POLLHUP)) + break; + } + + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Swap command + * ---------------------------------------------------------------- */ + +/* + * Encode and write a swap command with a path. + */ +static int cbor_write_swap_cmd(int fd, const char *path) +{ + uint8_t buf[512]; + size_t pos = 0; + const char *cmd_str = "swap"; + size_t cmd_len = strlen(cmd_str); + size_t path_len = strlen(path); + ssize_t n; + + /* map(2) */ + pos += cbor_write_head(buf + pos, 5, 2); + + /* key 1: "swap" */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_CMD); + pos += cbor_write_head(buf + pos, 3, cmd_len); + memcpy(buf + pos, cmd_str, cmd_len); + pos += cmd_len; + + /* key 3: path */ + pos += cbor_write_head(buf + pos, 0, LBD_WATCH_KEY_PATH); + pos += cbor_write_head(buf + pos, 3, path_len); + if (pos + path_len > sizeof(buf)) { + fprintf(stderr, "Path too long for CBOR buffer\n"); + return -1; + } + memcpy(buf + pos, path, path_len); + pos += path_len; + + n = write(fd, buf, pos); + if (n < 0) { + fprintf(stderr, "Failed to write swap command: %s\n", + strerror(errno)); + return -1; + } + if ((size_t)n != pos) { + fprintf(stderr, "Short write on swap command\n"); + return -1; + } + return 0; +} + +static int cmd_swap(int argc, char **argv) +{ + int fd; + int dev_index = -1; + const char *path = NULL; + char resolved[PATH_MAX]; + int json = 0; + + for (int i = 0; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) { + json = 1; + } else if (strcmp(argv[i], "--dev") == 0) { + if (++i >= argc) { + fprintf(stderr, "--dev requires a value\n"); + return 1; + } + dev_index = atoi(argv[i]); + } else if (!path) { + path = argv[i]; + } else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + return 1; + } + } + + if (dev_index < 0) { + fprintf(stderr, "swap requires --dev N\n"); + return 1; + } + if (!path) { + fprintf(stderr, "swap requires a path argument\n"); + return 1; + } + + if (!realpath(path, resolved)) { + fprintf(stderr, "Cannot resolve path '%s': %s\n", + path, strerror(errno)); + return 1; + } + + fd = open(LBD_CTL_PATH, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Cannot open %s: %s\n", + LBD_CTL_PATH, strerror(errno)); + if (errno == ENOENT) + fprintf(stderr, "Is the lbd module loaded?\n"); + return 1; + } + + /* First register as miss handler for the device */ + if (cbor_write_manage_misses_cmd(fd, dev_index) < 0) { + close(fd); + return 1; + } + + /* Send swap command */ + if (cbor_write_swap_cmd(fd, resolved) < 0) { + close(fd); + return 1; + } + + if (json) { + printf("{\"device\": \"/dev/lbd%d\", \"index\": %d, \"base\": ", + dev_index, dev_index); + json_print_string(stdout, resolved); + printf("}\n"); + } else { + printf("Swapped base layer for lbd%d to %s\n", dev_index, resolved); + } + close(fd); + return 0; +} + +/* ---------------------------------------------------------------- + * Main + * ---------------------------------------------------------------- */ + +static void usage(void) +{ + fprintf(stderr, + "Usage:\n" + " lbdctl add [opts] Create a new lbd device backed by \n" + " lbdctl remove Remove /dev/lbdN\n" + " lbdctl list List all active lbd devices\n" + " lbdctl watch [opts] Watch for log rotation events\n" + " lbdctl miss-handler [opts] Handle block miss events\n" + " lbdctl swap [opts] Swap base layer for a device\n" + " lbdctl log [opts] Read and display a log file\n" + " lbdctl create --size Create empty qcow2-lz4 image\n" + " lbdctl convert Convert flat image to qcow2-lz4\n" + " lbdctl extract Extract qcow2-lz4 to flat image\n" + " lbdctl compact Compact qcow2-lz4 image (reclaim space)\n" + "\n" + "Global options:\n" + " --json Output as JSON\n" + "\n" + "Add options:\n" + " --log-dir Directory to write log files (required)\n" + " --base Read-only base layer image (thin snapshot)\n" + " --log-max-size Segment rotation size (default 64 MiB)\n" + " --log-max-age Segment rotation age (default 60s)\n" + "\n" + "Watch options:\n" + " --dev Only watch device N (default: all)\n" + " --json Output events as JSON\n" + "\n" + "Miss handler options:\n" + " --dev Device to handle misses for (required)\n" + " --json Output events as JSON\n" + "\n" + "Swap options:\n" + " --dev Device to swap base for (required)\n" + "\n" + "Create options:\n" + " --size Virtual device size (supports K/M/G/T suffixes)\n" + " --cluster-bits log2(cluster_size), default 16 (64 KiB)\n" + "\n" + "Log options:\n" + " --json Output as JSON\n" + " --data Include hex data in output\n"); +} + +int main(int argc, char **argv) +{ + if (argc < 2) { + usage(); + return 1; + } + + if (strcmp(argv[1], "add") == 0) { + if (argc < 3) { + fprintf(stderr, "add requires a path argument\n"); + return 1; + } + return cmd_add(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "remove") == 0) { + return cmd_remove(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "list") == 0) { + return cmd_list(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "watch") == 0) { + return cmd_watch(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "miss-handler") == 0) { + return cmd_miss_handler(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "swap") == 0) { + return cmd_swap(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "create") == 0) { + if (argc < 3) { + fprintf(stderr, "create requires --size and a path\n"); + return 1; + } + return cmd_create(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "convert") == 0) { + if (argc < 4) { + fprintf(stderr, "convert requires \n"); + return 1; + } + return cmd_convert(argv[2], argv[3]); + } + + if (strcmp(argv[1], "extract") == 0) { + if (argc < 4) { + fprintf(stderr, "extract requires \n"); + return 1; + } + return cmd_extract(argv[2], argv[3]); + } + + if (strcmp(argv[1], "compact") == 0) { + if (argc < 3) { + fprintf(stderr, "compact requires a qcow2 file path\n"); + return 1; + } + return cmd_compact(argc - 2, argv + 2); + } + + if (strcmp(argv[1], "log") == 0) { + int json = 0, show_data = 0; + const char *logpath = NULL; + + for (int i = 2; i < argc; i++) { + if (strcmp(argv[i], "--json") == 0) + json = 1; + else if (strcmp(argv[i], "--data") == 0) + show_data = 1; + else if (!logpath) + logpath = argv[i]; + else { + fprintf(stderr, "Unexpected argument: %s\n", + argv[i]); + return 1; + } + } + + if (!logpath) { + fprintf(stderr, "log requires a log file path\n"); + return 1; + } + return cmd_log(logpath, json, show_data); + } + + fprintf(stderr, "Unknown command: %s\n", argv[1]); + usage(); + return 1; +} diff --git a/third_party/lbd/src/lz4/lz4.c b/third_party/lbd/src/lz4/lz4.c new file mode 100644 index 000000000..72b349f88 --- /dev/null +++ b/third_party/lbd/src/lz4/lz4.c @@ -0,0 +1,2842 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (c) Yann Collet. All rights reserved. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4_HEAPMODE : + * Select how stateless compression functions like `LZ4_compress_default()` + * allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * LZ4_ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) \ + || (defined(__riscv) && defined(__riscv_zicclsm)) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + + +/*-************************************ +* Dependency +**************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +# define LZ4_STATIC_LINKING_ONLY +#endif +#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ +# pragma warning(disable : 6239) /* disable: C6239: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6240) /* disable: C6240: ( && ) always evaluates to the result of */ +# pragma warning(disable : 6326) /* disable: C6326: Potential comparison of a constant with another constant */ +#endif /* _MSC_VER */ + +#ifndef LZ4_FORCE_INLINE +# if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + + +/*-************************************ +* Memory routines +**************************************/ + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " %i: ", __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + + +/*-************************************ +* Types +**************************************/ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) + /* types provided by the embedding environment */ + typedef unsigned char BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef size_t uptrval; +#else +#include +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef unsigned char BYTE; /*uint8_t not necessarily blessed to alias arbitrary type*/ + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif +#endif /* LZ4_FREESTANDING */ + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + +/*-************************************ +* Reading and writing into memory +**************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif +#endif + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define LZ4_PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__)) +#elif defined(_MSC_VER) +#define LZ4_PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop)) +#endif + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16; +LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32; +LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST; + +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalignST*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign16*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign32*)memPtr)->u32 = value; } + +#else /* safe and portable access using memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] | (p[1]<<8)); + } +} + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT +static U32 LZ4_readLE32(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read32(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U32)p[0] | (p[1]<<8) | (p[2]<<16) | (p[3]<<24); + } +} +#endif + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 12 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif + LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(pop) +#endif + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif + + +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# endif +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +#endif +# endif + } else /* 32 bits */ { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } + } +} + + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } + + +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT + return LZ4_hash4(LZ4_readLE32(p), tableType); +#else + return LZ4_hash4(LZ4_read32(p), tableType); +#endif +} + +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +/* LZ4_putPosition*() : only used in byPtr mode */ +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType) +{ + const BYTE** const hashTable = (const BYTE**)tableBase; + assert(tableType == byPtr); (void)tableType; + hashTable[h] = p; +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType); +} + +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + assert(tableType == byPtr); (void)tableType; + { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } +} + +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType); +} + +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", (void*)cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic_validated() : + * inlined, to ensure branches are decided at compilation time. + * The following conditions are presumed already validated: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int* inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + int result; + const BYTE* ip = (const BYTE*)source; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*)source - startIndex; + const BYTE* lowLimit; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = + (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with indexes in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE* const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 offset = 0; + U32 forwardH; + + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + if (tableType == byU16) assert(inputSize= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, byPtr); + } else { + LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); + } } + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + const BYTE* filledIp; + + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); + } + + /* Catch up */ + filledIp = ip; + assert(ip > anchor); /* this is always true as ip has been advanced before entering the main loop */ + if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) { + do { ip--; match--; } while (((ip > anchor) & (match > lowLimit)) && (unlikely(ip[-1] == match[-1]))); + } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + unsigned len = litLength - RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + + /* Encode Offset */ + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } + + /* Encode MatchLength */ + { unsigned matchCode; + + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); + } + + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); + + anchor = ip; + + /* Test end of chunk */ + if (ip >= mflimitPlusOne) break; + + /* Fill table */ + { U32 const h = LZ4_hashPosition(ip-2, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); + } else { + U32 const idx = (U32)((ip-2) - base); + LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); + } } + + /* Test next position */ + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType); + LZ4_putPosition(ip, cctx->hashTable, tableType); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + assert(ctx != NULL); + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + + +int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, src, dest, srcSize, dstCapacity, acceleration); + +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast(src, dst, srcSize, dstCapacity, 1); +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration); + } } +} + +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration); + /* clean the state on exit */ + LZ4_initStream(state, sizeof (LZ4_stream_t)); + return r; +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* const ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); + +#if (LZ4_HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/*-****************************** +* Streaming functions +********************************/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); + DEBUGLOG(4, "LZ4_createStream %p", (void*)lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} +#endif + +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", (void*)LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", (void*)LZ4_stream); + FREEMEM(LZ4_stream); + return (0); +} +#endif + + +typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, + const char* dictionary, int dictSize, + LoadDict_mode_e _ld) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + U32 idx32; + + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, (void*)dictionary, (void*)LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = (U32)tableType; + idx32 = dict->currentOffset - dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + /* Note: overwriting => favors positions end of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + p+=3; idx32+=3; + } + + if (_ld == _ld_slow) { + /* Fill hash table with additional references, to improve compression capability */ + p = dict->dictionary; + idx32 = dict->currentOffset - dict->dictSize; + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + U32 const limit = dict->currentOffset - 64 KB; + if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) { + /* Note: not overwriting => favors positions beginning of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + } + p++; idx32++; + } + } + + return (int)dict->dictSize; +} + +int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast); +} + +int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow); +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + (void*)workingStream, (void*)dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) +{ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + DEBUGLOG(4, "LZ4_renormDictT"); + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + const tableType_t tableType = byU32; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, (void*)streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = source; + } + + /* Check overlapping input/dictionary space */ + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == source) { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); + else + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); + } + + /* external dictionary mode */ + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) +{ + LZ4_stream_t_internal* const streamPtr = &LZ4_dict->internal_donotuse; + int result; + + LZ4_renormDictT(streamPtr, srcSize); + + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)srcSize; + + return result; +} + + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, (void*)safeBuffer); + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +static size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; + } + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + if (likely(s != 255)) return length; + do { + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + } while (s == 255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. + */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + if ((src == NULL) || (outputSize < 0)) { return -1; } + + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(64 KB)); + + + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + + /* Special cases */ + assert(lowPrefix <= op); + if (unlikely(outputSize==0)) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if (unlikely(srcSize==0)) { return -1; } + + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "move to safe decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using fast decode loop"); + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long literal length"); + goto _output_error; + } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, op+length); + ip += length; op += length; + } else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) { + /* We don't need to check oend, since we check it once for each loop below */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); + ip += length; op += length; + } else { + goto safe_literal_copy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + DEBUGLOG(6, "blockPos%6u: offset = %u", (unsigned)(op-(BYTE*)dst), (unsigned)offset); + match = op - offset; + assert(match <= op); /* overflow check */ + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH); + + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { + DEBUGLOG(5, "error reading long match length"); + goto _output_error; + } + length += addl; + length += MINMATCH; + DEBUGLOG(7, " long match length == %u", (unsigned)length); + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length); + goto safe_match_copy; + } + + /* Fastpath check: skip LZ4_wildCopy32 when true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { + DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); + goto _output_error; + } + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + DEBUGLOG(6, "end-of-block condition violated") + goto _output_error; + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using safe decode loop"); + while (1) { + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (length != RUN_MASK) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((ip < shortiend) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, 16); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4); + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + /* copy literals */ + cpy = op+length; + + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((ip+length != iend) || (cpy > oend)) { + DEBUGLOG(5, "should have been last run of literals") + DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", (void*)ip, (int)length, (void*)(ip+length), (void*)iend); + DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", (void*)cpy, (void*)(oend-MFLIMIT)); + DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize); + goto _output_error; + } + } + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + _copy_match: + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } + + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ + } + + /* end of decoding */ + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } +} + + +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + decode_full_block, noDict, + (BYTE*)dest, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + partial_decode, + noDict, (BYTE*)dst, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; +} +#endif + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +LZ4_FORCE_O2 +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); + int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + DEBUGLOG(5, "prefix becomes extDict"); + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); +} + + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} + +/* +These decompression functions are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_create (char* inputBuffer) +{ + (void)inputBuffer; + return LZ4_createStream(); +} +#endif + +char* LZ4_slideInputBuffer (void* state) +{ + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; +} + +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/third_party/lbd/src/lz4/lz4.h b/third_party/lbd/src/lz4/lz4.h new file mode 100644 index 000000000..b25d83f0b --- /dev/null +++ b/third_party/lbd/src/lz4/lz4.h @@ -0,0 +1,894 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (c) Yann Collet. All rights reserved. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#if !defined(LZ4_FREESTANDING) || (LZ4_FREESTANDING != 1) +#include /* size_t */ +#endif + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. +*/ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ + + +/*-************************************ +* Tuning memory usage +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Can be selected at compile time, by setting LZ4_MEMORY_USAGE. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB) + * Increasing memory usage improves compression ratio, generally at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. + * Default value is 14, for 16KB, which nicely fits into most L1 caches. + */ +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +/* These are absolute limits, they should not be changed by users */ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * @compressedSize : is the exact complete size of the compressed block. + * @dstCapacity : is the size of destination buffer (which must be already allocated), + * presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). +*/ +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'dstCapacity'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : in+out parameter. Initially contains size of input. + * Will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails. + * + * Note : 'targetDstSize' must be >= 1, because it's the smallest valid lz4 payload. + * + * Note 2:from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): + * the produced compressed content could, in rare circumstances, + * require to be decompressed into a destination buffer + * larger by at least 1 byte than decompressesSize. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize); + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic. + * When in doubt, employ the Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_loadDictSlow() : v1.10.0+ + * Same as LZ4_loadDict(), + * but uses a bit more cpu to reference the dictionary content more thoroughly. + * This is expected to slightly improve compression ratio. + * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_attach_dictionary() : stable since v1.10.0 + * + * This allows efficient re-use of a static dictionary multiple times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references @dictionaryStream in-place. + * + * Several assumptions are made about the state of @dictionaryStream. + * Currently, only states which have been prepared by LZ4_loadDict() or + * LZ4_loadDictSlow() should be expected to work. + * + * Alternatively, the provided @dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. + * @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the compression session. + * + * Note: there is no equivalent LZ4_attach_*() method on the decompression side + * because there is no initialization cost, hence no need to share the cost across multiple sessions. + * To decompress LZ4 blocks using dictionary, attached or not, + * just employ the regular LZ4_setStreamDecode() for streaming, + * or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression. + */ +LZ4LIB_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_safe_continue() : + * This decoding function allows decompression of consecutive blocks in "streaming" mode. + * The difference with the usual independent blocks is that + * new blocks are allowed to find references into former blocks. + * A block is an unsplittable entity, and must be presented entirely to the decompression function. + * LZ4_decompress_safe_continue() only accepts one block at a time. + * It's modeled after `LZ4_decompress_safe()` and behaves similarly. + * + * @LZ4_streamDecode : decompression state, tracking the position in memory of past data + * @compressedSize : exact complete size of one compressed block. + * @dstCapacity : size of destination buffer (which must be already allocated), + * must be an upper bound of decompressed size. + * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * + * The last 64KB of previously decoded data *must* remain available and unmodified + * at the memory position where they were previously decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. +*/ +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); + + +/*! LZ4_decompress_safe_usingDict() : + * Works the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() + * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +/*! LZ4_decompress_safe_partial_usingDict() : + * Behaves the same as LZ4_decompress_safe_partial() + * with the added ability to specify a memory segment for past data. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +# define LZ4LIB_STATIC_API LZ4LIB_API +#else +# define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize_extState() : introduced in v1.10.0 + * Same as LZ4_compress_destSize(), but using an externally allocated state. + * Also: exposes @acceleration + */ +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration); + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly constrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) + /* types provided by the embedding environment (e.g. ) */ + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#elif defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef int8_t LZ4_i8; + typedef unsigned char LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif + +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ +}; + +#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */ +union LZ4_stream_u { + char minStateSize[LZ4_STREAM_MINSIZE]; + LZ4_stream_t_internal internal_donotuse; +}; /* previously typedef'd to LZ4_stream_t */ + + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead +**/ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size); + + +/*! LZ4_streamDecode_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 +union LZ4_streamDecode_u { + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + + +/*-************************************ +* Obsolete Functions +**************************************/ + +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# else +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. " + "Note that the contract will change (requires block's compressed size, instead of decompressed size)") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + +#endif /* LZ4_H_98237428734687 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/third_party/lbd/src/lz4_kcompat.h b/third_party/lbd/src/lz4_kcompat.h new file mode 100644 index 000000000..d1e7703d4 --- /dev/null +++ b/third_party/lbd/src/lz4_kcompat.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LZ4_KCOMPAT_H +#define _LZ4_KCOMPAT_H + +#ifdef __KERNEL__ +#define LZ4_FREESTANDING 1 +#define LZ4_memcpy __builtin_memcpy +#define LZ4_memmove __builtin_memmove +#define LZ4_memset __builtin_memset +#define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +#include +#endif + +#include "lz4/lz4.h" + +#endif /* _LZ4_KCOMPAT_H */ From ab57bb070dc074b03c265e2ba73523b7698d3432 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 22:10:34 -0700 Subject: [PATCH 02/15] Build and load the lbd kernel module on demand Accelerator mode has never actually been reachable on a real install. It needs the lbd kernel module and lbdctl, and miren shipped neither, so every disk quietly fell back to loop devices unless an operator had built lbd by hand. The module ships a dkms.conf, but DKMS wants a C toolchain and DKMS itself on every node, which is a prerequisite we did not want to put in front of people. So the toolchain goes in a container instead. `miren disk accelerator install` pulls a builder image, compiles the embedded source against the running kernel inside it, then installs and loads the result. The image carries the toolchain and no source, so a new module version ships as a new miren rather than a new image. Kernel headers come from the host when it has them, bind-mounted at their real paths so the build tree resolves its own absolute symlinks. When it does not, the builder installs them for itself, which works on Debian and Ubuntu because that is what the image is built from. Anywhere else the operator gets told which package to install. A module only loads on the kernel it was built for, so a kernel upgrade would otherwise strand a node back on loop devices. The server notices at startup and rebuilds, but only on hosts that installed the module in the first place, and under a timeout: accelerator mode is an optimisation, and a wedged build must not hold up boot. --- .github/workflows/release.yml | 68 +++ cli/commands/commands.go | 31 +- cli/commands/disk_accelerator.go | 203 ++++++++ cli/commands/disk_accelerator_doc.go | 44 ++ cli/commands/disk_accelerator_other.go | 30 ++ components/runner/lbd.go | 75 +++ docker/Dockerfile.lbd-builder | 58 +++ docker/lbd-builder/build.sh | 189 ++++++++ docs/command-sidebar.json | 4 + docs/docs/command/disk-accelerator-install.md | 46 ++ docs/docs/command/disk-accelerator-status.md | 39 ++ .../command/disk-accelerator-uninstall.md | 37 ++ docs/docs/command/disk-accelerator.md | 68 +++ docs/docs/disk-accelerator.md | 134 ++++++ docs/docs/disks.md | 2 + docs/docs/system-requirements.md | 4 +- docs/sidebars.ts | 1 + pkg/imagerefs/imagerefs.go | 6 + pkg/lbdmod/build.go | 450 ++++++++++++++++++ pkg/lbdmod/build_test.go | 429 +++++++++++++++++ pkg/lbdmod/builder.go | 69 +++ pkg/lbdmod/ctrbuild/ctrbuild.go | 268 +++++++++++ pkg/lbdmod/ctrbuild/ctrbuild_test.go | 76 +++ pkg/lbdmod/kernel.go | 240 ++++++++++ pkg/lbdmod/kernel_test.go | 156 ++++++ pkg/lbdmod/lock.go | 67 +++ pkg/lbdmod/lock_test.go | 75 +++ pkg/lbdmod/probe.go | 137 ++++++ pkg/lbdmod/source.go | 50 ++ pkg/lbdmod/source_test.go | 64 +++ pkg/lbdmod/status.go | 213 +++++++++ pkg/lbdmod/status_test.go | 264 ++++++++++ 32 files changed, 3595 insertions(+), 2 deletions(-) create mode 100644 cli/commands/disk_accelerator.go create mode 100644 cli/commands/disk_accelerator_doc.go create mode 100644 cli/commands/disk_accelerator_other.go create mode 100644 components/runner/lbd.go create mode 100644 docker/Dockerfile.lbd-builder create mode 100755 docker/lbd-builder/build.sh create mode 100644 docs/docs/command/disk-accelerator-install.md create mode 100644 docs/docs/command/disk-accelerator-status.md create mode 100644 docs/docs/command/disk-accelerator-uninstall.md create mode 100644 docs/docs/command/disk-accelerator.md create mode 100644 docs/docs/disk-accelerator.md create mode 100644 pkg/lbdmod/build.go create mode 100644 pkg/lbdmod/build_test.go create mode 100644 pkg/lbdmod/builder.go create mode 100644 pkg/lbdmod/ctrbuild/ctrbuild.go create mode 100644 pkg/lbdmod/ctrbuild/ctrbuild_test.go create mode 100644 pkg/lbdmod/kernel.go create mode 100644 pkg/lbdmod/kernel_test.go create mode 100644 pkg/lbdmod/lock.go create mode 100644 pkg/lbdmod/lock_test.go create mode 100644 pkg/lbdmod/probe.go create mode 100644 pkg/lbdmod/source.go create mode 100644 pkg/lbdmod/source_test.go create mode 100644 pkg/lbdmod/status.go create mode 100644 pkg/lbdmod/status_test.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 326f1efdb..46b9cae02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -469,6 +469,74 @@ jobs: # this path makes worse. cache-to: ${{ needs.init.outputs.cache == 'true' && 'type=gha,mode=max' || '' }} + build-and-push-lbd-builder: + # The toolchain image that compiles the lbd kernel module on a node. It + # carries no module source, so its tag only moves when the toolchain does: + # this publishes the tag pkg/imagerefs pins, and does nothing if that tag + # already exists. Bumping imagerefs.LbdBuilder is what ships a new one. + needs: [init, test] + if: >- + !cancelled() && + (needs.test.result == 'success' || needs.test.result == 'skipped') + runs-on: depot-ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ needs.init.outputs.sha }} + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + workload_identity_provider: ${{ secrets.WIF_PROVIDER }} + service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} + + - name: Configure Docker for Artifact Registry + run: gcloud auth configure-docker us-central1-docker.pkg.dev + + - name: Decide whether the builder image needs publishing + id: lbd-builder + run: | + set -euo pipefail + # The pinned reference is the single source of truth for the tag. + REF=$(grep -oE 'oci\.miren\.cloud/lbd-builder:[A-Za-z0-9._-]+' pkg/imagerefs/imagerefs.go | head -1) + if [ -z "$REF" ]; then + echo "could not find the lbd-builder reference in pkg/imagerefs/imagerefs.go" >&2 + exit 1 + fi + TAG="${REF##*:}" + IMAGE="us-central1-docker.pkg.dev/miren-cloud/miren-oci/lbd-builder:$TAG" + echo "image=$IMAGE" >> $GITHUB_OUTPUT + + # Republishing an existing tag would move a pinned reference under + # already-deployed clusters, so only a new tag is ever pushed. + if gcloud artifacts docker images describe "$IMAGE" >/dev/null 2>&1; then + echo "$IMAGE already exists; nothing to publish" + echo "publish=false" >> $GITHUB_OUTPUT + else + echo "publish=true" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + if: steps.lbd-builder.outputs.publish == 'true' + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Build and push the lbd builder image + if: steps.lbd-builder.outputs.publish == 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./docker/Dockerfile.lbd-builder + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.lbd-builder.outputs.image }} + cache-from: type=gha + cache-to: ${{ needs.init.outputs.cache == 'true' && 'type=gha,mode=max' || '' }} + upload-to-miren: needs: [init, test, package, build-binaries] # Same reasoning as build-and-push-docker: tolerate `test` being skipped on diff --git a/cli/commands/commands.go b/cli/commands/commands.go index 4102cb8bf..c4b4c55b7 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1402,12 +1402,41 @@ Warning: These commands are intended for advanced users and developers. They may d.Dispatch("debug entity ensure", Infer("debug entity ensure", "Ensure an entity exists", EntityEnsure)) // Disk commands - d.Dispatch("disk", Section("disk", "Disk backup and recovery", "", WithSectionGroup(GroupServer))) + d.Dispatch("disk", Section("disk", "Disk backup, recovery, and acceleration", "", WithSectionGroup(GroupServer))) d.Dispatch("disk backup", Infer("disk backup", "Backup a disk to a snapshot file", DiskBackup)) d.Dispatch("disk restore", Infer("disk restore", "Restore a disk from a snapshot file", DiskRestore)) d.Dispatch("disk undelete", Infer("disk undelete", "Restore a recently deleted disk", DiskUndelete)) d.Dispatch("disk list-deleted", Infer("disk list-deleted", "List deleted disks available for recovery", DiskListDeleted)) + // Accelerator mode. These build and load the lbd kernel module, so they + // only do anything on Linux; the non-Linux builds register stubs that say + // so rather than leaving the command missing. + d.Dispatch("disk accelerator", Section("disk accelerator", "Faster block-device disks via the lbd kernel module", "", + WithSectionGroup(GroupServer), + WithSectionDescription(acceleratorSectionDescription))) + d.Dispatch("disk accelerator status", Infer("disk accelerator status", "Show whether accelerator mode can run on this host", DiskAcceleratorStatus, + WithExample(mflags.Example{ + Name: "Check accelerator mode", + Body: "miren disk accelerator status", + }), + )) + d.Dispatch("disk accelerator install", Infer("disk accelerator install", "Build and load the lbd kernel module for this kernel", DiskAcceleratorInstall, + WithExample(mflags.Example{ + Name: "Enable accelerator mode", + Body: "sudo miren disk accelerator install", + }), + WithExample(mflags.Example{ + Name: "Rebuild after a kernel upgrade", + Body: "sudo miren disk accelerator install --force", + }), + )) + d.Dispatch("disk accelerator uninstall", Infer("disk accelerator uninstall", "Unload and remove the lbd kernel module", DiskAcceleratorUninstall, + WithExample(mflags.Example{ + Name: "Go back to loop devices", + Body: "sudo miren disk accelerator uninstall", + }), + )) + // Debug disk commands d.Dispatch("debug disk", Section("debug disk", "Disk entity debug commands", "", WithSectionDescription(diskSectionDescription))) d.Dispatch("debug disk create", Infer("debug disk create", "Create a disk entity for testing", DebugDiskCreate, diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go new file mode 100644 index 000000000..461f57492 --- /dev/null +++ b/cli/commands/disk_accelerator.go @@ -0,0 +1,203 @@ +//go:build linux + +package commands + +import ( + "context" + "fmt" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/lbdmod/ctrbuild" +) + +// acceleratorOptions builds the lookup paths for lbd on this host. The release +// directory is included because it is prepended to containerd's PATH, not +// miren's, so a bundled lbdctl would otherwise be invisible here. +func acceleratorOptions(dataPath string) lbdmod.Options { + return lbdmod.Options{ + DataPath: dataPath, + SearchPath: []string{FindReleasePath()}, + } +} + +// DiskAcceleratorStatus reports whether accelerator mode can run on this host. +// It only reads, so it does not need root. +func DiskAcceleratorStatus(ctx *Context, opts struct { + FormatOptions + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + status, err := lbdmod.Probe(acceleratorOptions(opts.DataPath)) + if err != nil { + return err + } + + if opts.IsJSON() { + return PrintJSON(newAcceleratorStatusJSON(status)) + } + + rows := [][]string{ + {"Available", yesNo(status.Available())}, + {"State", status.Explain()}, + {"Kernel", status.Host.KernelRelease}, + {"Module loaded", yesNo(status.Loaded)}, + {"Control device", yesNo(status.ControlDevicePresent)}, + {"Module installed", yesNo(status.ModuleInstalled)}, + {"lbdctl", orNone(status.LbdctlPath)}, + {"Kernel headers", orNone(status.Host.HeadersDir)}, + {"Bundled lbd version", status.EmbeddedVersion}, + } + if status.Marker != nil { + rows = append(rows, + []string{"Installed version", status.Marker.LbdVersion}, + []string{"Built for kernel", status.Marker.KernelRelease}, + []string{"Built at", status.Marker.BuiltAt.Local().Format(time.RFC3339)}, + ) + } + ctx.DisplayTable([]string{"", ""}, rows) + + switch { + case status.Available() && !status.Stale(): + return nil + case status.Stale(): + ctx.Warn("The installed module no longer matches this host. Run: sudo miren disk accelerator install") + case status.Host.HeadersDir == "" && status.Host.CanFetchHeaders(): + ctx.Info("This host has no kernel headers; the builder will fetch them. Run: sudo miren disk accelerator install") + case status.Host.HeadersDir == "": + ctx.Warn("This host has no kernel headers, which the build needs. %s", status.Host.InstallHint()) + default: + ctx.Info("To enable accelerator mode, run: sudo miren disk accelerator install") + } + return nil +} + +// DiskAcceleratorInstall compiles the lbd kernel module against the running +// kernel and loads it, so disks can use accelerator mode instead of falling +// back to loop devices. +func DiskAcceleratorInstall(ctx *Context, opts struct { + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Image string `long:"image" description:"Override the builder image"` + Socket string `long:"socket" description:"Path to the containerd socket"` + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + cc, err := dialContainerd(ctx, opts.Socket) + if err != nil { + return err + } + defer cc.Close() + + installer := &lbdmod.Installer{ + Log: ctx.Log, + Builder: ctrbuild.New(cc, ctx.Log), + Options: acceleratorOptions(opts.DataPath), + Image: opts.Image, + } + + ctx.Begin("Installing the lbd kernel module") + status, err := installer.Install(ctx, opts.Force) + if err != nil { + return err + } + + ctx.Completed("Accelerator mode is ready on kernel %s", status.Host.KernelRelease) + ctx.Info("Restart the miren server to pick it up: sudo systemctl restart miren") + return nil +} + +// DiskAcceleratorUninstall unloads the module and removes what the install put +// on the host, including the record that would otherwise rebuild it after a +// kernel upgrade. +func DiskAcceleratorUninstall(ctx *Context, opts struct { + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + installer := &lbdmod.Installer{ + Log: ctx.Log, + Options: acceleratorOptions(opts.DataPath), + } + + ctx.Begin("Removing the lbd kernel module") + if err := installer.Uninstall(ctx); err != nil { + return err + } + + ctx.Completed("Accelerator mode removed; disks will use loop devices") + return nil +} + +// dialContainerd connects to the containerd miren runs its own containers on, +// preferring miren's socket over the system one. +func dialContainerd(ctx context.Context, socket string) (*containerd.Client, error) { + if socket == "" { + socket = defaultContainerdSocket() + } + + cc, err := containerd.New(socket) + if err != nil { + return nil, fmt.Errorf("could not reach containerd at %s, which the builder needs: %w", socket, err) + } + + // containerd.New does not connect, so without this a dead socket would + // surface much later as an opaque failure to pull the builder image. + serving, err := cc.IsServing(ctx) + if err != nil || !serving { + cc.Close() + return nil, fmt.Errorf("containerd at %s is not responding, which the builder needs: %w", socket, err) + } + + return cc, nil +} + +// acceleratorStatusJSON is the machine-readable shape of the status command. +type acceleratorStatusJSON struct { + Available bool `json:"available"` + State string `json:"state"` + Kernel string `json:"kernel"` + ModuleLoaded bool `json:"module_loaded"` + ControlDevicePresent bool `json:"control_device_present"` + ModuleInstalled bool `json:"module_installed"` + Stale bool `json:"stale"` + LbdctlPath string `json:"lbdctl_path"` + KernelHeaders string `json:"kernel_headers"` + HeaderPackage string `json:"header_package"` + BundledVersion string `json:"bundled_version"` + InstalledVersion string `json:"installed_version,omitempty"` + BuiltForKernel string `json:"built_for_kernel,omitempty"` + BuiltAt string `json:"built_at,omitempty"` +} + +func newAcceleratorStatusJSON(s lbdmod.Status) acceleratorStatusJSON { + out := acceleratorStatusJSON{ + Available: s.Available(), + State: s.Explain(), + Kernel: s.Host.KernelRelease, + ModuleLoaded: s.Loaded, + ControlDevicePresent: s.ControlDevicePresent, + ModuleInstalled: s.ModuleInstalled, + Stale: s.Stale(), + LbdctlPath: s.LbdctlPath, + KernelHeaders: s.Host.HeadersDir, + HeaderPackage: s.Host.HeaderPackage(), + BundledVersion: s.EmbeddedVersion, + } + if s.Marker != nil { + out.InstalledVersion = s.Marker.LbdVersion + out.BuiltForKernel = s.Marker.KernelRelease + out.BuiltAt = s.Marker.BuiltAt.UTC().Format(time.RFC3339) + } + return out +} + +func yesNo(b bool) string { + if b { + return "yes" + } + return "no" +} + +func orNone(s string) string { + if s == "" { + return "not found" + } + return s +} diff --git a/cli/commands/disk_accelerator_doc.go b/cli/commands/disk_accelerator_doc.go new file mode 100644 index 000000000..a05fb990d --- /dev/null +++ b/cli/commands/disk_accelerator_doc.go @@ -0,0 +1,44 @@ +package commands + +const acceleratorSectionDescription = `Miren serves block-device disks in one of two modes. + +**Universal mode** is the default and works everywhere. It backs each disk with a +loop device, which the Linux kernel provides out of the box. + +**Accelerator mode** uses ` + "`lbd`" + `, a Miren kernel module that puts a +write-ahead log in front of the disk. It is faster, and it is what continuous +backup to Miren Cloud is built on. + +` + "`lbd`" + ` is not part of the Linux kernel, so it has to be compiled for the +exact kernel your node is running. ` + "`miren disk accelerator install`" + ` does +that for you: it downloads a builder image, compiles the module against your +kernel's headers inside a container, then installs and loads the result. Nothing +but Miren has to be installed on the host beyond the kernel headers themselves. + +## Getting started + +` + "```" + `bash +miren disk accelerator status # can this host run it? +sudo miren disk accelerator install # build and load the module +sudo systemctl restart miren # pick up the new mode +` + "```" + ` + +## Requirements + +- The kernel headers for your running kernel. On Debian and Ubuntu the builder + fetches them itself if the host has none. Everywhere else you install them + first, and ` + "`status`" + ` names the package -- ` + "`kernel-devel-$(uname -r)`" + ` + on Fedora and RHEL. +- Secure Boot disabled. A self-built module is unsigned, and firmware with Secure + Boot enforcing will refuse to load it. +- A kernel built with GCC. Clang-built kernels are not supported. + +## After a kernel upgrade + +A module only loads on the kernel it was built for. Once a host has installed the +module, Miren notices on startup that the running kernel has changed and rebuilds +it. You can also do it by hand with +` + "`sudo miren disk accelerator install --force`" + `. + +Until the module is back, disks fall back to universal mode. Nothing breaks; they +are just slower.` diff --git a/cli/commands/disk_accelerator_other.go b/cli/commands/disk_accelerator_other.go new file mode 100644 index 000000000..ee5c38930 --- /dev/null +++ b/cli/commands/disk_accelerator_other.go @@ -0,0 +1,30 @@ +//go:build !linux + +package commands + +import "fmt" + +// DiskAcceleratorStatus is not supported on non-Linux platforms +func DiskAcceleratorStatus(ctx *Context, opts struct { + FormatOptions + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + return fmt.Errorf("disk accelerator status is only available on Linux") +} + +// DiskAcceleratorInstall is not supported on non-Linux platforms +func DiskAcceleratorInstall(ctx *Context, opts struct { + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Image string `long:"image" description:"Override the builder image"` + Socket string `long:"socket" description:"Path to the containerd socket"` + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + return fmt.Errorf("disk accelerator install is only available on Linux") +} + +// DiskAcceleratorUninstall is not supported on non-Linux platforms +func DiskAcceleratorUninstall(ctx *Context, opts struct { + DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` +}) error { + return fmt.Errorf("disk accelerator uninstall is only available on Linux") +} diff --git a/components/runner/lbd.go b/components/runner/lbd.go new file mode 100644 index 000000000..cf1db68c6 --- /dev/null +++ b/components/runner/lbd.go @@ -0,0 +1,75 @@ +package runner + +import ( + "context" + "errors" + "log/slog" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/lbdmod/ctrbuild" +) + +// rebuildTimeout bounds the unattended rebuild at startup. +// +// A rebuild pulls the builder image, may fetch kernel headers, and compiles; +// on a real node that is tens of seconds, and this leaves generous room for a +// slow link. What it must not do is wait forever: without a bound, a builder +// that wedges -- a stalled fetch, a hung pull -- holds up the runner's whole +// startup with no way out, which is a far worse outcome than the slower disks +// we get by giving up. +const rebuildTimeout = 10 * time.Minute + +// setupLbd brings accelerator mode up, rebuilding the lbd kernel module if a +// kernel upgrade left the installed one unusable. +// +// A module only loads on the kernel it was built for, so an operator who +// enabled accelerator mode and then took a kernel update would otherwise find +// their disks silently back on loop devices. Rebuilding is limited to hosts +// that already installed the module: a host that never opted in should not pay +// for an unattended compile at startup. +// +// This blocks rather than running in the background because the disk +// controller picks universal or accelerator mode once, at startup, from +// whether lbd is usable. Deciding that before the module is ready would pin +// the node to loop devices until the next restart. +// +// Neither failing nor timing out is fatal. Universal mode works everywhere, so +// the worst case is slower disks, not a runner that will not start. +func setupLbd(ctx context.Context, cc *containerd.Client, log *slog.Logger) { + if err := diskio.EnsureLbdDevices(log); err == nil { + return + } + + installer := &lbdmod.Installer{ + Log: log, + Builder: ctrbuild.New(cc, log), + } + + ctx, cancel := context.WithTimeout(ctx, rebuildTimeout) + defer cancel() + + rebuilt, err := installer.EnsureCurrent(ctx) + if errors.Is(err, context.DeadlineExceeded) { + log.Warn("gave up rebuilding the lbd kernel module, disks will use loop devices", + "timeout", rebuildTimeout, + "retry_with", "miren disk accelerator install") + return + } + if err != nil { + log.Warn("could not rebuild the lbd kernel module, disks will use loop devices", "error", err) + return + } + if !rebuilt { + // Nothing to rebuild: this host never installed the module. + log.Info("accelerator mode is not enabled on this host, disks will use loop devices", + "enable_with", "miren disk accelerator install") + return + } + + if err := diskio.EnsureLbdDevices(log); err != nil { + log.Warn("rebuilt the lbd kernel module but it is still not usable", "error", err) + } +} diff --git a/docker/Dockerfile.lbd-builder b/docker/Dockerfile.lbd-builder new file mode 100644 index 000000000..67ce29763 --- /dev/null +++ b/docker/Dockerfile.lbd-builder @@ -0,0 +1,58 @@ +# Toolchain for compiling the lbd kernel module against a node's running kernel. +# +# This image deliberately does NOT contain the module source. miren embeds the +# source and bind-mounts it at /src, so shipping a new module version means +# shipping a new miren, not republishing this image. +# +# The contract with pkg/lbdmod: +# +# /src read-write module source; object files are written here +# /out read-write where lbd.ko and lbdctl are left +# /lib/modules read-only the host's module tree, mounted at its real +# /usr/src read-only path so the kernel build tree's absolute +# symlinks still resolve +# +# KERNEL_RELEASE the kernel to build for; defaults to uname -r, which is +# already the host's kernel since containers share it +# KERNEL_HEADERS the build tree to compile against; probed when unset +# HOST_DISTRO_ID the host's os-release ID, for error messages +# HOST_DISTRO_LIKE the host's os-release ID_LIKE + +FROM ubuntu:24.04 + +# bc, flex, bison, libelf-dev and libssl-dev are what the kernel build system +# reaches for; kmod gives us modinfo to verify what we produced. +# +# One gcc, not several. A module's ABI contract is its vermagic and symbol +# CRCs, both of which come from the kernel headers rather than the compiler -- +# DKMS has always built out-of-tree modules with whatever gcc the host happens +# to have. Carrying gcc-12 and gcc-14 as well cost ~300MB to silence a warning. +# build.sh still reports a major-version mismatch, and the Clang case, which is +# the one that genuinely fails to load, is refused outright. +# +# Deliberately not build-essential: that drags in g++ and libstdc++-dev (~50MB) +# to compile C++ we never write. libc6-dev and make are what it was wanted for. +# +# The dpkg excludes also apply to the kernel headers the builder installs for +# itself on the fetch path, so they keep paying off after image build. +RUN printf 'path-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-exclude=/usr/share/info/*\npath-exclude=/usr/share/lintian/*\npath-exclude=/usr/share/locale/*\npath-include=/usr/share/locale/en*\n' \ + > /etc/dpkg/dpkg.cfg.d/01-nodoc \ + && apt-get update && apt-get install -y --no-install-recommends \ + bc \ + binutils \ + bison \ + ca-certificates \ + flex \ + gcc-13 \ + kmod \ + libc6-dev \ + libelf-dev \ + libssl-dev \ + make \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/* /usr/share/doc /usr/share/man + +COPY docker/lbd-builder/build.sh /usr/local/bin/build-lbd +RUN chmod +x /usr/local/bin/build-lbd + +ENTRYPOINT ["/usr/local/bin/build-lbd"] diff --git a/docker/lbd-builder/build.sh b/docker/lbd-builder/build.sh new file mode 100755 index 000000000..856b1404a --- /dev/null +++ b/docker/lbd-builder/build.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# Compile the lbd kernel module and lbdctl against the running kernel. +# +# Runs inside the lbd-builder image; see docker/Dockerfile.lbd-builder for the +# mounts and environment it expects. Everything it prints is shown to the +# operator, so messages here are the ones they will act on. + +set -euo pipefail + +SRC=${SRC:-/src} +OUT=${OUT:-/out} + +fail() { + echo "error: $*" >&2 + exit 1 +} + +[ -d "$SRC" ] || fail "no module source at $SRC" +[ -f "$SRC/Makefile" ] || fail "$SRC has no Makefile; the source mount looks wrong" +mkdir -p "$OUT" + +# A container shares the host's kernel, so uname and /proc/version already +# describe the machine we are building for. +KERNEL_RELEASE=${KERNEL_RELEASE:-$(uname -r)} +[ -n "$KERNEL_RELEASE" ] || fail "could not determine the kernel release" + +# --------------------------------------------------------------------------- +# Kernel headers +# --------------------------------------------------------------------------- + +find_headers() { + local candidate + for candidate in \ + "/lib/modules/$KERNEL_RELEASE/build" \ + "/usr/src/kernels/$KERNEL_RELEASE" \ + "/usr/src/linux-headers-$KERNEL_RELEASE"; do + if [ -f "$candidate/Makefile" ]; then + echo "$candidate" + return 0 + fi + done + return 1 +} + +header_package() { + local id + for id in "${HOST_DISTRO_ID:-}" ${HOST_DISTRO_LIKE:-}; do + case "$id" in + debian | ubuntu) echo "linux-headers-$KERNEL_RELEASE"; return 0 ;; + fedora | rhel | centos) echo "kernel-devel-$KERNEL_RELEASE"; return 0 ;; + arch | alpine) echo "linux-headers"; return 0 ;; + suse | opensuse* | sles) echo "kernel-devel"; return 0 ;; + esac + done + return 1 +} + +KDIR=${KERNEL_HEADERS:-} +if [ -n "$KDIR" ] && [ ! -f "$KDIR/Makefile" ]; then + echo "warning: $KDIR is not a kernel build tree; looking elsewhere" >&2 + KDIR="" +fi +if [ -z "$KDIR" ]; then + KDIR=$(find_headers || true) +fi +# No build tree came in from the host. miren leaves /lib/modules and /usr/src +# unmounted in that case, so the builder can install headers into its own +# filesystem instead. This only reaches a Debian-family archive, because that is +# what this image is built from. +if [ -z "$KDIR" ] && [ "${FETCH_HEADERS:-0}" = "1" ]; then + pkg="linux-headers-$KERNEL_RELEASE" + echo "No kernel headers on the host; fetching $pkg" + + if ! apt-get update -qq; then + fail "could not reach the package archive to fetch $pkg. Install it on the host and try again." + fi + + if ! DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$pkg"; then + fail "$pkg is not available from the package archive, which usually means this kernel is too new or too old for it. Install the headers on the host and try again." + fi + + KDIR=$(find_headers || true) + if [ -z "$KDIR" ]; then + fail "$pkg installed but left no build tree for $KERNEL_RELEASE" + fi + echo "Fetched kernel headers into $KDIR" +fi + +if [ -z "$KDIR" ]; then + pkg=$(header_package || true) + if [ -n "$pkg" ]; then + fail "no kernel headers for $KERNEL_RELEASE. Install $pkg on the host and try again." + fi + fail "no kernel headers for $KERNEL_RELEASE. Install this kernel's headers on the host and try again." +fi + +echo "Building lbd for kernel $KERNEL_RELEASE against $KDIR" + +# --------------------------------------------------------------------------- +# Compiler +# +# A module should be built by roughly the compiler that built the kernel. A +# major-version gap is usually only a modpost warning, but a Clang-built kernel +# with control-flow integrity rejects a GCC-built module outright -- better to +# say so than to hand back a module that silently will not load. +# --------------------------------------------------------------------------- + +kernel_compiler_line=$(cat /proc/version 2>/dev/null || echo "") + +if echo "$kernel_compiler_line" | grep -qi 'clang version'; then + fail "this kernel was built with Clang, which the lbd builder does not support. Install the module from your distribution instead, or build it on the host." +fi + +want_major=$(echo "$kernel_compiler_line" | grep -oE '\bgcc[^0-9]*([0-9]+)' | grep -oE '[0-9]+' | head -1 || true) + +pick_gcc() { + local want=$1 candidate best="" + if [ -n "$want" ] && command -v "gcc-$want" >/dev/null 2>&1; then + echo "gcc-$want" + return 0 + fi + # No exact match: take the newest installed major and warn. Ordering here + # is oldest to newest so the last hit wins. + for candidate in 12 13 14 15 16; do + command -v "gcc-$candidate" >/dev/null 2>&1 && best="gcc-$candidate" + done + if [ -n "$best" ]; then + echo "$best" + return 0 + fi + command -v gcc >/dev/null 2>&1 && echo gcc && return 0 + return 1 +} + +CC=$(pick_gcc "$want_major") || fail "no C compiler in the builder image" + +if [ -n "$want_major" ] && [ "$CC" != "gcc-$want_major" ]; then + echo "warning: this kernel was built with gcc-$want_major but the builder only has $CC." >&2 + echo "warning: the module should still load, but report it if modprobe refuses it." >&2 +elif [ -z "$want_major" ]; then + echo "warning: could not tell which compiler built this kernel; using $CC" >&2 +fi + +echo "Compiling with $CC ($($CC -dumpversion))" + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +# The module's Makefile passes KBUILD_MODPOST_WARN=1, which downgrades +# unresolved symbols to warnings. That means a zero exit code alone does not +# prove the module will load, so the log is checked below. +log=$(mktemp) +if ! make -C "$SRC" KDIR="$KDIR" CC="$CC" lbd.ko 2>&1 | tee "$log"; then + fail "compiling lbd.ko failed" +fi + +if grep -q 'undefined!' "$log"; then + grep 'undefined!' "$log" >&2 + fail "lbd.ko references symbols this kernel does not export, so it would fail to load" +fi + +# lbdctl is linked statically: it runs on the host, whose libc is not the +# builder image's. +if ! make -C "$SRC" CC="$CC -static" lbdctl; then + fail "compiling lbdctl failed" +fi + +# --------------------------------------------------------------------------- +# Verify and hand back +# --------------------------------------------------------------------------- + +[ -f "$SRC/lbd.ko" ] || fail "the build reported success but produced no lbd.ko" +[ -f "$SRC/lbdctl" ] || fail "the build reported success but produced no lbdctl" + +# vermagic is what the kernel checks at load time. If it disagrees with the +# running kernel, modprobe will refuse the module, so catch it here where we +# can explain why. +vermagic=$(modinfo -F vermagic "$SRC/lbd.ko" 2>/dev/null || echo "") +case "$vermagic" in +"$KERNEL_RELEASE"*) ;; +"") echo "warning: could not read vermagic from lbd.ko" >&2 ;; +*) fail "lbd.ko was built for '$vermagic' but this host runs $KERNEL_RELEASE; the headers at $KDIR do not match the running kernel" ;; +esac + +install -m 0644 "$SRC/lbd.ko" "$OUT/lbd.ko" +install -m 0755 "$SRC/lbdctl" "$OUT/lbdctl" + +echo "Built lbd.ko ($(stat -c %s "$OUT/lbd.ko") bytes) and lbdctl for $KERNEL_RELEASE" diff --git a/docs/command-sidebar.json b/docs/command-sidebar.json index 7619d394d..7897e9476 100644 --- a/docs/command-sidebar.json +++ b/docs/command-sidebar.json @@ -188,6 +188,10 @@ "id": "command/disk" }, "items": [ + "command/disk-accelerator", + "command/disk-accelerator-install", + "command/disk-accelerator-status", + "command/disk-accelerator-uninstall", "command/disk-backup", "command/disk-list-deleted", "command/disk-restore", diff --git a/docs/docs/command/disk-accelerator-install.md b/docs/docs/command/disk-accelerator-install.md new file mode 100644 index 000000000..29fe82eb0 --- /dev/null +++ b/docs/docs/command/disk-accelerator-install.md @@ -0,0 +1,46 @@ +--- +title: "miren disk accelerator install" +sidebar_label: "disk accelerator install" +description: "Build and load the lbd kernel module for this kernel" +--- + +# miren disk accelerator install + +Build and load the lbd kernel module for this kernel + +## Usage + +```bash +miren disk accelerator install [flags] +``` + +## Flags + +- `--data-path` — Path to miren data (default: `/var/lib/miren`) +- `--force, -f` — Rebuild even when the module is already current +- `--image` — Override the builder image +- `--socket` — Path to the containerd socket + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Enable accelerator mode:** + +```bash +sudo miren disk accelerator install +``` + +**Rebuild after a kernel upgrade:** + +```bash +sudo miren disk accelerator install --force +``` + +## See also + +- [`miren disk accelerator`](/command/disk-accelerator) diff --git a/docs/docs/command/disk-accelerator-status.md b/docs/docs/command/disk-accelerator-status.md new file mode 100644 index 000000000..8ae8b3950 --- /dev/null +++ b/docs/docs/command/disk-accelerator-status.md @@ -0,0 +1,39 @@ +--- +title: "miren disk accelerator status" +sidebar_label: "disk accelerator status" +description: "Show whether accelerator mode can run on this host" +--- + +# miren disk accelerator status + +Show whether accelerator mode can run on this host + +## Usage + +```bash +miren disk accelerator status [flags] +``` + +## Flags + +- `--data-path` — Path to miren data (default: `/var/lib/miren`) +- `--format` — Output format (text, json) (default: `text`) +- `--json` — Shorthand for --format json + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Check accelerator mode:** + +```bash +miren disk accelerator status +``` + +## See also + +- [`miren disk accelerator`](/command/disk-accelerator) diff --git a/docs/docs/command/disk-accelerator-uninstall.md b/docs/docs/command/disk-accelerator-uninstall.md new file mode 100644 index 000000000..778a9911d --- /dev/null +++ b/docs/docs/command/disk-accelerator-uninstall.md @@ -0,0 +1,37 @@ +--- +title: "miren disk accelerator uninstall" +sidebar_label: "disk accelerator uninstall" +description: "Unload and remove the lbd kernel module" +--- + +# miren disk accelerator uninstall + +Unload and remove the lbd kernel module + +## Usage + +```bash +miren disk accelerator uninstall [flags] +``` + +## Flags + +- `--data-path` — Path to miren data (default: `/var/lib/miren`) + +## Global Options + +- `--options` — Path to file containing options +- `--server-address` — Server address to connect to (default: `127.0.0.1:8443`) +- `--verbose, -v` — Enable verbose output + +## Examples + +**Go back to loop devices:** + +```bash +sudo miren disk accelerator uninstall +``` + +## See also + +- [`miren disk accelerator`](/command/disk-accelerator) diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md new file mode 100644 index 000000000..36ff6b397 --- /dev/null +++ b/docs/docs/command/disk-accelerator.md @@ -0,0 +1,68 @@ +--- +title: "miren disk accelerator" +sidebar_label: "disk accelerator" +description: "Faster block-device disks via the lbd kernel module" +--- + +# miren disk accelerator + +Faster block-device disks via the lbd kernel module + +Miren serves block-device disks in one of two modes. + +**Universal mode** is the default and works everywhere. It backs each disk with a +loop device, which the Linux kernel provides out of the box. + +**Accelerator mode** uses `lbd`, a Miren kernel module that puts a +write-ahead log in front of the disk. It is faster, and it is what continuous +backup to Miren Cloud is built on. + +`lbd` is not part of the Linux kernel, so it has to be compiled for the +exact kernel your node is running. `miren disk accelerator install` does +that for you: it downloads a builder image, compiles the module against your +kernel's headers inside a container, then installs and loads the result. Nothing +but Miren has to be installed on the host beyond the kernel headers themselves. + +## Getting started + +```bash +miren disk accelerator status # can this host run it? +sudo miren disk accelerator install # build and load the module +sudo systemctl restart miren # pick up the new mode +``` + +## Requirements + +- The kernel headers for your running kernel. On Debian and Ubuntu the builder + fetches them itself if the host has none. Everywhere else you install them + first, and `status` names the package -- `kernel-devel-$(uname -r)` + on Fedora and RHEL. +- Secure Boot disabled. A self-built module is unsigned, and firmware with Secure + Boot enforcing will refuse to load it. +- A kernel built with GCC. Clang-built kernels are not supported. + +## After a kernel upgrade + +A module only loads on the kernel it was built for. Once a host has installed the +module, Miren notices on startup that the running kernel has changed and rebuilds +it. You can also do it by hand with +`sudo miren disk accelerator install --force`. + +Until the module is back, disks fall back to universal mode. Nothing breaks; they +are just slower. + +## Usage + +```bash +miren disk accelerator [flags] +``` + +## Subcommands + +- [`miren disk accelerator install`](/command/disk-accelerator-install) — Build and load the lbd kernel module for this kernel +- [`miren disk accelerator status`](/command/disk-accelerator-status) — Show whether accelerator mode can run on this host +- [`miren disk accelerator uninstall`](/command/disk-accelerator-uninstall) — Unload and remove the lbd kernel module + +## See also + +- [`miren disk`](/command/disk) diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md new file mode 100644 index 000000000..13ad44dac --- /dev/null +++ b/docs/docs/disk-accelerator.md @@ -0,0 +1,134 @@ +--- +title: Disk Accelerator +description: Turn on accelerator mode for Miren Disks by building and loading the lbd kernel module against your node's running kernel. +keywords: [accelerator, lbd, kernel module, disk performance, disk mode, loop device] +--- + +# Disk Accelerator + +Miren serves [Miren Disks](/disks#miren-disks) in one of two modes, and picks +between them on its own. + +**Universal mode** is the default and works everywhere. It backs each disk with a +*loop device*, which is the kernel's built-in way of presenting a file as though +it were a disk. Nothing to install, no configuration. + +**Accelerator mode** uses `lbd`, a Miren kernel module that puts a write-ahead +log in front of the disk. Writes land in the log first and are folded into the +disk image behind them, which makes writes faster and gives Miren an exact, +ordered record of every change. That record is what continuous backup to Miren +Cloud is built on. + +`lbd` is not part of Linux, so it has to be compiled for the exact kernel your +node is running. One command does that. + +## Minimum working example + +```bash +miren disk accelerator status # can this host run it? +sudo miren disk accelerator install # build and load the module +sudo systemctl restart miren # pick up the new mode +``` + +`install` downloads a builder image, compiles the module against your kernel's +headers inside a container, then installs and loads the result. The toolchain +lives in the image, so on Debian and Ubuntu there is nothing to install first — +if the host has no kernel headers, the builder fetches them for itself. + +Once the server restarts, new disks use accelerator mode. Existing disks keep +whatever mode they were created with. + +## Requirements + +| Requirement | Why | If it is missing | +|---|---|---| +| Kernel headers for the running kernel | The module is compiled against them | On Debian and Ubuntu the builder fetches them; elsewhere `status` names the package | +| Secure Boot off | A self-built module is unsigned, and enforcing firmware refuses it | `install` stops and says so | +| A GCC-built kernel | The builder ships GCC, not Clang | `install` stops and says so | +| Root | Loading a kernel module needs it | Run under `sudo` | + +### About the headers + +The builder image is Debian-based, so on a Debian or Ubuntu host it can install +`linux-headers-$(uname -r)` for itself and you need nothing on the host. That +needs a route to the distribution's package archive from the node, and it can +still come up empty for a kernel too new or too old to be in the archive — the +build says so, naming the package it could not find. + +On any other distribution, install the headers yourself first. +`miren disk accelerator status` prints the exact package name; it is +`kernel-devel-$(uname -r)` on Fedora and RHEL. + +Installing the headers on the host is always the faster path, because the +builder then borrows them read-only instead of downloading them, and needs no +network at all. + +## Checking what is going on + +```bash +miren disk accelerator status +``` + +``` +Available yes +State lbd v0.0.0-20260824210626-be4cec661034 is loaded for kernel 6.8.0-51-generic +Kernel 6.8.0-51-generic +Module loaded yes +Control device yes +Module installed yes +lbdctl /usr/local/bin/lbdctl +Kernel headers /lib/modules/6.8.0-51-generic/build +Bundled lbd version v0.0.0-20260824210626-be4cec661034 +``` + +`--format json` gives the same thing as machine-readable fields. + +**Available** is the answer to the only question that matters: can this node +serve accelerator disks right now. It is true only when the module is loaded, +its control device exists, and `lbdctl` is installed to drive it. Any one of +those missing puts disks back on loop devices. + +## After a kernel upgrade + +A module only loads on the kernel it was built for, so a kernel upgrade leaves +the installed module unusable. + +Miren handles this. On startup it notices the running kernel no longer matches +the module it built, and rebuilds. You do not have to do anything, though you +can force it by hand: + +```bash +sudo miren disk accelerator install --force +``` + +This only happens on hosts that installed the module in the first place. A host +that never turned accelerator mode on never pays for an unattended compile at +startup. + +Until the module is back, disks fall back to universal mode. Nothing breaks; +they are just slower. + +## Choosing the mode yourself + +Auto-detection can be overridden in the server config: + +```toml title="/etc/miren/server.toml" +disk_mode = "universal" # or "accelerator", or "auto" (the default) +``` + +`universal` forces loop devices even where the module is loaded. `accelerator` +insists on `lbd`, and disks will fail to attach if it is not there — useful when +you would rather find out loudly than quietly run slower. See +[Server Configuration](/server-config). + +## Turning it off + +```bash +sudo miren disk accelerator uninstall +sudo systemctl restart miren +``` + +This unloads the module, removes it along with `lbdctl`, and forgets that the +host ever had it, so nothing rebuilds it later. It fails if a disk is still +attached — the kernel will not unload a module in use. Stop the apps holding +disks first. diff --git a/docs/docs/disks.md b/docs/docs/disks.md index 92000bf97..5a6d17aab 100644 --- a/docs/docs/disks.md +++ b/docs/docs/disks.md @@ -103,6 +103,8 @@ Miren Disks live on your server. Back up important data with `miren disk backup` Miren Disks provide managed persistent storage for your applications. Disks are provisioned with a specific size and filesystem, support exclusive leasing for data consistency, and persist across app restarts and redeployments. +They work out of the box on any Linux host. For faster writes, [accelerator mode](/disk-accelerator) builds and loads a Miren kernel module tuned for this job; it is one command, and disks fall back to the default cleanly if you skip it. + ### Why Use Disks? - **Managed lifecycle**: Miren handles disk creation, formatting, and attachment automatically diff --git a/docs/docs/system-requirements.md b/docs/docs/system-requirements.md index 6d6a8d84e..32c4f99c1 100644 --- a/docs/docs/system-requirements.md +++ b/docs/docs/system-requirements.md @@ -40,7 +40,9 @@ sudo dnf install iptables nftables `miren server install` and `miren runner install` verify these are present before installing, so a missing tool stops the install with instructions rather than surfacing later as a broken network. :::note[Extra tooling for optional features] -Some features reach for more commands, installed automatically or only when you opt in. [Block-device volumes](./managing-disk-space.md) use disk tooling (`lbdctl`, `mkfs.*`, `blkid`) when a disk is provisioned, and on SELinux-enforcing hosts the installer uses `semanage` and `restorecon` to label the binary. Both paths degrade gracefully if the tools are absent, so you only need them if you use the corresponding feature. +Some features reach for more commands, installed automatically or only when you opt in. [Block-device volumes](./managing-disk-space.md) use disk tooling (`mkfs.*`, `blkid`) when a disk is provisioned, and on SELinux-enforcing hosts the installer uses `semanage` and `restorecon` to label the binary. Both paths degrade gracefully if the tools are absent, so you only need them if you use the corresponding feature. + +[Accelerator mode](./disk-accelerator.md) compiles a kernel module for your running kernel, so it needs that kernel's headers. On Debian and Ubuntu the builder fetches them itself; elsewhere install them first (`kernel-devel-$(uname -r)` on Fedora and RHEL). Without them, disks use loop devices instead — slower, but working. ::: ## Why these numbers? diff --git a/docs/sidebars.ts b/docs/sidebars.ts index ac3278e3c..7e980cec3 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -36,6 +36,7 @@ const sidebars: SidebarsConfig = { collapsed: false, items: [ 'disks', + 'disk-accelerator', 'addons', 'managing-disk-space', ], diff --git a/pkg/imagerefs/imagerefs.go b/pkg/imagerefs/imagerefs.go index 0eabb3097..e7246563f 100644 --- a/pkg/imagerefs/imagerefs.go +++ b/pkg/imagerefs/imagerefs.go @@ -38,6 +38,12 @@ const ( // Miren runtime server Miren = "oci.miren.cloud/miren:latest" + + // LbdBuilder carries the toolchain that compiles the lbd kernel module + // against a node's running kernel (docker/Dockerfile.lbd-builder). It + // holds no module source -- miren embeds that and mounts it in -- so the + // tag only moves when the toolchain itself needs to. + LbdBuilder = "oci.miren.cloud/lbd-builder:v1" ) // Base images for language stacks diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go new file mode 100644 index 000000000..86950b42f --- /dev/null +++ b/pkg/lbdmod/build.go @@ -0,0 +1,450 @@ +package lbdmod + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "miren.dev/runtime/pkg/imagerefs" +) + +const ( + // buildContainerName is fixed so a build killed before its own cleanup + // leaves something the next run can find and remove. + buildContainerName = "miren-lbd-build" + + // lbdctlInstallDir is where lbdctl goes. It has to be somewhere on the + // server process's PATH: the release directory is prepended to + // containerd's PATH, not miren's. + lbdctlInstallDir = "/usr/local/bin" + + // modulesLoadConf makes the kernel load lbd at boot, so a reboot does not + // depend on miren starting first. + modulesLoadConf = "/etc/modules-load.d/lbd.conf" +) + +// Installer builds and installs the lbd kernel module on this host. +type Installer struct { + // Log receives progress and the builder container's output. + Log *slog.Logger + + // Builder runs the builder image. Required for Install; Uninstall does + // not need it. pkg/lbdmod/ctrbuild provides the containerd one. + Builder Builder + + // Options say where to read host state and keep the install record. + Options Options + + // Image overrides the builder image. Empty means imagerefs.LbdBuilder. + Image string +} + +// buildDir is the scratch directory a build works in. It is keyed by kernel +// and module version so a rebuild after a kernel upgrade cannot pick up stale +// object files from the previous kernel. +func (i *Installer) buildDir(release string) string { + key := fmt.Sprintf("%s-%s", SourceVersion(), release) + return filepath.Join(i.Options.dataPath(), "lbd", "build", key) +} + +func (i *Installer) image() string { + if i.Image != "" { + return i.Image + } + return imagerefs.LbdBuilder +} + +// Install compiles the module against the running kernel and loads it. It is +// safe to call when the module is already installed and current: that is +// reported as a no-op unless force is set. +// +// The caller must be root. +func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { + status, err := Probe(i.Options) + if err != nil { + return status, err + } + + if !force && status.Available() && !status.Stale() { + i.Log.Info("lbd is already installed and current", "kernel", status.Host.KernelRelease) + return status, nil + } + + if err := i.checkCanBuild(status); err != nil { + return status, err + } + + // Held for the whole build-and-load, so a concurrent install cannot clear + // the build directory or delete the builder container out from under this + // one. Taken after the cheap checks so an obviously impossible install + // still fails with the real reason rather than a lock error. + lock, err := acquireBuildLock(i.Options.dataPath()) + if err != nil { + return status, err + } + defer lock.release() + + // Another process may have finished the very build this one was about to + // start while we waited to be let in. + if !force { + if current, err := Probe(i.Options); err == nil && current.Available() && !current.Stale() { + i.Log.Info("another process installed lbd while this one waited", + "kernel", current.Host.KernelRelease) + return current, nil + } + } + + if err := i.build(ctx, status.Host); err != nil { + return status, err + } + + if err := i.load(ctx, status.Host); err != nil { + return status, err + } + + // Prove the module is usable before recording the install. Writing the + // marker first would leave a record claiming success behind a failure, so + // `status` would show an installed version and build time for something + // that never worked. Availability does not depend on the marker, so this + // check is meaningful without it. + verified, err := Probe(i.Options) + if err != nil { + return status, err + } + if !verified.Available() { + return verified, fmt.Errorf("lbd was built and loaded but is still not usable: %s", verified.Explain()) + } + + marker := Marker{ + LbdVersion: SourceVersion(), + KernelRelease: status.Host.KernelRelease, + ModulePath: modulePath(status.Host.KernelRelease), + LbdctlPath: filepath.Join(lbdctlInstallDir, "lbdctl"), + BuiltAt: time.Now().UTC(), + } + if err := writeMarker(i.Options.dataPath(), marker); err != nil { + return verified, err + } + + // Re-probe so the caller gets a status that includes the record just + // written, which is what `status` renders. + after, err := Probe(i.Options) + if err != nil { + return verified, err + } + + i.Log.Info("lbd installed", + "kernel", after.Host.KernelRelease, + "version", SourceVersion(), + "module", marker.ModulePath) + return after, nil +} + +// EnsureCurrent rebuilds the module when this host has installed it before but +// what is on disk no longer fits -- almost always because the kernel was +// upgraded, which leaves a module that cannot load. +// +// A host with no install record is left alone: it never opted into accelerator +// mode, so it should not pay for an unattended compile at startup. It reports +// whether it rebuilt. +func (i *Installer) EnsureCurrent(ctx context.Context) (bool, error) { + status, err := Probe(i.Options) + if err != nil { + return false, err + } + + if status.Available() && !status.Stale() { + return false, nil + } + + if status.Marker == nil { + return false, nil + } + + i.Log.Info("rebuilding the lbd kernel module", "reason", status.Explain()) + if _, err := i.Install(ctx, false); err != nil { + return false, err + } + return true, nil +} + +// checkCanBuild refuses the cases where a build would either fail confusingly +// or produce a module that cannot be loaded, and says why. +func (i *Installer) checkCanBuild(status Status) error { + if os.Geteuid() != 0 { + return fmt.Errorf("installing a kernel module requires root privileges (use sudo)") + } + + if i.Builder == nil { + return fmt.Errorf("no container runtime to run the lbd builder in") + } + + return i.checkCompilerAndHeaders(status) +} + +// checkCompilerAndHeaders covers the host conditions that make a build +// pointless: firmware that will refuse the result, a toolchain we cannot +// match, or no build tree to compile against. +func (i *Installer) checkCompilerAndHeaders(status Status) error { + if secureBootEnforcing(i.Options.root()) { + return fmt.Errorf("this host has Secure Boot enabled, which refuses unsigned kernel modules. " + + "miren cannot sign the module, so accelerator mode needs Secure Boot disabled or a signed module from your distribution") + } + + if status.Host.Compiler.Name == "clang" { + return fmt.Errorf("this kernel was built with %s, which the lbd builder does not support", + status.Host.Compiler) + } + + if status.Host.HeadersDir == "" && !status.Host.CanFetchHeaders() { + return fmt.Errorf("no kernel headers for %s on this host: %s", + status.Host.KernelRelease, status.Host.InstallHint()) + } + + return nil +} + +// build runs the builder container and leaves lbd.ko and lbdctl in the build +// directory's out/ subdirectory. +func (i *Installer) build(ctx context.Context, host Host) error { + dir := i.buildDir(host.KernelRelease) + srcDir := filepath.Join(dir, "src") + outDir := filepath.Join(dir, "out") + + // Start from clean source every time. A retry after a failed build must + // not inherit half-written object files. + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("clearing the build directory %s: %w", dir, err) + } + if err := materializeSource(srcDir); err != nil { + return err + } + if err := os.MkdirAll(outDir, 0755); err != nil { + return fmt.Errorf("creating %s: %w", outDir, err) + } + + i.Log.Info("building the lbd kernel module", + "kernel", host.KernelRelease, + "headers", host.HeadersDir, + "version", SourceVersion()) + + spec := BuildSpec{ + Name: buildContainerName, + Image: i.image(), + Args: []string{"/usr/local/bin/build-lbd"}, + Env: []string{ + "KERNEL_RELEASE=" + host.KernelRelease, + "KERNEL_HEADERS=" + host.HeadersDir, + "HOST_DISTRO_ID=" + host.DistroID, + "HOST_DISTRO_LIKE=" + strings.Join(host.DistroLike, " "), + }, + Mounts: []Mount{ + {Destination: "/src", Source: srcDir}, + {Destination: "/out", Source: outDir}, + }, + } + + if host.HeadersDir != "" { + // Mounted at their real paths, not under a prefix: a kernel build + // tree is full of absolute symlinks (/lib/modules//build usually + // points into /usr/src) and they only resolve if the paths match the + // host's. + spec.Mounts = append(spec.Mounts, + Mount{Destination: "/lib/modules", Source: "/lib/modules", ReadOnly: true}, + Mount{Destination: "/usr/src", Source: "/usr/src", ReadOnly: true}, + ) + } else { + // No build tree on the host, so the builder installs one for itself. + // Those paths are left unmounted precisely so it can write to them, + // and it needs a network to reach the distro archive. + spec.Env = append(spec.Env, "FETCH_HEADERS=1") + spec.HostNetwork = true + } + + if err := i.Builder.Build(ctx, spec); err != nil { + return err + } + + for _, name := range []string{"lbd.ko", "lbdctl"} { + if _, err := os.Stat(filepath.Join(outDir, name)); err != nil { + return fmt.Errorf("the build reported success but produced no %s", name) + } + } + return nil +} + +// uninstallPaths lists the files an install left on the host, taken from what +// it recorded rather than from the current state. +// +// Both details matter. The module path has to come from the marker, because +// after a kernel upgrade the running kernel is no longer the one the module +// was built for, and deriving the path from it would miss the real artifact +// and orphan it. lbdctl has to come from the marker too, because the lbd +// repo's README tells people to install their own at the same location, and a +// path we never recorded is not ours to delete. +func uninstallPaths(m *Marker) []string { + if m == nil { + return nil + } + + var paths []string + for _, p := range []string{m.ModulePath, m.LbdctlPath} { + if p != "" { + paths = append(paths, p) + } + } + return append(paths, modulesLoadConf) +} + +// installedLbdctl reports whether a previous miren install is what put lbdctl +// at this path. +func (i *Installer) installedLbdctl(path string) bool { + marker, err := readMarker(i.Options.dataPath()) + if err != nil || marker == nil { + return false + } + return marker.LbdctlPath == path +} + +// load installs the built artifacts and brings the module up. +func (i *Installer) load(ctx context.Context, host Host) error { + outDir := filepath.Join(i.buildDir(host.KernelRelease), "out") + + // Unload before touching anything on disk. rmmod refuses while a device is + // attached, and that is the common case rather than a rare one -- a node + // with a running app that has a disk. Overwriting lbd.ko and lbdctl first + // and only then discovering the refusal would leave a userspace lbdctl + // talking to a kernel module built from different source, with no marker + // written to say so. Failing here leaves the host exactly as it was. + if isModuleLoaded(i.Options.root(), ModuleName) { + i.Log.Info("unloading the previous lbd module") + if out, err := exec.CommandContext(ctx, "rmmod", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("could not unload the running lbd module, which is usually because a disk is still attached: %w: %s", + err, strings.TrimSpace(string(out))) + } + } + + dest := modulePath(host.KernelRelease) + if err := installFile(filepath.Join(outDir, "lbd.ko"), dest, 0644); err != nil { + return err + } + + // The lbd repo's own README tells people to install lbdctl here by hand, + // so an existing binary may well be theirs rather than a previous install + // of ours. Replacing it is still the right move -- lbdctl and the module + // have to come from the same source -- but it should not happen silently. + lbdctl := filepath.Join(lbdctlInstallDir, "lbdctl") + if _, err := os.Stat(lbdctl); err == nil && !i.installedLbdctl(lbdctl) { + i.Log.Warn("replacing an lbdctl that miren did not install", "path", lbdctl) + } + if err := installFile(filepath.Join(outDir, "lbdctl"), lbdctl, 0755); err != nil { + return err + } + + // depmod rebuilds the dependency index modprobe consults; without it + // modprobe cannot find a module that was just dropped into extra/. + if out, err := exec.CommandContext(ctx, "depmod", "-a", host.KernelRelease).CombinedOutput(); err != nil { + return fmt.Errorf("depmod failed: %w: %s", err, strings.TrimSpace(string(out))) + } + + if out, err := exec.CommandContext(ctx, "modprobe", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("modprobe %s failed: %w: %s", ModuleName, err, strings.TrimSpace(string(out))) + } + + if err := os.MkdirAll(filepath.Dir(modulesLoadConf), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(modulesLoadConf), err) + } + if err := os.WriteFile(modulesLoadConf, []byte(ModuleName+"\n"), 0644); err != nil { + return fmt.Errorf("writing %s: %w", modulesLoadConf, err) + } + + return nil +} + +// Uninstall unloads the module and removes everything the install put on the +// host, including the record that would otherwise trigger a rebuild later. +func (i *Installer) Uninstall(ctx context.Context) error { + if os.Geteuid() != 0 { + return fmt.Errorf("removing a kernel module requires root privileges (use sudo)") + } + + status, err := Probe(i.Options) + if err != nil { + return err + } + + if status.Marker == nil { + return fmt.Errorf("miren did not install lbd on this host, so there is nothing to remove") + } + + if status.Loaded { + if out, err := exec.CommandContext(ctx, "rmmod", ModuleName).CombinedOutput(); err != nil { + return fmt.Errorf("could not unload lbd, which is usually because a disk is still attached: %w: %s", + err, strings.TrimSpace(string(out))) + } + } + + for _, path := range uninstallPaths(status.Marker) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing %s: %w", path, err) + } + } + + // depmod has to reindex the kernel the module was installed for, which + // after an upgrade is not the one running now. + if out, err := exec.CommandContext(ctx, "depmod", "-a", status.Marker.KernelRelease).CombinedOutput(); err != nil { + i.Log.Warn("depmod failed after removing lbd", "error", err, + "kernel", status.Marker.KernelRelease, "output", strings.TrimSpace(string(out))) + } + + if err := os.RemoveAll(filepath.Join(i.Options.dataPath(), "lbd", "build")); err != nil { + i.Log.Warn("failed to remove the lbd build directory", "error", err) + } + + return removeMarker(i.Options.dataPath()) +} + +// installFile copies src to dest, creating the destination directory. It writes +// to a temporary name and renames, so a reader never sees a half-written module. +func installFile(src, dest string, mode os.FileMode) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("reading %s: %w", src, err) + } + + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(dest), err) + } + + tmp := dest + ".tmp" + if err := os.WriteFile(tmp, data, mode); err != nil { + return fmt.Errorf("writing %s: %w", tmp, err) + } + if err := os.Rename(tmp, dest); err != nil { + os.Remove(tmp) + return fmt.Errorf("installing %s: %w", dest, err) + } + return nil +} + +// secureBootEnforcing reports whether the firmware will refuse unsigned +// modules. The efivars file carries a five-byte value whose last byte is the +// flag; anything we cannot read is treated as "not enforcing", since guessing +// yes would block hosts that are simply not using EFI. +func secureBootEnforcing(root string) bool { + matches, err := filepath.Glob(filepath.Join(root, "sys/firmware/efi/efivars/SecureBoot-*")) + if err != nil || len(matches) == 0 { + return false + } + + data, err := os.ReadFile(matches[0]) + if err != nil || len(data) == 0 { + return false + } + return data[len(data)-1] == 1 +} diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go new file mode 100644 index 000000000..492f51c73 --- /dev/null +++ b/pkg/lbdmod/build_test.go @@ -0,0 +1,429 @@ +package lbdmod + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testInstaller(t *testing.T, root, dataPath string) *Installer { + t.Helper() + return &Installer{ + Log: slog.New(slog.DiscardHandler), + Options: Options{Root: root, DataPath: dataPath}, + } +} + +func TestCheckCanBuildRefusesClangKernels(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "proc/version", + "Linux version "+testRelease+" (build@) (Android clang version 17.0.4, LLD 17.0.4) #1 SMP\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + // Root and containerd are checked first, so exercise the compiler rule + // directly rather than depending on how the test runner is invoked. + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "built with clang-17") +} + +func TestCheckCanBuildNeedsHeadersOffTheDebianFamily(t *testing.T) { + // The builder image is Debian-based, so it can only fetch headers from a + // Debian-family archive. Anywhere else the operator has to install them, + // and the message has to name the package or they have to go looking. + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.11.4-301.fc41.x86_64\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel headers") + assert.Contains(t, err.Error(), "dnf install kernel-devel-6.11.4-301.fc41.x86_64") +} + +func TestCheckCanBuildLetsDebianFamilyHostsFetchHeaders(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", testRelease+"\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + require.Empty(t, status.Host.HeadersDir) + + // Missing headers are not fatal here: the builder installs them itself. + require.NoError(t, testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status)) +} + +func TestBuildAsksTheBuilderToFetchHeadersWhenTheHostHasNone(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + + host := Host{KernelRelease: testRelease, DistroID: "ubuntu", DistroLike: []string{"debian"}} + require.NoError(t, i.build(t.Context(), host)) + + assert.Contains(t, builder.spec.Env, "FETCH_HEADERS=1") + assert.True(t, builder.spec.HostNetwork, "fetching headers needs to reach the archive") + + // /lib/modules and /usr/src are deliberately left unmounted so the + // builder can install headers into its own filesystem. + for _, m := range builder.spec.Mounts { + assert.NotEqual(t, "/lib/modules", m.Destination) + assert.NotEqual(t, "/usr/src", m.Destination) + } +} + +func TestBuildAgainstHostHeadersNeedsNoNetwork(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + + host := Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"} + require.NoError(t, i.build(t.Context(), host)) + + assert.False(t, builder.spec.HostNetwork, "a build against host headers should reach nothing") + assert.NotContains(t, builder.spec.Env, "FETCH_HEADERS=1") +} + +func TestCheckCanBuildAcceptsAGoodHost(t *testing.T) { + root := ubuntuRoot(t) + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + require.NoError(t, testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status)) +} + +func TestSecureBootDetection(t *testing.T) { + // No EFI at all: not enforcing, rather than guessing yes and blocking a + // host that simply is not using EFI. + assert.False(t, secureBootEnforcing(t.TempDir())) + + off := t.TempDir() + writeFile(t, off, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 0})) + assert.False(t, secureBootEnforcing(off)) + + on := t.TempDir() + writeFile(t, on, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 1})) + assert.True(t, secureBootEnforcing(on)) +} + +func TestSecureBootBlocksTheBuild(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", + string([]byte{6, 0, 0, 0, 1})) + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + err = testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status) + require.Error(t, err) + assert.Contains(t, err.Error(), "Secure Boot") +} + +func TestBuildDirIsKeyedByKernelAndVersion(t *testing.T) { + i := testInstaller(t, "/", "/var/lib/miren") + + // A rebuild after a kernel upgrade must not reuse the old kernel's object + // files, so the two kernels get separate directories. + assert.NotEqual(t, i.buildDir("6.8.0-51-generic"), i.buildDir("6.8.0-52-generic")) + assert.Contains(t, i.buildDir("6.8.0-51-generic"), SourceVersion()) + assert.Contains(t, i.buildDir("6.8.0-51-generic"), "6.8.0-51-generic") +} + +func TestInstallFileIsAtomicAndCreatesParents(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "lbd.ko") + require.NoError(t, os.WriteFile(src, []byte("module"), 0644)) + + dest := filepath.Join(dir, "lib", "modules", testRelease, "extra", "lbd.ko") + require.NoError(t, installFile(src, dest, 0644)) + + data, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "module", string(data)) + + info, err := os.Stat(dest) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0644), info.Mode().Perm()) + + // No temporary file is left behind. + _, err = os.Stat(dest + ".tmp") + assert.True(t, os.IsNotExist(err)) + + // Overwriting an existing module works, which is the kernel-upgrade path. + require.NoError(t, os.WriteFile(src, []byte("newer"), 0644)) + require.NoError(t, installFile(src, dest, 0644)) + data, err = os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, "newer", string(data)) +} + +func TestInstallFileReportsAMissingSource(t *testing.T) { + dir := t.TempDir() + err := installFile(filepath.Join(dir, "absent"), filepath.Join(dir, "dest"), 0644) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading") +} + +// fakeBuilder stands in for the container runtime. onBuild may write into the +// spec's /out mount to imitate a successful compile. +type fakeBuilder struct { + spec BuildSpec + called bool + err error + onBuild func(spec BuildSpec) error +} + +func (f *fakeBuilder) Build(_ context.Context, spec BuildSpec) error { + f.called = true + f.spec = spec + if f.err != nil { + return f.err + } + if f.onBuild != nil { + return f.onBuild(spec) + } + return nil +} + +// hostMount finds a mount by its path inside the container. +func hostMount(t *testing.T, spec BuildSpec, dest string) Mount { + t.Helper() + for _, m := range spec.Mounts { + if m.Destination == dest { + return m + } + } + t.Fatalf("no mount at %s", dest) + return Mount{} +} + +// produceArtifacts imitates a builder that compiled successfully. +func produceArtifacts(t *testing.T) func(BuildSpec) error { + t.Helper() + return func(spec BuildSpec) error { + out := hostMount(t, spec, "/out").Source + for _, name := range []string{"lbd.ko", "lbdctl"} { + if err := os.WriteFile(filepath.Join(out, name), []byte(name), 0644); err != nil { + return err + } + } + return nil + } +} + +func TestBuildHandsTheBuilderSourceAndHeaders(t *testing.T) { + dataPath := t.TempDir() + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + + i := testInstaller(t, ubuntuRoot(t), dataPath) + i.Builder = builder + + host := Host{ + KernelRelease: testRelease, + HeadersDir: "/lib/modules/" + testRelease + "/build", + DistroID: "ubuntu", + DistroLike: []string{"debian"}, + } + require.NoError(t, i.build(t.Context(), host)) + require.True(t, builder.called) + + // The source is materialized into a writable directory, because the + // kernel build writes its object files next to the source. + src := hostMount(t, builder.spec, "/src") + assert.False(t, src.ReadOnly) + _, err := os.Stat(filepath.Join(src.Source, "lbd_main.c")) + require.NoError(t, err) + + // The host's kernel tree is mounted at its real path, so the absolute + // symlinks inside it resolve, and read-only so a build cannot damage it. + modules := hostMount(t, builder.spec, "/lib/modules") + assert.Equal(t, "/lib/modules", modules.Source) + assert.True(t, modules.ReadOnly) + assert.True(t, hostMount(t, builder.spec, "/usr/src").ReadOnly) + + assert.Contains(t, builder.spec.Env, "KERNEL_RELEASE="+testRelease) + assert.Contains(t, builder.spec.Env, "KERNEL_HEADERS=/lib/modules/"+testRelease+"/build") + assert.Contains(t, builder.spec.Env, "HOST_DISTRO_ID=ubuntu") + assert.Contains(t, builder.spec.Env, "HOST_DISTRO_LIKE=debian") + // The build runs unprivileged: compiling against read-only bind mounts + // needs no extra capabilities, and this container runs on every node with + // accelerator mode on. + assert.False(t, builder.spec.HostNetwork, + "a build against host headers should reach nothing") +} + +func TestBuildRejectsABuilderThatProducedNothing(t *testing.T) { + // The module Makefile downgrades unresolved symbols to warnings, so a + // zero exit does not prove there is a module to install. + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + i.Builder = &fakeBuilder{} + + err := i.build(t.Context(), Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "produced no lbd.ko") +} + +func TestBuildSurfacesTheBuilderError(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + i.Builder = &fakeBuilder{err: &BuildFailedError{ExitCode: 2, Output: "error: no kernel headers"}} + + err := i.build(t.Context(), Host{KernelRelease: testRelease}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel headers") +} + +func TestBuildStartsFromCleanSource(t *testing.T) { + dataPath := t.TempDir() + i := testInstaller(t, ubuntuRoot(t), dataPath) + host := Host{KernelRelease: testRelease, HeadersDir: "/lib/modules/" + testRelease + "/build"} + + // Leave debris from a build that failed partway through. + stale := filepath.Join(i.buildDir(testRelease), "src", "lbd_main.o") + require.NoError(t, os.MkdirAll(filepath.Dir(stale), 0755)) + require.NoError(t, os.WriteFile(stale, []byte("stale object"), 0644)) + + i.Builder = &fakeBuilder{onBuild: produceArtifacts(t)} + require.NoError(t, i.build(t.Context(), host)) + + _, err := os.Stat(stale) + assert.True(t, os.IsNotExist(err), "object files from a failed build must not survive into the retry") +} + +func TestBuildUsesTheConfiguredImage(t *testing.T) { + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{onBuild: produceArtifacts(t)} + i.Builder = builder + i.Image = "example.test/lbd-builder:local" + + require.NoError(t, i.build(t.Context(), Host{KernelRelease: testRelease})) + assert.Equal(t, "example.test/lbd-builder:local", builder.spec.Image) +} + +func TestEnsureCurrentLeavesAHostThatNeverOptedInAlone(t *testing.T) { + // No install record means accelerator mode was never enabled here, so + // startup must not pay for an unattended compile. + i := testInstaller(t, ubuntuRoot(t), t.TempDir()) + builder := &fakeBuilder{} + i.Builder = builder + + rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) + assert.False(t, rebuilt) + assert.False(t, builder.called, "a host with no install record must not be built for") +} + +func TestEnsureCurrentSkipsAHealthyHost(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + builder := &fakeBuilder{} + i.Builder = builder + + rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) + assert.False(t, rebuilt) + assert.False(t, builder.called) +} + +func TestEnsureCurrentRebuildsAfterAKernelUpgrade(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + // The host installed lbd for a kernel it is no longer running. + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: "6.8.0-45-generic", + ModulePath: modulePath("6.8.0-45-generic"), + })) + + i := testInstaller(t, root, dataPath) + builder := &fakeBuilder{} + i.Builder = builder + + // It decides to act, which is the point. Whether it then gets past the + // root check depends on how the tests were invoked, and the install needs + // a real kernel and depmod either way -- so what is asserted is that it + // did not quietly do nothing. + rebuilt, err := i.EnsureCurrent(t.Context()) + assert.False(t, rebuilt && err != nil, "a rebuild cannot both succeed and fail") + assert.True(t, err != nil || builder.called, + "a stale module must trigger a rebuild attempt, not silence") +} + +func TestBuildFailedErrorQuotesTheOutput(t *testing.T) { + err := &BuildFailedError{ExitCode: 2, Output: "error: no kernel headers for 6.8.0-51-generic"} + assert.Contains(t, err.Error(), "exit 2") + assert.Contains(t, err.Error(), "no kernel headers") + + bare := &BuildFailedError{ExitCode: 1} + assert.Equal(t, "the lbd build failed (exit 1)", bare.Error()) + assert.False(t, strings.HasSuffix(bare.Error(), ":\n")) +} + +func TestUninstallRemovesWhatWasInstalledNotWhatIsRunning(t *testing.T) { + // The host has moved on to a newer kernel since the install. Deriving the + // module path from the running kernel would miss the real artifact and + // leave it on disk forever. + installedKernel := "6.8.0-45-generic" + m := &Marker{ + LbdVersion: SourceVersion(), + KernelRelease: installedKernel, + ModulePath: modulePath(installedKernel), + LbdctlPath: "/usr/local/bin/lbdctl", + } + + paths := uninstallPaths(m) + + assert.Contains(t, paths, modulePath(installedKernel)) + assert.NotContains(t, paths, modulePath(testRelease), + "the running kernel's path was never installed") + assert.Contains(t, paths, "/usr/local/bin/lbdctl") + assert.Contains(t, paths, modulesLoadConf) +} + +func TestUninstallLeavesAnLbdctlItDidNotInstall(t *testing.T) { + // An operator who followed the lbd repo's README has their own lbdctl at + // the same path. A marker that never recorded one must not license + // deleting it. + m := &Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + } + + paths := uninstallPaths(m) + + assert.Contains(t, paths, modulePath(testRelease)) + for _, p := range paths { + assert.NotContains(t, p, "lbdctl", "an unrecorded lbdctl is not ours to remove") + } +} + +func TestUninstallPathsWithNoMarker(t *testing.T) { + // No install record means miren put nothing on this host. + assert.Empty(t, uninstallPaths(nil)) +} diff --git a/pkg/lbdmod/builder.go b/pkg/lbdmod/builder.go new file mode 100644 index 000000000..3408fa272 --- /dev/null +++ b/pkg/lbdmod/builder.go @@ -0,0 +1,69 @@ +package lbdmod + +import ( + "context" + "fmt" +) + +// Mount is a bind mount into the builder. +type Mount struct { + // Source is the path on the host. + Source string + + // Destination is where it appears inside the builder. + Destination string + + // ReadOnly keeps the builder from writing through the mount. + ReadOnly bool +} + +// BuildSpec describes one run of the builder image. +type BuildSpec struct { + // Name is the container's id. It is fixed rather than random so a build + // killed before its own cleanup leaves something the next run can find. + Name string + + // Image is the builder image reference. + Image string + + // Args replaces the image's entrypoint arguments. + Args []string + + // Env is added to the image's environment as "KEY=value" pairs. + Env []string + + // Mounts are bind mounts into the container. + Mounts []Mount + + // HostNetwork gives the builder the host's network and resolver. It is + // set only when the builder has to fetch kernel headers for itself; a + // build against a host's own headers needs no network at all. + HostNetwork bool +} + +// Builder runs the builder image once and waits for it to finish. It is an +// interface so this package stays free of container runtime dependencies -- +// components/diskio and controllers/disk import it only to ask whether lbd is +// available, and should not pull containerd along with them. The containerd +// implementation is pkg/lbdmod/ctrbuild. +type Builder interface { + // Build runs the container to completion and returns nil only if it + // exited zero. A non-zero exit should be reported as a BuildFailedError + // so the build output survives. + Build(ctx context.Context, spec BuildSpec) error +} + +// BuildFailedError reports a builder container that ran but exited non-zero. +// It carries the tail of the build output, which is where the real explanation +// lives. +type BuildFailedError struct { + ExitCode uint32 + Output string +} + +func (e *BuildFailedError) Error() string { + if e.Output == "" { + return fmt.Sprintf("the lbd build failed (exit %d)", e.ExitCode) + } + return fmt.Sprintf("the lbd build failed (exit %d):\n%s", e.ExitCode, e.Output) +} diff --git a/pkg/lbdmod/ctrbuild/ctrbuild.go b/pkg/lbdmod/ctrbuild/ctrbuild.go new file mode 100644 index 000000000..e03ea004e --- /dev/null +++ b/pkg/lbdmod/ctrbuild/ctrbuild.go @@ -0,0 +1,268 @@ +// Package ctrbuild runs the lbd builder image on containerd. +// +// It is separate from pkg/lbdmod so that package can stay free of container +// runtime dependencies: components/diskio and controllers/disk import lbdmod +// only to ask whether lbd is usable, and should not pull containerd in behind +// that question. +package ctrbuild + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync" + "syscall" + "time" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/pkg/cio" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "miren.dev/runtime/pkg/lbdmod" +) + +// DefaultNamespace is the containerd namespace miren's own containers live in. +const DefaultNamespace = "miren" + +// cleanupTimeout bounds the teardown of a container after the work is done, +// including the case where the caller's context has already been cancelled. +const cleanupTimeout = 30 * time.Second + +// outputTailLines is how much of the build output a failure quotes back. +const outputTailLines = 40 + +// Builder runs a build container on containerd. +// +// Nothing in components/ does this: every component there is a supervised +// daemon with a restart policy. A build is the opposite -- it runs to +// completion, its exit code is the answer, and it must leave nothing behind. +type Builder struct { + cc *containerd.Client + log *slog.Logger +} + +// New returns a Builder that runs containers on cc. +func New(cc *containerd.Client, log *slog.Logger) *Builder { + return &Builder{cc: cc, log: log} +} + +// Build pulls the image, runs the container to completion, and tears +// everything down. It returns an error unless the container exited zero. +func (b *Builder) Build(ctx context.Context, spec lbdmod.BuildSpec) error { + ctx = namespaces.WithNamespace(ctx, DefaultNamespace) + + image, err := b.resolveImage(ctx, spec.Image) + if err != nil { + return err + } + + // A previous run that died before its own cleanup leaves the container + // behind and its name taken. + if existing, err := b.cc.LoadContainer(ctx, spec.Name); err == nil { + b.log.Info("removing a container left by an earlier build", "container", spec.Name) + b.removeContainer(ctx, existing) + } + + // Deliberately not privileged. Compiling C against read-only bind mounts + // needs no extra capabilities -- runc's default masked and read-only + // paths cover /proc and /sys, not /lib/modules or /usr/src -- and this + // container runs on every node that turns accelerator mode on. Verified + // against a real containerd for both the host-headers and header-fetch + // paths; if that ever stops holding, add the one capability that is + // missing rather than all of them. + opts := []oci.SpecOpts{ + oci.WithImageConfig(image), + oci.WithProcessArgs(spec.Args...), + oci.WithEnv(spec.Env), + oci.WithMounts(ociMounts(spec.Mounts)), + } + if spec.HostNetwork { + opts = append(opts, oci.WithHostNamespace(specs.NetworkNamespace), oci.WithHostResolvconf) + } + + container, err := b.cc.NewContainer(ctx, spec.Name, + containerd.WithImage(image), + containerd.WithNewSnapshot(spec.Name+"-snapshot", image), + containerd.WithNewSpec(opts...), + ) + if err != nil { + return fmt.Errorf("creating the build container: %w", err) + } + defer b.removeContainer(context.WithoutCancel(ctx), container) + + output := newTailWriter(b.log, spec.Name, outputTailLines) + task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, output, output))) + if err != nil { + return fmt.Errorf("creating the build task: %w", err) + } + defer func() { + output.flush() + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + if _, err := task.Delete(cleanupCtx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete the build task", "error", err) + } + }() + + // Establish the exit channel before starting, so the exit event cannot be + // missed by a build that finishes immediately. + exitCh, err := task.Wait(ctx) + if err != nil { + return fmt.Errorf("waiting on the build task: %w", err) + } + + if err := task.Start(ctx); err != nil { + return fmt.Errorf("starting the build: %w", err) + } + + select { + case status := <-exitCh: + output.flush() + if err := status.Error(); err != nil { + return fmt.Errorf("the build did not report a result: %w", err) + } + if code := status.ExitCode(); code != 0 { + return &lbdmod.BuildFailedError{ExitCode: code, Output: output.Tail()} + } + return nil + + case <-ctx.Done(): + // Kill rather than leaving a compile running against a directory we + // are about to delete. + if err := task.Kill(context.WithoutCancel(ctx), syscall.SIGKILL); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to kill the build task", "error", err) + } + return ctx.Err() + } +} + +// resolveImage returns the builder image, pulling it only if it is not already +// in the local store. +// +// Preferring the local copy is what lets a node that has already built once +// rebuild without a registry, and what makes a side-loaded image usable at all +// -- `ctr images import` plus --image is the only way to run a builder on a +// node with no reachable registry. It is safe here because the builder +// reference is pinned to a tag that never moves. +func (b *Builder) resolveImage(ctx context.Context, ref string) (containerd.Image, error) { + if img, err := b.cc.GetImage(ctx, ref); err == nil { + b.log.Info("using the lbd builder image already on this node", "image", ref) + return img, nil + } + + b.log.Info("pulling the lbd builder image", "image", ref) + img, err := b.cc.Pull(ctx, ref, containerd.WithPullUnpack) + if err != nil { + return nil, fmt.Errorf("pulling %s: %w", ref, err) + } + return img, nil +} + +// removeContainer deletes a container and its snapshot, tolerating a container +// that is already gone. +func (b *Builder) removeContainer(ctx context.Context, container containerd.Container) { + ctx, cancel := context.WithTimeout(namespaces.WithNamespace(ctx, DefaultNamespace), cleanupTimeout) + defer cancel() + + if task, err := container.Task(ctx, nil); err == nil { + if _, err := task.Delete(ctx, containerd.WithProcessKill); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete a leftover build task", "error", err) + } + } + + if err := container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil && !errdefs.IsNotFound(err) { + b.log.Warn("failed to delete the build container", "error", err, "container", container.ID()) + } +} + +// ociMounts converts the runtime-agnostic mounts into OCI bind mounts. +func ociMounts(mounts []lbdmod.Mount) []specs.Mount { + out := make([]specs.Mount, 0, len(mounts)) + for _, m := range mounts { + access := "rw" + if m.ReadOnly { + access = "ro" + } + out = append(out, specs.Mount{ + Destination: m.Destination, + Type: "bind", + Source: m.Source, + Options: []string{"rbind", access}, + }) + } + return out +} + +// tailWriter forwards container output to a logger a line at a time and keeps +// the last few lines, so a failure can quote what actually went wrong instead +// of just its exit code. +type tailWriter struct { + log *slog.Logger + name string + limit int + + mu sync.Mutex + partial []byte + lines []string +} + +func newTailWriter(log *slog.Logger, name string, limit int) *tailWriter { + return &tailWriter{log: log, name: name, limit: limit} +} + +func (w *tailWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + w.partial = append(w.partial, p...) + for { + idx := strings.IndexByte(string(w.partial), '\n') + if idx < 0 { + break + } + line := strings.TrimRight(string(w.partial[:idx]), "\r") + w.partial = w.partial[idx+1:] + w.record(line) + } + return len(p), nil +} + +// flush emits whatever the container left without a trailing newline. +func (w *tailWriter) flush() { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.partial) > 0 { + w.record(strings.TrimRight(string(w.partial), "\r")) + w.partial = nil + } +} + +// record must be called with w.mu held. +// +// Build output goes to Debug, not Info. A compile is thirty-odd lines of make +// output that would be identical the next thousand times it runs, and the +// automatic rebuild after a kernel upgrade emits them into the daemon log with +// nobody watching. The outcome is logged at Info by the installer, and a +// failure carries the tail in BuildFailedError regardless of level -- so the +// case that actually needs this output never depended on it being Info. +// Operators watching a build can see it with -v. +func (w *tailWriter) record(line string) { + if strings.TrimSpace(line) == "" { + return + } + w.log.Debug(line, "source", w.name) + w.lines = append(w.lines, line) + if len(w.lines) > w.limit { + w.lines = w.lines[len(w.lines)-w.limit:] + } +} + +// Tail returns the retained lines as a single block. +func (w *tailWriter) Tail() string { + w.mu.Lock() + defer w.mu.Unlock() + return strings.Join(w.lines, "\n") +} diff --git a/pkg/lbdmod/ctrbuild/ctrbuild_test.go b/pkg/lbdmod/ctrbuild/ctrbuild_test.go new file mode 100644 index 000000000..9d0d440ac --- /dev/null +++ b/pkg/lbdmod/ctrbuild/ctrbuild_test.go @@ -0,0 +1,76 @@ +package ctrbuild + +import ( + "bytes" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/lbdmod" +) + +func TestTailWriterLogsLinesAndKeepsTheTail(t *testing.T) { + var logged bytes.Buffer + log := slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{Level: slog.LevelDebug})) + w := newTailWriter(log, "build", 3) + + // Split across writes, as a pipe delivers it. + _, err := w.Write([]byte("first\nsec")) + require.NoError(t, err) + _, err = w.Write([]byte("ond\nthird\nfourth\n")) + require.NoError(t, err) + + assert.Equal(t, "second\nthird\nfourth", w.Tail(), "only the last few lines are kept") + assert.Contains(t, logged.String(), "first", "every line still reaches the log") + assert.Contains(t, logged.String(), "fourth") +} + +func TestTailWriterKeepsBuildOutputOffInfo(t *testing.T) { + // The automatic rebuild after a kernel upgrade runs unattended, so a whole + // compile at Info would be noise in the daemon log. The tail is kept + // regardless of level, which is what a failure reports. + var atInfo bytes.Buffer + log := slog.New(slog.NewTextHandler(&atInfo, &slog.HandlerOptions{Level: slog.LevelInfo})) + w := newTailWriter(log, "build", 10) + + _, err := w.Write([]byte(" CC [M] /src/lbd_main.o\n")) + require.NoError(t, err) + + assert.Empty(t, atInfo.String(), "compile output must not reach an Info-level log") + assert.Contains(t, w.Tail(), "lbd_main.o", "the tail is still captured for failure reporting") +} + +func TestTailWriterFlushesAnUnterminatedLine(t *testing.T) { + w := newTailWriter(slog.New(slog.DiscardHandler), "build", 10) + _, err := w.Write([]byte("no trailing newline")) + require.NoError(t, err) + + assert.Empty(t, w.Tail(), "an unterminated line is not a line yet") + w.flush() + assert.Equal(t, "no trailing newline", w.Tail()) +} + +func TestTailWriterIgnoresBlankLines(t *testing.T) { + w := newTailWriter(slog.New(slog.DiscardHandler), "build", 10) + _, err := w.Write([]byte("\n \nreal\n\n")) + require.NoError(t, err) + assert.Equal(t, "real", w.Tail()) +} + +func TestOCIMountsCarryReadOnlyThrough(t *testing.T) { + mounts := ociMounts([]lbdmod.Mount{ + {Source: "/build/src", Destination: "/src"}, + {Source: "/lib/modules", Destination: "/lib/modules", ReadOnly: true}, + }) + + require.Len(t, mounts, 2) + + assert.Equal(t, "/src", mounts[0].Destination) + assert.Equal(t, "/build/src", mounts[0].Source) + assert.Equal(t, "bind", mounts[0].Type) + assert.Equal(t, []string{"rbind", "rw"}, mounts[0].Options) + + // The host's module tree must never be writable from the builder. + assert.Equal(t, []string{"rbind", "ro"}, mounts[1].Options) +} diff --git a/pkg/lbdmod/kernel.go b/pkg/lbdmod/kernel.go new file mode 100644 index 000000000..1e76f7576 --- /dev/null +++ b/pkg/lbdmod/kernel.go @@ -0,0 +1,240 @@ +package lbdmod + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +// Host describes the machine we are about to build a module for. +type Host struct { + // KernelRelease is `uname -r`, e.g. "6.8.0-51-generic". A module built + // here only loads on this exact release. + KernelRelease string + + // HeadersDir is the kernel build tree the module must compile against, + // or empty if the host does not have one installed. + HeadersDir string + + // Compiler is the toolchain the running kernel was built with, as + // reported by /proc/version. + Compiler Compiler + + // DistroID and DistroLike come from /etc/os-release: ID is the specific + // distribution ("ubuntu"), and DistroLike is its family ("debian"), which + // is what decides how to name a header package. Either may be empty. + DistroID string + DistroLike []string +} + +// Compiler identifies what built the running kernel. +type Compiler struct { + // Name is "gcc" or "clang", or empty when /proc/version says something + // we do not recognize. + Name string + + // Major is the compiler's major version, or 0 if it could not be read. + Major int +} + +// String renders the compiler for logs and error messages. +func (c Compiler) String() string { + switch { + case c.Name == "": + return "unknown" + case c.Major == 0: + return c.Name + default: + return fmt.Sprintf("%s-%d", c.Name, c.Major) + } +} + +// HeaderPackage names the distro package that provides the build tree for this +// kernel, so an error message can tell the operator exactly what to install. +// Returns an empty string when the distribution is not one we recognize. +func (h Host) HeaderPackage() string { + for _, id := range append([]string{h.DistroID}, h.DistroLike...) { + switch id { + case "debian", "ubuntu": + return "linux-headers-" + h.KernelRelease + case "fedora", "rhel", "centos": + return "kernel-devel-" + h.KernelRelease + case "arch": + return "linux-headers" + case "alpine": + return "linux-headers" + case "suse", "opensuse", "opensuse-leap", "opensuse-tumbleweed", "sles": + return "kernel-devel" + } + } + return "" +} + +// CanFetchHeaders reports whether the builder can install kernel headers for +// itself rather than borrowing the host's. +// +// The builder image is Debian-based, so it can only reach a Debian-family +// archive. On any other distribution the operator has to install the headers, +// which is what InstallHint tells them to do. Even on a Debian-family host the +// fetch can still come up empty -- a kernel that has aged out of the archive, +// or a Debian host whose package is not in the builder's Ubuntu sources -- and +// the builder reports that itself, naming the package it could not find. +func (h Host) CanFetchHeaders() bool { + return h.hasFamily("debian") || h.hasFamily("ubuntu") +} + +// InstallHint is the sentence to show an operator whose host has no kernel +// build tree. +func (h Host) InstallHint() string { + pkg := h.HeaderPackage() + if pkg == "" { + return fmt.Sprintf("install the kernel headers for %s and try again", h.KernelRelease) + } + switch { + case h.hasFamily("debian"): + return "run: apt-get install " + pkg + case h.hasFamily("fedora"), h.hasFamily("rhel"), h.hasFamily("centos"): + return "run: dnf install " + pkg + case h.hasFamily("arch"): + return "run: pacman -S " + pkg + case h.hasFamily("alpine"): + return "run: apk add " + pkg + case h.hasFamily("suse"), h.hasFamily("sles"): + return "run: zypper install " + pkg + } + return "install " + pkg + " and try again" +} + +func (h Host) hasFamily(id string) bool { + if h.DistroID == id { + return true + } + for _, like := range h.DistroLike { + if like == id { + return true + } + } + return false +} + +// headerCandidates lists where a kernel build tree may live, in the order to +// try. Debian and Ubuntu populate /lib/modules//build; Fedora ships the +// tree under /usr/src/kernels and does not always leave that symlink behind. +func headerCandidates(release string) []string { + return []string{ + filepath.Join("/lib/modules", release, "build"), + filepath.Join("/usr/src/kernels", release), + filepath.Join("/usr/src", "linux-headers-"+release), + } +} + +// findHeaders returns the first candidate that looks like a usable kernel build +// tree, or an empty string. Makefile is the file the module's own build invokes, +// so its absence means the tree is unusable however complete it otherwise looks. +func findHeaders(root, release string) string { + for _, dir := range headerCandidates(release) { + if _, err := os.Stat(filepath.Join(root, dir, "Makefile")); err == nil { + return dir + } + } + return "" +} + +// compilerPattern matches the toolchain stanza /proc/version carries, e.g. +// "(gcc-13 (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0, ...)" or "(clang version 18.1.3". +var compilerPattern = regexp.MustCompile(`\b(gcc|clang)\b[^0-9]*([0-9]+)`) + +// parseCompiler pulls the building toolchain out of a /proc/version line. +func parseCompiler(procVersion string) Compiler { + m := compilerPattern.FindStringSubmatch(procVersion) + if m == nil { + return Compiler{} + } + major, err := strconv.Atoi(m[2]) + if err != nil { + return Compiler{Name: m[1]} + } + return Compiler{Name: m[1], Major: major} +} + +// parseOSRelease reads the ID and ID_LIKE fields of an os-release file. Values +// may be quoted, and ID_LIKE is a space-separated list ordered most-specific +// first. +func parseOSRelease(content string) (id string, like []string) { + for _, line := range strings.Split(content, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + value = strings.Trim(value, `"'`) + switch key { + case "ID": + id = value + case "ID_LIKE": + like = strings.Fields(value) + } + } + return id, like +} + +// kernelRelease reports `uname -r`. It reads the procfs file rather than +// shelling out, and falls back to uname(1) only for a real host, since a +// fixture root has no process to ask. +func kernelRelease(root string) (string, error) { + data, err := os.ReadFile(filepath.Join(root, "proc/sys/kernel/osrelease")) + if err == nil { + if release := strings.TrimSpace(string(data)); release != "" { + return release, nil + } + } + + if root != "/" { + return "", fmt.Errorf("no kernel release under %s: %w", root, err) + } + + out, unameErr := exec.Command("uname", "-r").Output() + if unameErr != nil { + return "", fmt.Errorf("could not determine the kernel release: %w", err) + } + release := strings.TrimSpace(string(out)) + if release == "" { + return "", fmt.Errorf("could not determine the kernel release: uname -r said nothing") + } + return release, nil +} + +// DetectHost inspects the machine miren is running on. root is the filesystem +// to read from, normally "/"; tests pass a fixture directory. +// +// Missing kernel headers are not an error here: whether they can be fetched +// instead is the caller's decision. +func DetectHost(root string) (Host, error) { + release, err := kernelRelease(root) + if err != nil { + return Host{}, err + } + + h := Host{ + KernelRelease: release, + HeadersDir: findHeaders(root, release), + } + + if data, err := os.ReadFile(filepath.Join(root, "proc/version")); err == nil { + h.Compiler = parseCompiler(string(data)) + } + + for _, path := range []string{"etc/os-release", "usr/lib/os-release"} { + data, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + continue + } + h.DistroID, h.DistroLike = parseOSRelease(string(data)) + break + } + + return h, nil +} diff --git a/pkg/lbdmod/kernel_test.go b/pkg/lbdmod/kernel_test.go new file mode 100644 index 000000000..65e7623bd --- /dev/null +++ b/pkg/lbdmod/kernel_test.go @@ -0,0 +1,156 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeFile creates path under root, making its parents. +func writeFile(t *testing.T, root, path, content string) { + t.Helper() + full := filepath.Join(root, path) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0755)) + require.NoError(t, os.WriteFile(full, []byte(content), 0644)) +} + +func TestParseCompiler(t *testing.T) { + tests := []struct { + name string + input string + want Compiler + }{ + { + name: "ubuntu gcc", + input: "Linux version 6.8.0-51-generic (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0, GNU ld (GNU Binutils) 2.42) #52-Ubuntu SMP\n", + want: Compiler{Name: "gcc", Major: 13}, + }, + { + name: "fedora gcc", + input: "Linux version 6.11.4-301.fc41.x86_64 (mockbuild@) (gcc (GCC) 14.2.1 20240912, GNU ld version 2.43.1) #1 SMP\n", + want: Compiler{Name: "gcc", Major: 14}, + }, + { + name: "clang built", + input: "Linux version 6.6.30-android14 (build@) (Android (11368139) clang version 17.0.4, LLD 17.0.4) #1 SMP\n", + want: Compiler{Name: "clang", Major: 17}, + }, + { + name: "unrecognized", + input: "Linux version 5.10.0 (someone@somewhere) #1 SMP\n", + want: Compiler{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, parseCompiler(tt.input)) + }) + } +} + +func TestCompilerString(t *testing.T) { + assert.Equal(t, "gcc-13", Compiler{Name: "gcc", Major: 13}.String()) + assert.Equal(t, "clang", Compiler{Name: "clang"}.String()) + assert.Equal(t, "unknown", Compiler{}.String()) +} + +func TestParseOSRelease(t *testing.T) { + id, like := parseOSRelease(`NAME="Ubuntu" +ID=ubuntu +ID_LIKE=debian +VERSION_ID="24.04" +`) + assert.Equal(t, "ubuntu", id) + assert.Equal(t, []string{"debian"}, like) + + id, like = parseOSRelease("ID=fedora\n") + assert.Equal(t, "fedora", id) + assert.Empty(t, like) + + id, like = parseOSRelease(`ID="rocky" +ID_LIKE="rhel centos fedora" +`) + assert.Equal(t, "rocky", id) + assert.Equal(t, []string{"rhel", "centos", "fedora"}, like) +} + +func TestDetectHostReadsFixtureRoot(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.8.0-51-generic\n") + writeFile(t, root, "proc/version", + "Linux version 6.8.0-51-generic (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0) 13.3.0) #52-Ubuntu SMP\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + writeFile(t, root, "lib/modules/6.8.0-51-generic/build/Makefile", "# kernel build tree\n") + + host, err := DetectHost(root) + require.NoError(t, err) + + assert.Equal(t, "6.8.0-51-generic", host.KernelRelease) + assert.Equal(t, "/lib/modules/6.8.0-51-generic/build", host.HeadersDir) + assert.Equal(t, Compiler{Name: "gcc", Major: 13}, host.Compiler) + assert.Equal(t, "ubuntu", host.DistroID) + assert.Equal(t, []string{"debian"}, host.DistroLike) +} + +func TestDetectHostWithoutHeaders(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.11.4-301.fc41.x86_64\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + + host, err := DetectHost(root) + require.NoError(t, err) + + // Missing headers are reported, not an error: the builder may be able to + // fetch them instead. + assert.Empty(t, host.HeadersDir) + assert.Equal(t, "kernel-devel-6.11.4-301.fc41.x86_64", host.HeaderPackage()) + assert.Equal(t, "run: dnf install kernel-devel-6.11.4-301.fc41.x86_64", host.InstallHint()) +} + +func TestDetectHostFindsFedoraStyleHeaders(t *testing.T) { + root := t.TempDir() + release := "6.11.4-301.fc41.x86_64" + writeFile(t, root, "proc/sys/kernel/osrelease", release+"\n") + writeFile(t, root, "etc/os-release", "ID=fedora\n") + // Fedora ships the build tree here and does not always leave the + // /lib/modules//build symlink behind. + writeFile(t, root, "usr/src/kernels/"+release+"/Makefile", "# kernel build tree\n") + + host, err := DetectHost(root) + require.NoError(t, err) + assert.Equal(t, "/usr/src/kernels/"+release, host.HeadersDir) +} + +func TestDetectHostFallsBackToUsrLibOSRelease(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "6.8.0-51-generic\n") + writeFile(t, root, "usr/lib/os-release", "ID=debian\n") + + host, err := DetectHost(root) + require.NoError(t, err) + assert.Equal(t, "debian", host.DistroID) +} + +func TestDetectHostNeedsAKernelRelease(t *testing.T) { + _, err := DetectHost(t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel release") +} + +func TestHeaderPackageUsesTheDistroFamily(t *testing.T) { + // A derivative distro we do not name explicitly still gets the right + // package via ID_LIKE. + h := Host{KernelRelease: "6.8.0-51", DistroID: "pop", DistroLike: []string{"ubuntu", "debian"}} + assert.Equal(t, "linux-headers-6.8.0-51", h.HeaderPackage()) + assert.Equal(t, "run: apt-get install linux-headers-6.8.0-51", h.InstallHint()) +} + +func TestHeaderPackageUnknownDistro(t *testing.T) { + h := Host{KernelRelease: "6.8.0-51", DistroID: "somethingelse"} + assert.Empty(t, h.HeaderPackage()) + assert.Equal(t, "install the kernel headers for 6.8.0-51 and try again", h.InstallHint()) +} diff --git a/pkg/lbdmod/lock.go b/pkg/lbdmod/lock.go new file mode 100644 index 000000000..8520fa11f --- /dev/null +++ b/pkg/lbdmod/lock.go @@ -0,0 +1,67 @@ +package lbdmod + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +// ErrBuildInProgress is returned when another process is already building the +// module on this host. +var ErrBuildInProgress = errors.New("an lbd build is already running on this host") + +// buildLock serializes installs across processes. +// +// Two can genuinely overlap now that the server rebuilds unattended: an +// operator running `miren disk accelerator install` while a kernel upgrade has +// the server rebuilding in the background. Both would use the same build +// directory, which build() clears with RemoveAll, and the same fixed container +// name, which the builder deletes before creating its own -- so each would +// destroy the other's work and report a baffling failure. +// +// The lock is an flock rather than a lockfile whose existence is the signal, +// so a process killed mid-build releases it instead of wedging the host until +// someone deletes a stale file. +type buildLock struct { + f *os.File +} + +// acquireBuildLock takes the host-wide build lock without waiting. It returns +// ErrBuildInProgress if another process holds it. +func acquireBuildLock(dataPath string) (*buildLock, error) { + dir := filepath.Join(dataPath, "lbd") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("creating %s: %w", dir, err) + } + + path := filepath.Join(dir, "build.lock") + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", path, err) + } + + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + f.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, ErrBuildInProgress + } + return nil, fmt.Errorf("locking %s: %w", path, err) + } + + return &buildLock{f: f}, nil +} + +// release drops the lock. The file is left behind on purpose: removing it +// would let a second process create and lock a new file at the same path while +// a third still holds the old one. +func (l *buildLock) release() { + if l == nil || l.f == nil { + return + } + unix.Flock(int(l.f.Fd()), unix.LOCK_UN) + l.f.Close() + l.f = nil +} diff --git a/pkg/lbdmod/lock_test.go b/pkg/lbdmod/lock_test.go new file mode 100644 index 000000000..3d0ed956a --- /dev/null +++ b/pkg/lbdmod/lock_test.go @@ -0,0 +1,75 @@ +package lbdmod + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildLockExcludesASecondHolder(t *testing.T) { + dataPath := t.TempDir() + + first, err := acquireBuildLock(dataPath) + require.NoError(t, err) + require.NotNil(t, first) + + // The case this guards: an operator running `install` while the server is + // already rebuilding after a kernel upgrade. Without the lock both would + // clear the same build directory and delete each other's container. + second, err := acquireBuildLock(dataPath) + require.ErrorIs(t, err, ErrBuildInProgress) + assert.Nil(t, second) + + first.release() + + // Once the first is done the lock is available again. + third, err := acquireBuildLock(dataPath) + require.NoError(t, err) + third.release() +} + +func TestBuildLockCreatesItsDirectory(t *testing.T) { + // A host installing for the first time has no /var/lib/miren/lbd yet. + dataPath := filepath.Join(t.TempDir(), "fresh") + + lock, err := acquireBuildLock(dataPath) + require.NoError(t, err) + defer lock.release() + + _, err = os.Stat(filepath.Join(dataPath, "lbd", "build.lock")) + require.NoError(t, err) +} + +func TestReleasingTwiceIsSafe(t *testing.T) { + lock, err := acquireBuildLock(t.TempDir()) + require.NoError(t, err) + + lock.release() + lock.release() + + var nilLock *buildLock + nilLock.release() +} + +func TestBuildLockIsNotHeldByAStaleFile(t *testing.T) { + // An flock dies with the process that held it, so a build killed partway + // through must not wedge the host behind a leftover file. + dataPath := t.TempDir() + + lock, err := acquireBuildLock(dataPath) + require.NoError(t, err) + lock.release() + + // The file survives on purpose; only the lock is gone. + _, err = os.Stat(filepath.Join(dataPath, "lbd", "build.lock")) + require.NoError(t, err) + + again, err := acquireBuildLock(dataPath) + require.NoError(t, err, "a leftover lock file must not block a later build") + assert.False(t, errors.Is(err, ErrBuildInProgress)) + again.release() +} diff --git a/pkg/lbdmod/probe.go b/pkg/lbdmod/probe.go new file mode 100644 index 000000000..ae5ba7843 --- /dev/null +++ b/pkg/lbdmod/probe.go @@ -0,0 +1,137 @@ +package lbdmod + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + + lbdsrc "miren.dev/runtime/third_party/lbd" +) + +// DefaultDataPath is where miren keeps the lbd install record and build +// scratch space. +const DefaultDataPath = "/var/lib/miren" + +// Options say where to look. The zero value probes the real host with miren's +// default data directory. +type Options struct { + // Root is the filesystem to read, normally "" for "/". Tests point this + // at a fixture directory. + Root string + + // DataPath is miren's data directory, holding the install record. + // Defaults to DefaultDataPath. + DataPath string + + // SearchPath holds extra directories to look for lbdctl in, ahead of + // PATH. The server's release directory belongs here: it is prepended to + // containerd's PATH but not to miren's own. + SearchPath []string +} + +// systemReleasePath is searched for lbdctl by default. It is prepended to +// containerd's PATH but not to miren's own, so without this every caller would +// have to remember to add it. +const systemReleasePath = "/var/lib/miren/release" + +func (o Options) root() string { + if o.Root == "" { + return "/" + } + return o.Root +} + +// searchPath returns the directories to look for lbdctl in. Callers that name +// none still get the release directory, so every caller agrees on whether +// lbdctl is present. +// +// This builds a new slice rather than appending to o.SearchPath, which would +// write into the caller's backing array whenever it has spare capacity. +func (o Options) searchPath() []string { + return slices.Concat(o.SearchPath, []string{systemReleasePath}) +} + +func (o Options) dataPath() string { + if o.DataPath == "" { + return DefaultDataPath + } + return o.DataPath +} + +// findLbdctl locates the lbdctl binary, checking the caller's directories +// before falling back to PATH. +func findLbdctl(root string, searchPath []string) string { + for _, dir := range searchPath { + if dir == "" { + continue + } + path := filepath.Join(dir, "lbdctl") + if info, err := os.Stat(filepath.Join(root, path)); err == nil && !info.IsDir() { + return path + } + } + + // PATH only makes sense against the real filesystem. + if root != "/" { + return "" + } + if path, err := exec.LookPath("lbdctl"); err == nil { + return path + } + return "" +} + +// Probe reports what miren knows about lbd on this host. It never modifies +// anything, so it is safe to call without root. +func Probe(opts Options) (Status, error) { + root := opts.root() + + host, err := DetectHost(root) + if err != nil { + return Status{}, err + } + + marker, err := readMarker(opts.dataPath()) + if err != nil { + return Status{}, err + } + + installed := false + if _, err := os.Stat(filepath.Join(root, modulePath(host.KernelRelease))); err == nil { + installed = true + } + + _, ctlErr := os.Stat(filepath.Join(root, ControlDevice)) + + return Status{ + Host: host, + Loaded: isModuleLoaded(root, ModuleName), + ControlDevicePresent: ctlErr == nil, + ModuleInstalled: installed, + LbdctlPath: findLbdctl(root, opts.searchPath()), + Marker: marker, + EmbeddedVersion: lbdsrc.Version(), + }, nil +} + +// Available reports whether accelerator mode can run right now: the module is +// loaded, its control device exists, and lbdctl is there to drive it. +// +// This is the check that decides a disk's mode, so it deliberately does not +// read the install record and cannot fail -- unlike Probe, which is for +// explaining the situation to a person. lbdctl being on PATH is not enough on +// its own: miren installs lbdctl alongside the module, so a host that has the +// binary but no loaded module would otherwise be sent down the accelerator +// path and fail at attach time. +func Available(opts Options) bool { + root := opts.root() + + if !isModuleLoaded(root, ModuleName) { + return false + } + if _, err := os.Stat(filepath.Join(root, ControlDevice)); err != nil { + return false + } + return findLbdctl(root, opts.searchPath()) != "" +} diff --git a/pkg/lbdmod/source.go b/pkg/lbdmod/source.go new file mode 100644 index 000000000..9912796a2 --- /dev/null +++ b/pkg/lbdmod/source.go @@ -0,0 +1,50 @@ +package lbdmod + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + lbdsrc "miren.dev/runtime/third_party/lbd" +) + +// SourceVersion reports the miren.dev/lbd version this binary carries. +func SourceVersion() string { + return lbdsrc.Version() +} + +// materializeSource writes the embedded module source into dir, which the +// builder container then mounts. The kernel build system writes its object +// files next to the source, so this has to be a real writable directory rather +// than a read-only mount of something we already have. +func materializeSource(dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating the build directory %s: %w", dir, err) + } + + return fs.WalkDir(lbdsrc.FS, "src", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel("src", path) + if err != nil { + return err + } + target := filepath.Join(dir, rel) + + if d.IsDir() { + return os.MkdirAll(target, 0755) + } + + data, err := lbdsrc.FS.ReadFile(path) + if err != nil { + return fmt.Errorf("reading embedded %s: %w", path, err) + } + if err := os.WriteFile(target, data, 0644); err != nil { + return fmt.Errorf("writing %s: %w", target, err) + } + return nil + }) +} diff --git a/pkg/lbdmod/source_test.go b/pkg/lbdmod/source_test.go new file mode 100644 index 000000000..d002ee771 --- /dev/null +++ b/pkg/lbdmod/source_test.go @@ -0,0 +1,64 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSourceVersionMatchesTheCheckedInTree(t *testing.T) { + // hack/sync-lbd-src.sh writes VERSION beside the source it copied, and CI + // checks the tree against go.mod. If this is empty the embed is broken. + version := SourceVersion() + require.NotEmpty(t, version) + assert.NotEqual(t, "unknown", version) +} + +func TestMaterializeSourceWritesABuildableTree(t *testing.T) { + dir := filepath.Join(t.TempDir(), "src") + require.NoError(t, materializeSource(dir)) + + // Everything the kernel build and lbdctl need, including the vendored LZ4 + // in its subdirectory. + for _, name := range []string{ + "Makefile", + "dkms.conf", + "lbd_main.c", + "lbd_qcow2.c", + "lbdctl.c", + "lbd.h", + "lz4_kcompat.h", + "lz4/lz4.c", + "lz4/lz4.h", + } { + info, err := os.Stat(filepath.Join(dir, name)) + require.NoError(t, err, "missing %s", name) + assert.Positive(t, info.Size(), "%s is empty", name) + } + + // The build system writes object files next to the source, so the tree has + // to be writable -- it cannot be a read-only mount of the embed. + require.NoError(t, os.WriteFile(filepath.Join(dir, "lbd.o"), []byte("x"), 0644)) + + // The Makefile must still carry the kernel-version probe that lets the + // module build against recent kernels. + makefile, err := os.ReadFile(filepath.Join(dir, "Makefile")) + require.NoError(t, err) + assert.Contains(t, string(makefile), "LBD_RENAME_PARENT") + assert.Contains(t, string(makefile), "obj-m := lbd.o") +} + +func TestMaterializeSourceIsRepeatable(t *testing.T) { + dir := filepath.Join(t.TempDir(), "src") + require.NoError(t, materializeSource(dir)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "lbd_main.c"), []byte("clobbered"), 0644)) + + // A retry after a failed build has to restore the source it overwrote. + require.NoError(t, materializeSource(dir)) + data, err := os.ReadFile(filepath.Join(dir, "lbd_main.c")) + require.NoError(t, err) + assert.NotEqual(t, "clobbered", string(data)) +} diff --git a/pkg/lbdmod/status.go b/pkg/lbdmod/status.go new file mode 100644 index 000000000..bb2f2c442 --- /dev/null +++ b/pkg/lbdmod/status.go @@ -0,0 +1,213 @@ +package lbdmod + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + // ModuleName is what the module is called once loaded, and the name + // modprobe takes. + ModuleName = "lbd" + + // ControlDevice is the misc device the module registers. Its presence is + // the only trustworthy proof that the module is loaded and working -- + // lbdctl being on PATH proves nothing. + ControlDevice = "/dev/lbd-control" + + // markerName records what was built and for which kernel, so a later boot + // can tell a stale module from a missing one. + markerName = "installed.json" +) + +// Marker is the record left behind by a successful install. Its presence means +// this host has opted into accelerator mode, which is what licenses miren to +// rebuild the module unattended after a kernel upgrade. +type Marker struct { + // LbdVersion is the miren.dev/lbd version the source came from. + LbdVersion string `json:"lbd_version"` + + // KernelRelease is the kernel the module was built for. A module built + // for one release will not load on another. + KernelRelease string `json:"kernel_release"` + + // ModulePath is where the built module was installed. + ModulePath string `json:"module_path"` + + // LbdctlPath is where the built lbdctl was installed. + LbdctlPath string `json:"lbdctl_path"` + + // BuiltAt is when the build finished. + BuiltAt time.Time `json:"built_at"` +} + +// Status is what miren knows about lbd on this host. +type Status struct { + // Host is the machine as detected. + Host Host + + // Loaded is true when the module is in /proc/modules. + Loaded bool + + // ControlDevicePresent is true when /dev/lbd-control exists. Together + // with Loaded this is the real availability test. + ControlDevicePresent bool + + // ModuleInstalled is true when a built module exists for the running + // kernel, whether or not it is currently loaded. + ModuleInstalled bool + + // LbdctlPath is where lbdctl was found, or empty. + LbdctlPath string + + // Marker is the record of the last successful install, or nil if this + // host has never installed the module. + Marker *Marker + + // EmbeddedVersion is the miren.dev/lbd version this binary carries. + EmbeddedVersion string +} + +// Available reports whether accelerator mode can actually run right now: the +// module is loaded, its control device is there, and lbdctl exists to drive it. +func (s Status) Available() bool { + return s.Loaded && s.ControlDevicePresent && s.LbdctlPath != "" +} + +// Stale reports whether this host installed the module before but what is on +// disk no longer fits -- almost always because the kernel was upgraded, but +// also when miren itself now carries a newer lbd. Callers use this to decide +// whether to rebuild without being asked. +func (s Status) Stale() bool { + return s.staleReason() != "" +} + +// staleReason names what stopped fitting, phrased to follow "lbd is loaded +// but ...". It is empty when nothing is stale. +func (s Status) staleReason() string { + if s.Marker == nil { + return "" + } + switch { + case s.Marker.KernelRelease != s.Host.KernelRelease: + return fmt.Sprintf("it was built for kernel %s, not the %s this host is running", + s.Marker.KernelRelease, s.Host.KernelRelease) + case s.EmbeddedVersion != "" && s.Marker.LbdVersion != s.EmbeddedVersion: + return fmt.Sprintf("miren now bundles lbd %s and the installed module is %s", + s.EmbeddedVersion, s.Marker.LbdVersion) + case !s.ModuleInstalled: + return fmt.Sprintf("its module file %s is gone", s.Marker.ModulePath) + } + return "" +} + +// Explain renders the status as a sentence for logs and CLI output. +func (s Status) Explain() string { + // Concrete faults come first. A module can be loaded without miren having + // installed it -- by hand, or by a distro package -- and in that case the + // specific problem is more useful than "not installed". + stale := s.staleReason() + + switch { + // A loaded module can still be the wrong one -- most often after miren + // was upgraded to a build carrying a newer lbd. Saying only that it is + // loaded would read as healthy while a rebuild is pending. + case s.Available() && stale != "": + return "lbd is loaded but " + stale + case s.Available(): + return fmt.Sprintf("lbd %s is loaded for kernel %s", s.markerVersion(), s.Host.KernelRelease) + case s.Loaded && !s.ControlDevicePresent: + return fmt.Sprintf("lbd is loaded but %s is missing", ControlDevice) + case s.Loaded && s.LbdctlPath == "": + return "lbd is loaded but lbdctl is missing" + case s.Marker == nil: + return "lbd is not installed" + case stale != "": + return "lbd is not usable: " + stale + case !s.Loaded: + return "lbd is installed but not loaded" + default: + return "lbd is installed but lbdctl is missing" + } +} + +func (s Status) markerVersion() string { + if s.Marker != nil && s.Marker.LbdVersion != "" { + return s.Marker.LbdVersion + } + return s.EmbeddedVersion +} + +// isModuleLoaded reports whether the named module appears in /proc/modules. +func isModuleLoaded(root, name string) bool { + content, err := os.ReadFile(filepath.Join(root, "proc/modules")) + if err != nil { + return false + } + for line := range strings.SplitSeq(string(content), "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && fields[0] == name { + return true + } + } + return false +} + +// modulePath is where a built module for the given kernel lives. "extra" is the +// conventional home for out-of-tree modules and is on depmod's search path. +func modulePath(release string) string { + return filepath.Join("/lib/modules", release, "extra", ModuleName+".ko") +} + +// markerPath is where the install record lives, under miren's data directory. +func markerPath(dataPath string) string { + return filepath.Join(dataPath, "lbd", markerName) +} + +// readMarker loads the install record, returning nil when the host has never +// installed the module. A corrupt marker is reported as an error rather than +// silently treated as absent, since discarding it would strand a module that is +// actually installed. +func readMarker(dataPath string) (*Marker, error) { + data, err := os.ReadFile(markerPath(dataPath)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading the lbd install record: %w", err) + } + + var m Marker + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("the lbd install record at %s is corrupt: %w", markerPath(dataPath), err) + } + return &m, nil +} + +// writeMarker records a successful install. +func writeMarker(dataPath string, m Marker) error { + path := markerPath(dataPath) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("encoding the lbd install record: %w", err) + } + return os.WriteFile(path, append(data, '\n'), 0644) +} + +// removeMarker forgets that lbd was ever installed, so later boots stop +// rebuilding it. +func removeMarker(dataPath string) error { + err := os.Remove(markerPath(dataPath)) + if os.IsNotExist(err) { + return nil + } + return err +} diff --git a/pkg/lbdmod/status_test.go b/pkg/lbdmod/status_test.go new file mode 100644 index 000000000..8f4a92649 --- /dev/null +++ b/pkg/lbdmod/status_test.go @@ -0,0 +1,264 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testRelease = "6.8.0-51-generic" + +// ubuntuRoot builds a fixture filesystem for a plausible Ubuntu host with +// kernel headers installed and no lbd anywhere. +func ubuntuRoot(t *testing.T) string { + t.Helper() + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", testRelease+"\n") + writeFile(t, root, "proc/version", + "Linux version "+testRelease+" (buildd@lcy02) (x86_64-linux-gnu-gcc-13 (Ubuntu 13.3.0) 13.3.0) #52-Ubuntu SMP\n") + writeFile(t, root, "etc/os-release", "ID=ubuntu\nID_LIKE=debian\n") + writeFile(t, root, "lib/modules/"+testRelease+"/build/Makefile", "# kernel build tree\n") + writeFile(t, root, "proc/modules", "loop 69632 0 - Live 0x0000000000000000\n") + return root +} + +func TestIsModuleLoaded(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "proc/modules", `nf_tables 356352 0 - Live 0x0000000000000000 +lbd 65536 1 - Live 0x0000000000000000 +loop 69632 0 - Live 0x0000000000000000 +`) + assert.True(t, isModuleLoaded(root, "lbd")) + assert.True(t, isModuleLoaded(root, "loop")) + assert.False(t, isModuleLoaded(root, "lbdctl")) + assert.False(t, isModuleLoaded(t.TempDir(), "lbd")) +} + +func TestProbeOnAHostWithoutLbd(t *testing.T) { + root := ubuntuRoot(t) + + status, err := Probe(Options{Root: root, DataPath: t.TempDir()}) + require.NoError(t, err) + + assert.False(t, status.Available()) + assert.False(t, status.Loaded) + assert.False(t, status.ModuleInstalled) + assert.Nil(t, status.Marker) + assert.False(t, status.Stale(), "a host that never installed lbd has nothing to rebuild") + assert.Equal(t, "lbd is not installed", status.Explain()) + assert.NotEmpty(t, status.EmbeddedVersion, "the binary should know which lbd it carries") +} + +func TestProbeOnAHealthyHost(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{ + Root: root, + DataPath: dataPath, + SearchPath: []string{"/usr/local/bin"}, + }) + require.NoError(t, err) + + assert.True(t, status.Available()) + assert.False(t, status.Stale()) + assert.Equal(t, "/usr/local/bin/lbdctl", status.LbdctlPath) + assert.Contains(t, status.Explain(), "is loaded for kernel "+testRelease) +} + +func TestLbdctlPresenceAloneIsNotAvailability(t *testing.T) { + // The bug this guards: shipping lbdctl in the release bundle used to be + // enough to select accelerator mode, even with no module loaded. + root := ubuntuRoot(t) + writeFile(t, root, "usr/local/bin/lbdctl", "") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir(), SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + assert.NotEmpty(t, status.LbdctlPath) + assert.False(t, status.Available()) +} + +func TestLoadedWithoutControlDeviceIsNotAvailable(t *testing.T) { + root := ubuntuRoot(t) + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, "usr/local/bin/lbdctl", "") + + status, err := Probe(Options{Root: root, DataPath: t.TempDir(), SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + assert.True(t, status.Loaded) + assert.False(t, status.ControlDevicePresent) + assert.False(t, status.Available()) + assert.Contains(t, status.Explain(), ControlDevice+" is missing") +} + +func TestStaleAfterAKernelUpgrade(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + // The marker remembers the kernel the module was built for; the host is + // now running a different one. + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: "6.8.0-45-generic", + ModulePath: modulePath("6.8.0-45-generic"), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + + assert.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "built for kernel 6.8.0-45-generic") + assert.Contains(t, status.Explain(), testRelease+" this host is running") +} + +func TestALoadedButStaleModuleDoesNotReadAsHealthy(t *testing.T) { + // Upgrading miren to a build carrying a newer lbd leaves the old module + // loaded and working. Reporting only "is loaded" would read as healthy + // while a rebuild is pending. + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 1 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "v0.0.0-20250101000000-000000000000", + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath, SearchPath: []string{"/usr/local/bin"}}) + require.NoError(t, err) + + require.True(t, status.Available()) + require.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "miren now bundles lbd") + assert.Contains(t, status.Explain(), "v0.0.0-20250101000000-000000000000") +} + +func TestStaleWhenMirenCarriesANewerLbd(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, modulePath(testRelease), "") + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "v0.0.0-20250101000000-000000000000", + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + assert.True(t, status.Stale()) +} + +func TestStaleWhenTheModuleFileWentAway(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + BuiltAt: time.Now(), + })) + + status, err := Probe(Options{Root: root, DataPath: dataPath}) + require.NoError(t, err) + assert.True(t, status.Stale()) + assert.Contains(t, status.Explain(), "is gone") +} + +func TestMarkerRoundTrip(t *testing.T) { + dataPath := t.TempDir() + + marker, err := readMarker(dataPath) + require.NoError(t, err) + assert.Nil(t, marker, "no record means the host never installed lbd") + + want := Marker{ + LbdVersion: "v0.0.0-20260824210626-be4cec661034", + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + LbdctlPath: "/usr/local/bin/lbdctl", + BuiltAt: time.Now().UTC().Truncate(time.Second), + } + require.NoError(t, writeMarker(dataPath, want)) + + got, err := readMarker(dataPath) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, want, *got) + + require.NoError(t, removeMarker(dataPath)) + got, err = readMarker(dataPath) + require.NoError(t, err) + assert.Nil(t, got) + + // Removing an absent record is not an error. + require.NoError(t, removeMarker(dataPath)) +} + +func TestCorruptMarkerIsAnError(t *testing.T) { + dataPath := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dataPath, "lbd"), 0755)) + require.NoError(t, os.WriteFile(markerPath(dataPath), []byte("{not json"), 0644)) + + // Reported rather than treated as absent: silently discarding it would + // strand a module that really is installed. + _, err := readMarker(dataPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "corrupt") + + _, err = Probe(Options{Root: ubuntuRoot(t), DataPath: dataPath}) + require.Error(t, err) +} + +func TestFindLbdctlPrefersTheSearchPath(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "var/lib/miren/release/lbdctl", "") + + assert.Equal(t, "/var/lib/miren/release/lbdctl", + findLbdctl(root, []string{"", "/nowhere", "/var/lib/miren/release"})) + assert.Empty(t, findLbdctl(root, []string{"/nowhere"})) +} + +func TestSearchPathDoesNotWriteIntoTheCallersSlice(t *testing.T) { + // A slice with spare capacity is what makes append dangerous: it writes + // into the caller's backing array instead of allocating. + caller := make([]string, 1, 4) + caller[0] = "/opt/bin" + + opts := Options{SearchPath: caller} + got := opts.searchPath() + + assert.Equal(t, []string{"/opt/bin", systemReleasePath}, got) + assert.Equal(t, []string{"/opt/bin"}, caller, "the caller's slice must be untouched") + + // Extending the caller's slice writes into its spare capacity, which is + // the array searchPath would have appended into. What it returned must not + // move. + extended := append(caller, "/clobbered") + require.Equal(t, "/clobbered", extended[1]) + assert.Equal(t, []string{"/opt/bin", systemReleasePath}, got, + "the returned slice must not alias the caller's array") +} From cb3099f8bb0be276f2a794950abd4e8a6c5ae297 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 22:10:44 -0700 Subject: [PATCH 03/15] Prove lbd is usable rather than trusting lbdctl on PATH Three places decided whether to use accelerator mode, and all three did it the same way: look for lbdctl on PATH. That was a reasonable proxy while nothing installed lbdctl, since its presence meant someone had been through the whole manual build. It stops being true the moment miren installs lbdctl itself, which it now does alongside the module. A node would find the binary, choose accelerator mode, and then fail at attach time with no module loaded. All three now ask the same question, and it is the real one: is the module in /proc/modules, is its control device there, and is lbdctl around to drive it. EnsureLbdDevices gets the same treatment, since modprobe exiting zero does not mean the module came up either. This follows what EnsureLoopDevices already does for loop devices, which proves support with a real ioctl rather than inferring it. The CLI copy also honours MIREN_DISK_MODE=universal, which it silently ignored before. --- cli/commands/disk_resolver.go | 12 +++++++++--- components/diskio/disk_ops_linux.go | 24 ++++++++++++++++-------- controllers/disk/disk_controller.go | 9 ++++++--- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cli/commands/disk_resolver.go b/cli/commands/disk_resolver.go index 63533b40a..0dcdd126d 100644 --- a/cli/commands/disk_resolver.go +++ b/cli/commands/disk_resolver.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" "strings" @@ -14,6 +13,7 @@ import ( "miren.dev/runtime/api/storage/storage_v1alpha" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/lbdmod" "miren.dev/runtime/pkg/snapshot" ) @@ -282,10 +282,16 @@ func (r *entityDiskResolver) FindLeases(ctx context.Context, diskID string) ([]s } func detectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { - if mode := os.Getenv("MIREN_DISK_MODE"); mode == "accelerator" { + switch os.Getenv("MIREN_DISK_MODE") { + case "accelerator": return storage_v1alpha.VM_ACCELERATOR + case "universal": + return storage_v1alpha.VM_UNIVERSAL } - if _, err := exec.LookPath("lbdctl"); err == nil { + + // Auto-detect. This has to agree with controllers/disk's detectDiskMode, + // so both ask lbdmod rather than each testing for lbdctl on PATH. + if lbdmod.Available(lbdmod.Options{SearchPath: []string{FindReleasePath()}}) { return storage_v1alpha.VM_ACCELERATOR } return storage_v1alpha.VM_UNIVERSAL diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 0ea0c5dc8..5fabc9cdd 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -17,6 +17,7 @@ import ( "unsafe" "golang.org/x/sys/unix" + "miren.dev/runtime/pkg/lbdmod" ) const ( @@ -633,19 +634,26 @@ func ensureLoopDeviceNode(log *slog.Logger, index int) error { return nil } -// EnsureLbdDevices checks if the lbd kernel module and lbdctl are available. +// EnsureLbdDevices loads the lbd kernel module and proves it is usable. +// +// modprobe's exit code is not the test: a module can be absent, or present but +// wedged, and lbdctl can be installed on a host whose module never loaded. +// What settles it is the same probe accelerator mode itself relies on, so a +// node never selects accelerator mode it cannot serve. func EnsureLbdDevices(log *slog.Logger) error { - // Try modprobe lbd - if out, err := exec.Command("modprobe", "lbd").CombinedOutput(); err != nil { - log.Warn("modprobe lbd failed", "error", err, "output", string(out)) + if out, err := exec.Command("modprobe", lbdmod.ModuleName).CombinedOutput(); err != nil { + log.Debug("modprobe lbd failed", "error", err, "output", strings.TrimSpace(string(out))) } - // Check that lbdctl is in PATH - if _, err := exec.LookPath("lbdctl"); err != nil { - return fmt.Errorf("lbdctl not found in PATH: %w", err) + status, err := lbdmod.Probe(lbdmod.Options{}) + if err != nil { + return err + } + if !status.Available() { + return errors.New(status.Explain()) } - log.Info("lbd devices available") + log.Info("lbd devices available", "kernel", status.Host.KernelRelease, "lbdctl", status.LbdctlPath) return nil } diff --git a/controllers/disk/disk_controller.go b/controllers/disk/disk_controller.go index e062decdd..0d0940370 100644 --- a/controllers/disk/disk_controller.go +++ b/controllers/disk/disk_controller.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "os/exec" "strings" compute "miren.dev/runtime/api/compute/compute_v1alpha" @@ -13,6 +12,7 @@ import ( "miren.dev/runtime/pkg/controller" "miren.dev/runtime/pkg/entity" "miren.dev/runtime/pkg/idgen" + "miren.dev/runtime/pkg/lbdmod" ) // detectDiskMode determines which disk I/O mode to use. @@ -26,8 +26,11 @@ func detectDiskMode(configured string) storage_v1alpha.DiskMode { return storage_v1alpha.ACCELERATOR } - // Auto-detect: use accelerator mode if lbd is available - if _, err := exec.LookPath("lbdctl"); err == nil { + // Auto-detect: use accelerator mode only if the lbd module is actually + // loaded and drivable. lbdctl on PATH is not enough -- miren installs it + // alongside the module, so its presence says nothing about whether the + // module loaded. + if lbdmod.Available(lbdmod.Options{}) { return storage_v1alpha.ACCELERATOR } From fcef04c8b23b114d5c27bcc8cd7170af93ec0737 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 3 Sep 2026 23:09:45 -0700 Subject: [PATCH 04/15] address review: wrap the right errors, and agree on one host lookup Both reviewers caught kernelRelease wrapping the wrong error: when uname fails it reported the earlier procfs read failure instead. The same function could also wrap a nil, when the file exists but is empty, which renders as %!w() and tells an operator nothing. The containerd serving check had the same nil-wrapping shape. The larger one is that the three places deciding disk mode did not agree after all. The CLI searched the release directory it resolved through $HOME while the disk controller searched only the system one, so a host with lbdctl under ~/.miren/release would have had the CLI choose accelerator and the controller choose universal. They all go through lbdmod.HostOptions now, which searches the system release directory and PATH for everyone. Nothing resolves a per-user path, since the CLI and the server run as different users and would resolve it differently. setupLbd was also reading the install record from the package default rather than the runner data path it was given, so a host with a non-default data path looked like one that never installed lbd and would never have rebuilt after a kernel upgrade. Also: bound the startup modprobe, write the install record through a rename so a torn write cannot wedge later probes, fail the builder image publish on lookup errors that are not NOT_FOUND, log a leftover build container at Warn, and document the ten-minute rebuild cap. --- .github/workflows/release.yml | 14 +++++++++++--- cli/commands/disk_accelerator.go | 22 ++++++++-------------- cli/commands/disk_resolver.go | 2 +- components/diskio/disk_ops_darwin.go | 2 +- components/runner/lbd.go | 11 ++++++++--- controllers/disk/disk_controller.go | 2 +- docs/docs/disk-accelerator.md | 5 +++++ pkg/lbdmod/build_test.go | 5 ----- pkg/lbdmod/ctrbuild/ctrbuild.go | 2 +- pkg/lbdmod/kernel.go | 15 ++++++++++----- pkg/lbdmod/kernel_test.go | 21 +++++++++++++++++++++ pkg/lbdmod/probe.go | 17 +++++++++++++++++ pkg/lbdmod/status.go | 14 +++++++++++++- pkg/lbdmod/status_test.go | 21 +++++++++++++++++++++ 14 files changed, 118 insertions(+), 35 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46b9cae02..4afabe7cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -513,12 +513,20 @@ jobs: echo "image=$IMAGE" >> $GITHUB_OUTPUT # Republishing an existing tag would move a pinned reference under - # already-deployed clusters, so only a new tag is ever pushed. - if gcloud artifacts docker images describe "$IMAGE" >/dev/null 2>&1; then + # already-deployed clusters, so only a genuinely absent tag is + # published. An auth or transient failure must not read as absent: + # that is how a lookup blip turns into a moved tag, so anything + # other than a confirmed NOT_FOUND fails the job. + if lookup=$(gcloud artifacts docker images describe "$IMAGE" 2>&1); then echo "$IMAGE already exists; nothing to publish" echo "publish=false" >> $GITHUB_OUTPUT - else + elif printf '%s' "$lookup" | grep -qiE 'NOT_FOUND|was not found|does not exist'; then + echo "$IMAGE does not exist yet; publishing" echo "publish=true" >> $GITHUB_OUTPUT + else + echo "could not determine whether $IMAGE exists:" >&2 + printf '%s\n' "$lookup" >&2 + exit 1 fi - name: Set up Docker Buildx diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go index 461f57492..5ed627bb0 100644 --- a/cli/commands/disk_accelerator.go +++ b/cli/commands/disk_accelerator.go @@ -12,23 +12,13 @@ import ( "miren.dev/runtime/pkg/lbdmod/ctrbuild" ) -// acceleratorOptions builds the lookup paths for lbd on this host. The release -// directory is included because it is prepended to containerd's PATH, not -// miren's, so a bundled lbdctl would otherwise be invisible here. -func acceleratorOptions(dataPath string) lbdmod.Options { - return lbdmod.Options{ - DataPath: dataPath, - SearchPath: []string{FindReleasePath()}, - } -} - // DiskAcceleratorStatus reports whether accelerator mode can run on this host. // It only reads, so it does not need root. func DiskAcceleratorStatus(ctx *Context, opts struct { FormatOptions DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` }) error { - status, err := lbdmod.Probe(acceleratorOptions(opts.DataPath)) + status, err := lbdmod.Probe(lbdmod.HostOptions(opts.DataPath)) if err != nil { return err } @@ -90,7 +80,7 @@ func DiskAcceleratorInstall(ctx *Context, opts struct { installer := &lbdmod.Installer{ Log: ctx.Log, Builder: ctrbuild.New(cc, ctx.Log), - Options: acceleratorOptions(opts.DataPath), + Options: lbdmod.HostOptions(opts.DataPath), Image: opts.Image, } @@ -113,7 +103,7 @@ func DiskAcceleratorUninstall(ctx *Context, opts struct { }) error { installer := &lbdmod.Installer{ Log: ctx.Log, - Options: acceleratorOptions(opts.DataPath), + Options: lbdmod.HostOptions(opts.DataPath), } ctx.Begin("Removing the lbd kernel module") @@ -140,10 +130,14 @@ func dialContainerd(ctx context.Context, socket string) (*containerd.Client, err // containerd.New does not connect, so without this a dead socket would // surface much later as an opaque failure to pull the builder image. serving, err := cc.IsServing(ctx) - if err != nil || !serving { + if err != nil { cc.Close() return nil, fmt.Errorf("containerd at %s is not responding, which the builder needs: %w", socket, err) } + if !serving { + cc.Close() + return nil, fmt.Errorf("containerd at %s answered but is not serving, which the builder needs", socket) + } return cc, nil } diff --git a/cli/commands/disk_resolver.go b/cli/commands/disk_resolver.go index 0dcdd126d..612d55b36 100644 --- a/cli/commands/disk_resolver.go +++ b/cli/commands/disk_resolver.go @@ -291,7 +291,7 @@ func detectVolumeMode() storage_v1alpha.DiskVolumeVolumeMode { // Auto-detect. This has to agree with controllers/disk's detectDiskMode, // so both ask lbdmod rather than each testing for lbdctl on PATH. - if lbdmod.Available(lbdmod.Options{SearchPath: []string{FindReleasePath()}}) { + if lbdmod.Available(lbdmod.HostOptions("")) { return storage_v1alpha.VM_ACCELERATOR } return storage_v1alpha.VM_UNIVERSAL diff --git a/components/diskio/disk_ops_darwin.go b/components/diskio/disk_ops_darwin.go index 2c8050b22..1f184dbc0 100644 --- a/components/diskio/disk_ops_darwin.go +++ b/components/diskio/disk_ops_darwin.go @@ -108,7 +108,7 @@ func EnsureLoopDevices(_ *slog.Logger) error { return fmt.Errorf("loop devices not supported on darwin") } -func EnsureLbdDevices(_ *slog.Logger) error { +func EnsureLbdDevices(_ context.Context, _ *slog.Logger) error { return fmt.Errorf("lbd not supported on darwin") } diff --git a/components/runner/lbd.go b/components/runner/lbd.go index cf1db68c6..6d6366bd8 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -38,14 +38,19 @@ const rebuildTimeout = 10 * time.Minute // // Neither failing nor timing out is fatal. Universal mode works everywhere, so // the worst case is slower disks, not a runner that will not start. -func setupLbd(ctx context.Context, cc *containerd.Client, log *slog.Logger) { - if err := diskio.EnsureLbdDevices(log); err == nil { +func setupLbd(ctx context.Context, cc *containerd.Client, dataPath string, log *slog.Logger) { + if err := diskio.EnsureLbdDevices(ctx, log); err == nil { return } + // dataPath has to be the runner's own, not the package default: the + // install record lives under it, and reading it from the wrong place + // would make a host that installed lbd look like one that never did, so + // the rebuild after a kernel upgrade would never fire. installer := &lbdmod.Installer{ Log: log, Builder: ctrbuild.New(cc, log), + Options: lbdmod.HostOptions(dataPath), } ctx, cancel := context.WithTimeout(ctx, rebuildTimeout) @@ -69,7 +74,7 @@ func setupLbd(ctx context.Context, cc *containerd.Client, log *slog.Logger) { return } - if err := diskio.EnsureLbdDevices(log); err != nil { + if err := diskio.EnsureLbdDevices(ctx, log); err != nil { log.Warn("rebuilt the lbd kernel module but it is still not usable", "error", err) } } diff --git a/controllers/disk/disk_controller.go b/controllers/disk/disk_controller.go index 0d0940370..b7f1220f1 100644 --- a/controllers/disk/disk_controller.go +++ b/controllers/disk/disk_controller.go @@ -30,7 +30,7 @@ func detectDiskMode(configured string) storage_v1alpha.DiskMode { // loaded and drivable. lbdctl on PATH is not enough -- miren installs it // alongside the module, so its presence says nothing about whether the // module loaded. - if lbdmod.Available(lbdmod.Options{}) { + if lbdmod.Available(lbdmod.HostOptions("")) { return storage_v1alpha.ACCELERATOR } diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md index 13ad44dac..6a7eb655b 100644 --- a/docs/docs/disk-accelerator.md +++ b/docs/docs/disk-accelerator.md @@ -105,6 +105,11 @@ This only happens on hosts that installed the module in the first place. A host that never turned accelerator mode on never pays for an unattended compile at startup. +The startup rebuild is capped at ten minutes. If it runs over, the server logs a +warning, carries on booting, and the node stays on loop devices until you run +`install` yourself. Accelerator mode is a speed-up, so a build that will not +finish must never hold up a node coming back after a reboot. + Until the module is back, disks fall back to universal mode. Nothing breaks; they are just slower. diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 492f51c73..9d0c78fb5 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -258,11 +258,6 @@ func TestBuildHandsTheBuilderSourceAndHeaders(t *testing.T) { assert.Contains(t, builder.spec.Env, "KERNEL_HEADERS=/lib/modules/"+testRelease+"/build") assert.Contains(t, builder.spec.Env, "HOST_DISTRO_ID=ubuntu") assert.Contains(t, builder.spec.Env, "HOST_DISTRO_LIKE=debian") - // The build runs unprivileged: compiling against read-only bind mounts - // needs no extra capabilities, and this container runs on every node with - // accelerator mode on. - assert.False(t, builder.spec.HostNetwork, - "a build against host headers should reach nothing") } func TestBuildRejectsABuilderThatProducedNothing(t *testing.T) { diff --git a/pkg/lbdmod/ctrbuild/ctrbuild.go b/pkg/lbdmod/ctrbuild/ctrbuild.go index e03ea004e..d1c57795d 100644 --- a/pkg/lbdmod/ctrbuild/ctrbuild.go +++ b/pkg/lbdmod/ctrbuild/ctrbuild.go @@ -62,7 +62,7 @@ func (b *Builder) Build(ctx context.Context, spec lbdmod.BuildSpec) error { // A previous run that died before its own cleanup leaves the container // behind and its name taken. if existing, err := b.cc.LoadContainer(ctx, spec.Name); err == nil { - b.log.Info("removing a container left by an earlier build", "container", spec.Name) + b.log.Warn("removing a container left by an earlier build", "container", spec.Name) b.removeContainer(ctx, existing) } diff --git a/pkg/lbdmod/kernel.go b/pkg/lbdmod/kernel.go index 1e76f7576..6e73581a5 100644 --- a/pkg/lbdmod/kernel.go +++ b/pkg/lbdmod/kernel.go @@ -1,6 +1,7 @@ package lbdmod import ( + "errors" "fmt" "os" "os/exec" @@ -185,24 +186,28 @@ func parseOSRelease(content string) (id string, like []string) { // shelling out, and falls back to uname(1) only for a real host, since a // fixture root has no process to ask. func kernelRelease(root string) (string, error) { - data, err := os.ReadFile(filepath.Join(root, "proc/sys/kernel/osrelease")) - if err == nil { + path := filepath.Join(root, "proc/sys/kernel/osrelease") + data, readErr := os.ReadFile(path) + if readErr == nil { if release := strings.TrimSpace(string(data)); release != "" { return release, nil } + // The file is there but says nothing. Name that, so the error below + // never wraps a nil. + readErr = fmt.Errorf("%s is empty", path) } if root != "/" { - return "", fmt.Errorf("no kernel release under %s: %w", root, err) + return "", fmt.Errorf("no kernel release under %s: %w", root, readErr) } out, unameErr := exec.Command("uname", "-r").Output() if unameErr != nil { - return "", fmt.Errorf("could not determine the kernel release: %w", err) + return "", fmt.Errorf("could not determine the kernel release: %w", unameErr) } release := strings.TrimSpace(string(out)) if release == "" { - return "", fmt.Errorf("could not determine the kernel release: uname -r said nothing") + return "", errors.New("could not determine the kernel release: uname -r said nothing") } return release, nil } diff --git a/pkg/lbdmod/kernel_test.go b/pkg/lbdmod/kernel_test.go index 65e7623bd..d2fc91467 100644 --- a/pkg/lbdmod/kernel_test.go +++ b/pkg/lbdmod/kernel_test.go @@ -154,3 +154,24 @@ func TestHeaderPackageUnknownDistro(t *testing.T) { assert.Empty(t, h.HeaderPackage()) assert.Equal(t, "install the kernel headers for 6.8.0-51 and try again", h.InstallHint()) } + +func TestKernelReleaseErrorNamesTheRealCause(t *testing.T) { + // A fixture root with no procfs file at all: the error has to carry the + // read failure, not a nil wrapped by %w. + _, err := kernelRelease(t.TempDir()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no kernel release") + assert.NotContains(t, err.Error(), "%!w", "an error was wrapped that was nil") +} + +func TestKernelReleaseRejectsAnEmptyOsrelease(t *testing.T) { + // The file exists but says nothing. Before, err was nil here and %w + // rendered as %!w(), telling the operator nothing. + root := t.TempDir() + writeFile(t, root, "proc/sys/kernel/osrelease", "\n") + + _, err := kernelRelease(root) + require.Error(t, err) + assert.Contains(t, err.Error(), "is empty") + assert.NotContains(t, err.Error(), "%!w") +} diff --git a/pkg/lbdmod/probe.go b/pkg/lbdmod/probe.go index ae5ba7843..d4dfd2649 100644 --- a/pkg/lbdmod/probe.go +++ b/pkg/lbdmod/probe.go @@ -35,6 +35,23 @@ type Options struct { // have to remember to add it. const systemReleasePath = "/var/lib/miren/release" +// HostOptions builds the options for inspecting this host. dataPath is where +// miren keeps its data; empty means DefaultDataPath. +// +// Everything that decides whether a disk gets accelerator mode goes through +// here, because they all have to reach the same answer or a node picks a mode +// it cannot serve. They disagreed before: the CLI searched the release +// directory it resolved through $HOME while the disk controller searched only +// the system one, so a host with lbdctl under ~/.miren/release would have the +// CLI choose accelerator and the controller choose universal. +// +// The rule is now the system release directory and PATH, for every caller. +// Nothing resolves a per-user location, since the CLI and the server run as +// different users and would resolve it differently. +func HostOptions(dataPath string) Options { + return Options{DataPath: dataPath} +} + func (o Options) root() string { if o.Root == "" { return "/" diff --git a/pkg/lbdmod/status.go b/pkg/lbdmod/status.go index bb2f2c442..578ba591f 100644 --- a/pkg/lbdmod/status.go +++ b/pkg/lbdmod/status.go @@ -199,7 +199,19 @@ func writeMarker(dataPath string, m Marker) error { if err != nil { return fmt.Errorf("encoding the lbd install record: %w", err) } - return os.WriteFile(path, append(data, '\n'), 0644) + + // Written through a rename so a crash mid-write cannot leave a truncated + // record. readMarker reports a corrupt one as an error rather than + // treating it as absent, which would otherwise wedge every later probe. + tmp := path + ".tmp" + if err := os.WriteFile(tmp, append(data, '\n'), 0644); err != nil { + return fmt.Errorf("writing %s: %w", tmp, err) + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return fmt.Errorf("installing %s: %w", path, err) + } + return nil } // removeMarker forgets that lbd was ever installed, so later boots stop diff --git a/pkg/lbdmod/status_test.go b/pkg/lbdmod/status_test.go index 8f4a92649..548565fb5 100644 --- a/pkg/lbdmod/status_test.go +++ b/pkg/lbdmod/status_test.go @@ -262,3 +262,24 @@ func TestSearchPathDoesNotWriteIntoTheCallersSlice(t *testing.T) { assert.Equal(t, []string{"/opt/bin", systemReleasePath}, got, "the returned slice must not alias the caller's array") } + +func TestMarkerSurvivesATruncatedWrite(t *testing.T) { + // The record is written through a rename, so a crash mid-write cannot + // leave a half-file behind. readMarker treats a corrupt record as an + // error rather than as absent, so a torn write would wedge every probe. + dataPath := t.TempDir() + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), + KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + // No temporary file is left behind. + _, err := os.Stat(markerPath(dataPath) + ".tmp") + assert.True(t, os.IsNotExist(err)) + + got, err := readMarker(dataPath) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, testRelease, got.KernelRelease) +} From e4be441afadd74d1c602400b08a0c66df6b918c7 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 8 Sep 2026 15:31:11 -0700 Subject: [PATCH 05/15] Build the lbd toolchain image in the cluster, not our registry The builder image is a base plus a handful of build packages and a script; it holds no lbd source. Publishing it meant owning a versioned artifact, a release job, and a public-registry dependency, all to distribute something a cluster can build for itself in seconds. It also made a second builder for another distribution expensive, when what differs between them is a Dockerfile. So the Dockerfile is embedded alongside the module source, and the coordinator builds it with the BuildKit it already runs, into the registry it already serves. Nodes pull it from cluster.local the way they already pull app images: same host mapping, same bearer token, minted by the coordinator on a distributed runner behalf. The image is tagged by a content hash of the Dockerfile and its build script, so it is rebuilt exactly when the toolchain changes and never otherwise. Artifact GC needed to learn about this. It archives every active artifact no AppVersion references, and a system image belongs to no app, so the toolchain would have been collected within the hour and its blobs deleted underneath the nodes still pulling it. An artifact entity is named after the tag it was pushed under, so the tag carries a reserved prefix and GC retains what matches. That keeps the exemption narrow: a genuinely orphaned artifact is still collected. The pull reuses the sandbox controller identity rather than minting a new one. It is the identity that already means "this node pulling an image from the cluster registry", the only non-BuildKit workload the registry grants reads to, and already in the set a runner may ask the coordinator for. The published image is still referenced and still works; removing it waits until the install path no longer needs a locally reachable registry. --- api/core/core_v1alpha/extra.go | 21 +++ components/runner/lbd.go | 27 +++- controllers/artifact/gc.go | 6 +- controllers/artifact/gc_test.go | 30 ++++ .../lbdmod/builder/Dockerfile | 15 +- .../lbdmod/builder}/build.sh | 0 pkg/lbdmod/builderimage.go | 144 ++++++++++++++++++ pkg/lbdmod/builderimage_test.go | 102 +++++++++++++ pkg/lbdmod/ctrbuild/ctrbuild.go | 30 +++- pkg/lbdmod/ctrbuild/registry.go | 88 +++++++++++ pkg/lbdmod/ctrbuild/registry_test.go | 89 +++++++++++ pkg/lbdmod/source.go | 34 +---- servers/build/lbd_builder.go | 103 +++++++++++++ servers/build/lbd_builder_test.go | 67 ++++++++ 14 files changed, 712 insertions(+), 44 deletions(-) rename docker/Dockerfile.lbd-builder => pkg/lbdmod/builder/Dockerfile (77%) rename {docker/lbd-builder => pkg/lbdmod/builder}/build.sh (100%) create mode 100644 pkg/lbdmod/builderimage.go create mode 100644 pkg/lbdmod/builderimage_test.go create mode 100644 pkg/lbdmod/ctrbuild/registry.go create mode 100644 pkg/lbdmod/ctrbuild/registry_test.go create mode 100644 servers/build/lbd_builder.go create mode 100644 servers/build/lbd_builder_test.go diff --git a/api/core/core_v1alpha/extra.go b/api/core/core_v1alpha/extra.go index de5c915ce..19e818523 100644 --- a/api/core/core_v1alpha/extra.go +++ b/api/core/core_v1alpha/extra.go @@ -1,6 +1,8 @@ package core_v1alpha import ( + "strings" + entity "miren.dev/runtime/pkg/entity" ) @@ -9,3 +11,22 @@ func MD(ea entity.AttrGetter) Metadata { md.Decode(ea) return md } + +// SystemArtifactPrefix marks an image miren pushes to the cluster registry for +// its own use rather than on behalf of an app. +// +// An artifact's entity name is the tag it was pushed under (see the registry's +// putManifest), so the prefix rides in the tag and needs no schema field. It +// exists because artifact GC archives everything no AppVersion references, and +// a system image belongs to no app: without a way to tell it apart from a +// genuinely orphaned artifact, it would be collected within the hour and its +// blobs deleted underneath the nodes still pulling it. +const SystemArtifactPrefix = "miren-system-" + +// IsSystemArtifact reports whether an artifact is one miren pushed for itself, +// and so must survive garbage collection even though no AppVersion points at +// it. +func IsSystemArtifact(id entity.Id) bool { + name := strings.TrimPrefix(string(id), "artifact/") + return strings.HasPrefix(name, SystemArtifactPrefix) +} diff --git a/components/runner/lbd.go b/components/runner/lbd.go index 6d6366bd8..9457ae78b 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -8,10 +8,22 @@ import ( containerd "github.com/containerd/containerd/v2/client" "miren.dev/runtime/components/diskio" + "miren.dev/runtime/components/netresolve" "miren.dev/runtime/pkg/lbdmod" "miren.dev/runtime/pkg/lbdmod/ctrbuild" + "miren.dev/runtime/pkg/workloadidentity" ) +// lbdDeps is what bringing accelerator mode up needs from the runner: a +// containerd to run the build in, the cluster address mapping and identity to +// pull the toolchain image with, and the data path holding the install record. +type lbdDeps struct { + CC *containerd.Client + Resolver netresolve.Resolver + WorkloadIssuer workloadidentity.TokenIssuer + DataPath string +} + // rebuildTimeout bounds the unattended rebuild at startup. // // A rebuild pulls the builder image, may fetch kernel headers, and compiles; @@ -38,19 +50,28 @@ const rebuildTimeout = 10 * time.Minute // // Neither failing nor timing out is fatal. Universal mode works everywhere, so // the worst case is slower disks, not a runner that will not start. -func setupLbd(ctx context.Context, cc *containerd.Client, dataPath string, log *slog.Logger) { +func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { if err := diskio.EnsureLbdDevices(ctx, log); err == nil { return } + // The toolchain image lives in the cluster registry and nowhere public, so + // the pull needs the cluster's own address mapping and a registry token. + // A distributed runner holds both: its issuer proxies to the coordinator, + // which is what mints the token on its behalf. + registry := &ctrbuild.ClusterRegistry{ + Resolver: deps.Resolver, + Issuer: deps.WorkloadIssuer, + } + // dataPath has to be the runner's own, not the package default: the // install record lives under it, and reading it from the wrong place // would make a host that installed lbd look like one that never did, so // the rebuild after a kernel upgrade would never fire. installer := &lbdmod.Installer{ Log: log, - Builder: ctrbuild.New(cc, log), - Options: lbdmod.HostOptions(dataPath), + Builder: ctrbuild.New(deps.CC, log, ctrbuild.WithClusterRegistry(registry)), + Options: lbdmod.HostOptions(deps.DataPath), } ctx, cancel := context.WithTimeout(ctx, rebuildTimeout) diff --git a/controllers/artifact/gc.go b/controllers/artifact/gc.go index 338e16c12..2f6fa9abb 100644 --- a/controllers/artifact/gc.go +++ b/controllers/artifact/gc.go @@ -153,7 +153,11 @@ func (c *GCController) RunGC(ctx context.Context) (*GCResult, error) { var art core_v1alpha.Artifact art.Decode(e.Entity()) - if referenced[art.ID] { + // System artifacts belong to no app by design -- the toolchain image + // miren builds for itself is the case -- so the AppVersion set will + // never name them. Collecting them would delete blobs out from under + // nodes still pulling the image. + if referenced[art.ID] || core_v1alpha.IsSystemArtifact(art.ID) { result.RetainedArtifacts++ continue } diff --git a/controllers/artifact/gc_test.go b/controllers/artifact/gc_test.go index 975a4eadf..a28a0f649 100644 --- a/controllers/artifact/gc_test.go +++ b/controllers/artifact/gc_test.go @@ -124,3 +124,33 @@ func TestGCController_SkipsArchived(t *testing.T) { require.Equal(t, 0, result.TotalArtifacts, "archived artifacts are not evaluated") require.Equal(t, 0, len(result.ArchivedArtifacts)) } + +func TestGCController_RetainsSystemArtifacts(t *testing.T) { + // The toolchain image miren builds for itself belongs to no app, so the + // AppVersion set will never name it. Collecting it would delete blobs out + // from under nodes still pulling the image. + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + sysID, err := inmem.Client.Create(ctx, + core_v1alpha.SystemArtifactPrefix+"lbd-builder-21c0e11624c12f31", + &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + // A genuine orphan alongside it, to prove the exemption is narrow and not + // just "artifacts with no app survive". + orphanID, err := inmem.Client.Create(ctx, "orphan", &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + gc := &GCController{Log: log, EAC: inmem.EAC, Config: GCConfig{CheckInterval: time.Hour}} + + result, err := gc.RunGC(ctx) + require.NoError(t, err) + + require.Equal(t, 1, result.RetainedArtifacts) + require.Equal(t, []entity.Id{orphanID}, result.ArchivedArtifacts) + require.Equal(t, core_v1alpha.ACTIVE, artifactStatus(t, inmem.EAC, sysID)) + require.Equal(t, core_v1alpha.ARCHIVED, artifactStatus(t, inmem.EAC, orphanID)) +} diff --git a/docker/Dockerfile.lbd-builder b/pkg/lbdmod/builder/Dockerfile similarity index 77% rename from docker/Dockerfile.lbd-builder rename to pkg/lbdmod/builder/Dockerfile index 67ce29763..4ada89fbf 100644 --- a/docker/Dockerfile.lbd-builder +++ b/pkg/lbdmod/builder/Dockerfile @@ -1,8 +1,13 @@ # Toolchain for compiling the lbd kernel module against a node's running kernel. # -# This image deliberately does NOT contain the module source. miren embeds the -# source and bind-mounts it at /src, so shipping a new module version means -# shipping a new miren, not republishing this image. +# This is embedded in the miren binary and built by the cluster's own BuildKit +# into the cluster-local registry, not published by us. That is why it is here +# rather than under docker/: nothing outside miren ever builds it, and the tag +# is a hash of this directory, so it is rebuilt exactly when it changes. +# +# It deliberately does NOT contain the module source. miren embeds the source +# and bind-mounts it at /src, so a new module version ships as a new miren +# without touching this image at all. # # The contract with pkg/lbdmod: # @@ -52,7 +57,9 @@ RUN printf 'path-exclude=/usr/share/man/*\npath-exclude=/usr/share/doc/*\npath-e && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* /usr/share/doc /usr/share/man -COPY docker/lbd-builder/build.sh /usr/local/bin/build-lbd +# The build context is this directory, materialized from the embed, so the +# script sits next to the Dockerfile rather than under a docker/ prefix. +COPY build.sh /usr/local/bin/build-lbd RUN chmod +x /usr/local/bin/build-lbd ENTRYPOINT ["/usr/local/bin/build-lbd"] diff --git a/docker/lbd-builder/build.sh b/pkg/lbdmod/builder/build.sh similarity index 100% rename from docker/lbd-builder/build.sh rename to pkg/lbdmod/builder/build.sh diff --git a/pkg/lbdmod/builderimage.go b/pkg/lbdmod/builderimage.go new file mode 100644 index 000000000..54969ed89 --- /dev/null +++ b/pkg/lbdmod/builderimage.go @@ -0,0 +1,144 @@ +package lbdmod + +import ( + "crypto/sha256" + "embed" + "encoding/hex" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + + "miren.dev/runtime/api/core/core_v1alpha" +) + +// builderFS holds the toolchain image definition. It is embedded rather than +// published so the cluster builds its own copy: the image is a base plus a +// handful of packages and a script, and owning a released artifact to +// distribute that costs more than rebuilding it. +// +//go:embed all:builder +var builderFS embed.FS + +// BuilderRepository is the repository half of the builder's image reference. +// The registry resolves manifests by tag alone and ignores this, so it is here +// to make the reference readable in logs and `ctr images ls`. +const BuilderRepository = "miren-system/lbd-builder" + +// BuilderDockerfile is the Dockerfile's name inside the build context, which +// BuildKit's dockerfile frontend takes as its "filename" attribute. +const BuilderDockerfile = "Dockerfile" + +// BuilderTag is the tag the image is pushed under. +// +// The tag carries SystemArtifactPrefix because an artifact's entity name is +// the tag it was pushed under, and that name is the only thing artifact GC has +// to tell a system image apart from a genuinely orphaned one. Without the +// prefix the toolchain image is archived within the hour and its blobs +// deleted. The rest is a content hash, so the tag moves exactly when the +// toolchain does. +func BuilderTag() string { + return core_v1alpha.SystemArtifactPrefix + "lbd-builder-" + BuilderVersion() +} + +// BuilderVersion is a content hash of the toolchain definition, used as the +// image tag. Hashing the content rather than tagging by hand means the image +// is rebuilt exactly when the Dockerfile or its build script changes, and +// never otherwise. +func BuilderVersion() string { + sum, err := hashFS(builderFS, "builder") + if err != nil { + // The tree is embedded at compile time, so a walk over it cannot fail + // for any reason a caller could act on. + panic(fmt.Sprintf("hashing the embedded lbd builder: %v", err)) + } + return sum +} + +// BuilderImage is the full reference the coordinator pushes to and nodes pull +// from. registryHost is normally ocireg.Host. +func BuilderImage(registryHost string) string { + return fmt.Sprintf("%s/%s:%s", registryHost, BuilderRepository, BuilderTag()) +} + +// MaterializeBuilder writes the toolchain definition into dir, which then +// becomes the BuildKit context. It has to reach a real directory: fsutil.NewFS +// only takes a path, and the repo has no in-memory build context. +func MaterializeBuilder(dir string) error { + return materializeEmbed(builderFS, "builder", dir) +} + +// hashFS produces a stable digest over every file under root: each path and +// its bytes, in sorted order, so the result does not depend on walk order. +func hashFS(fsys fs.FS, root string) (string, error) { + var paths []string + err := fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return "", err + } + sort.Strings(paths) + + h := sha256.New() + for _, path := range paths { + data, err := fs.ReadFile(fsys, path) + if err != nil { + return "", err + } + // Length-prefix the name so a path and its content cannot be confused + // with a different split of the same bytes. + fmt.Fprintf(h, "%d:%s\n", len(path), path) + fmt.Fprintf(h, "%d:", len(data)) + h.Write(data) + } + + // Short enough to read in an image tag, long enough not to collide. + return hex.EncodeToString(h.Sum(nil))[:16], nil +} + +// materializeEmbed writes an embedded tree rooted at root into dir, stripping +// the root prefix. Executable bits are not carried by embed.FS, so anything +// that has to run is given one. +func materializeEmbed(fsys fs.FS, root, dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + + return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + target := filepath.Join(dir, rel) + + if d.IsDir() { + return os.MkdirAll(target, 0755) + } + + data, err := fs.ReadFile(fsys, path) + if err != nil { + return fmt.Errorf("reading embedded %s: %w", path, err) + } + + mode := os.FileMode(0644) + if filepath.Ext(rel) == ".sh" { + mode = 0755 + } + if err := os.WriteFile(target, data, mode); err != nil { + return fmt.Errorf("writing %s: %w", target, err) + } + return nil + }) +} diff --git a/pkg/lbdmod/builderimage_test.go b/pkg/lbdmod/builderimage_test.go new file mode 100644 index 000000000..b80fbbe51 --- /dev/null +++ b/pkg/lbdmod/builderimage_test.go @@ -0,0 +1,102 @@ +package lbdmod + +import ( + "os" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/pkg/entity" +) + +func TestBuilderVersionIsStable(t *testing.T) { + // The tag is a content hash, so it must not move between calls -- a tag + // that changed per process would rebuild and re-push on every install. + first := BuilderVersion() + assert.Equal(t, first, BuilderVersion()) + assert.Len(t, first, 16) + assert.NotEmpty(t, first) +} + +func TestBuilderImageReference(t *testing.T) { + ref := BuilderImage("cluster.local:5000") + assert.Equal(t, "cluster.local:5000/"+BuilderRepository+":"+BuilderTag(), ref) + assert.Contains(t, ref, BuilderVersion(), "the tag carries the content hash") +} + +func TestBuilderTagSurvivesArtifactGC(t *testing.T) { + // An artifact's entity name is the tag it was pushed under, and that name + // is all artifact GC has to tell a system image from an orphan. If this + // ever stops holding, the toolchain image is collected within the hour and + // its blobs deleted underneath the nodes still pulling it. + assert.True(t, core_v1alpha.IsSystemArtifact(entity.Id("artifact/"+BuilderTag()))) + assert.False(t, core_v1alpha.IsSystemArtifact(entity.Id("artifact/orphan"))) +} + +func TestHashFSChangesWithContent(t *testing.T) { + base := fstest.MapFS{ + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + "b/build.sh": {Data: []byte("echo hi\n")}, + } + changed := fstest.MapFS{ + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + "b/build.sh": {Data: []byte("echo bye\n")}, + } + + baseSum, err := hashFS(base, "b") + require.NoError(t, err) + changedSum, err := hashFS(changed, "b") + require.NoError(t, err) + + assert.NotEqual(t, baseSum, changedSum, "changing the build script must change the tag") + + // Same content hashes the same, so an unchanged toolchain is never rebuilt. + again, err := hashFS(fstest.MapFS{ + "b/build.sh": {Data: []byte("echo hi\n")}, + "b/Dockerfile": {Data: []byte("FROM ubuntu:24.04\n")}, + }, "b") + require.NoError(t, err) + assert.Equal(t, baseSum, again, "the hash must not depend on walk order") +} + +func TestHashFSSeparatesNamesFromContent(t *testing.T) { + // Without length-prefixing, a rename could be cancelled out by a content + // change that shifts the same bytes across the boundary. + a, err := hashFS(fstest.MapFS{"b/ab": {Data: []byte("c")}}, "b") + require.NoError(t, err) + b, err := hashFS(fstest.MapFS{"b/a": {Data: []byte("bc")}}, "b") + require.NoError(t, err) + assert.NotEqual(t, a, b) +} + +func TestMaterializeBuilderWritesABuildContext(t *testing.T) { + dir := filepath.Join(t.TempDir(), "ctx") + require.NoError(t, MaterializeBuilder(dir)) + + // BuildKit's dockerfile frontend takes the filename relative to the + // context root, so the Dockerfile has to sit at the top of it. + dockerfile, err := os.ReadFile(filepath.Join(dir, BuilderDockerfile)) + require.NoError(t, err) + assert.Contains(t, string(dockerfile), "FROM ubuntu:24.04") + assert.Contains(t, string(dockerfile), "build-lbd") + + script, err := os.Stat(filepath.Join(dir, "build.sh")) + require.NoError(t, err) + // embed.FS drops the executable bit, and the image COPYs this in and runs + // it, so materializing has to put it back. + assert.NotZero(t, script.Mode().Perm()&0100, "build.sh must be executable") +} + +func TestMaterializeBuilderIsRepeatable(t *testing.T) { + dir := filepath.Join(t.TempDir(), "ctx") + require.NoError(t, MaterializeBuilder(dir)) + require.NoError(t, os.WriteFile(filepath.Join(dir, BuilderDockerfile), []byte("clobbered"), 0644)) + + require.NoError(t, MaterializeBuilder(dir)) + data, err := os.ReadFile(filepath.Join(dir, BuilderDockerfile)) + require.NoError(t, err) + assert.NotEqual(t, "clobbered", string(data)) +} diff --git a/pkg/lbdmod/ctrbuild/ctrbuild.go b/pkg/lbdmod/ctrbuild/ctrbuild.go index d1c57795d..db3e0cc6b 100644 --- a/pkg/lbdmod/ctrbuild/ctrbuild.go +++ b/pkg/lbdmod/ctrbuild/ctrbuild.go @@ -42,11 +42,29 @@ const outputTailLines = 40 type Builder struct { cc *containerd.Client log *slog.Logger + + // registry, when set, teaches the pull about the cluster-local registry. + // The toolchain image lives there and nowhere public, so a builder without + // it can only run an image already in the node's containerd. + registry *ClusterRegistry } // New returns a Builder that runs containers on cc. -func New(cc *containerd.Client, log *slog.Logger) *Builder { - return &Builder{cc: cc, log: log} +func New(cc *containerd.Client, log *slog.Logger, opts ...Option) *Builder { + b := &Builder{cc: cc, log: log} + for _, o := range opts { + o(b) + } + return b +} + +// Option configures a Builder. +type Option func(*Builder) + +// WithClusterRegistry lets the builder pull from the cluster-local registry, +// which is where the toolchain image is published. +func WithClusterRegistry(r *ClusterRegistry) Option { + return func(b *Builder) { b.registry = r } } // Build pulls the image, runs the container to completion, and tears @@ -154,7 +172,13 @@ func (b *Builder) resolveImage(ctx context.Context, ref string) (containerd.Imag } b.log.Info("pulling the lbd builder image", "image", ref) - img, err := b.cc.Pull(ctx, ref, containerd.WithPullUnpack) + + pullOpts := []containerd.RemoteOpt{containerd.WithPullUnpack} + if b.registry != nil { + pullOpts = append(pullOpts, containerd.WithResolver(b.registry.resolver())) + } + + img, err := b.cc.Pull(ctx, ref, pullOpts...) if err != nil { return nil, fmt.Errorf("pulling %s: %w", ref, err) } diff --git a/pkg/lbdmod/ctrbuild/registry.go b/pkg/lbdmod/ctrbuild/registry.go new file mode 100644 index 000000000..3a48a08fe --- /dev/null +++ b/pkg/lbdmod/ctrbuild/registry.go @@ -0,0 +1,88 @@ +package ctrbuild + +import ( + "fmt" + "net/http" + + "github.com/containerd/containerd/v2/core/remotes" + "github.com/containerd/containerd/v2/core/remotes/docker" + "miren.dev/runtime/components/netresolve" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/containerdx" + "miren.dev/runtime/pkg/workloadidentity" +) + +// ClusterRegistry resolves images from the cluster-local registry, which is +// where the toolchain image lives. Without it a node can only reach public +// registries, and the builder image is deliberately not published to one. +// +// This is the same path app image pulls already take on a runner: the address +// comes from an in-process host map rather than DNS, the hop is plain HTTP on +// the cluster's own network, and the bearer token is minted by the coordinator +// on a distributed runner's behalf. +type ClusterRegistry struct { + // Resolver maps cluster.local to an address. On a runner that is the + // coordinator's IP; on the coordinator it is the local router. + Resolver netresolve.Resolver + + // Issuer mints the registry token. A distributed runner holds a remote + // issuer that proxies to the coordinator, since it has no signing key of + // its own. + Issuer workloadidentity.TokenIssuer +} + +// resolver returns a containerd resolver that knows the cluster registry and +// falls back to the normal public behavior for every other host. +func (c *ClusterRegistry) resolver() remotes.Resolver { + return docker.NewResolver(docker.ResolverOptions{ + Hosts: func(host string) ([]docker.RegistryHost, error) { + switch host { + case "cluster.local", ocireg.Host: + h, err := c.host() + if err != nil { + return nil, err + } + return []docker.RegistryHost{h}, nil + default: + return []docker.RegistryHost{containerdx.DefaultRegistryHost(host)}, nil + } + }, + }) +} + +func (c *ClusterRegistry) host() (docker.RegistryHost, error) { + addr, err := c.Resolver.LookupHost("cluster.local") + if err != nil { + return docker.RegistryHost{}, fmt.Errorf("resolving cluster.local: %w", err) + } + + h := docker.RegistryHost{ + Client: http.DefaultClient, + Host: addr.String() + ":5000", + Scheme: "http", + Path: "/v2", + // Pull only. The registry refuses a push under this identity anyway, + // and nothing here ever needs one. + Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, + } + + if c.Issuer == nil { + return h, nil + } + + // Reuses the sandbox controller's identity rather than minting a new one: + // it is the identity that means "this node pulling an image from the + // cluster registry", it is already the only non-BuildKit workload the + // registry grants GET and HEAD to, and it is already in the set a runner + // is allowed to ask the coordinator for. + token, err := c.Issuer.IssueSystemWorkloadToken( + workloadidentity.SystemWorkloadSandboxController, + workloadidentity.TokenOptions{Audience: []string{ocireg.Audience}}, + ) + if err != nil { + return docker.RegistryHost{}, fmt.Errorf("issuing a registry token: %w", err) + } + h.Header = http.Header{"Authorization": []string{"Bearer " + token}} + + return h, nil +} diff --git a/pkg/lbdmod/ctrbuild/registry_test.go b/pkg/lbdmod/ctrbuild/registry_test.go new file mode 100644 index 000000000..25a0f7440 --- /dev/null +++ b/pkg/lbdmod/ctrbuild/registry_test.go @@ -0,0 +1,89 @@ +package ctrbuild + +import ( + "net/netip" + "testing" + + "github.com/containerd/containerd/v2/core/remotes/docker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/workloadidentity" +) + +type fakeResolver struct { + addr netip.Addr + err error +} + +func (f fakeResolver) LookupHost(string) (netip.Addr, error) { return f.addr, f.err } + +// fakeIssuer records what identity and audience the pull asked for. +type fakeIssuer struct { + workloadidentity.TokenIssuer + + workload workloadidentity.SystemWorkload + audience []string +} + +func (f *fakeIssuer) IssueSystemWorkloadToken(w workloadidentity.SystemWorkload, opts workloadidentity.TokenOptions) (string, error) { + f.workload = w + f.audience = opts.Audience + return "test-token", nil +} + +func TestClusterRegistryHostTargetsTheClusterRegistry(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + + h, err := r.host() + require.NoError(t, err) + + // The registry is reached over the cluster's own network, so plain HTTP on + // the coordinator's address rather than a public registry over TLS. + assert.Equal(t, "10.1.2.3:5000", h.Host) + assert.Equal(t, "http", h.Scheme) + assert.Equal(t, "/v2", h.Path) + // Pull only: the registry refuses a push under this identity anyway. + assert.Zero(t, h.Capabilities&docker.HostCapabilityPush) + assert.NotZero(t, h.Capabilities&docker.HostCapabilityPull) +} + +func TestClusterRegistryWithoutAnIssuerSendsNoToken(t *testing.T) { + // A coordinator with no issuer configured still resolves; the registry + // only enforces when it has an issuer of its own. + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + + h, err := r.host() + require.NoError(t, err) + assert.Empty(t, h.Header.Get("Authorization")) +} + +func TestClusterRegistryReportsAResolveFailure(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{err: assert.AnError}} + + _, err := r.host() + require.Error(t, err) + assert.Contains(t, err.Error(), "resolving cluster.local") +} + +func TestClusterRegistryAsksForAPullIdentity(t *testing.T) { + issuer := &fakeIssuer{} + r := &ClusterRegistry{ + Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}, + Issuer: issuer, + } + + h, err := r.host() + require.NoError(t, err) + + assert.Equal(t, "Bearer test-token", h.Header.Get("Authorization")) + // The registry grants GET and HEAD to exactly this identity; anything else + // is a 403 on every pull. + assert.Equal(t, workloadidentity.SystemWorkloadSandboxController, issuer.workload) + assert.Equal(t, []string{ocireg.Audience}, issuer.audience) +} + +func TestResolverIsBuilt(t *testing.T) { + r := &ClusterRegistry{Resolver: fakeResolver{addr: netip.MustParseAddr("10.1.2.3")}} + assert.NotNil(t, r.resolver()) +} diff --git a/pkg/lbdmod/source.go b/pkg/lbdmod/source.go index 9912796a2..22ffd702e 100644 --- a/pkg/lbdmod/source.go +++ b/pkg/lbdmod/source.go @@ -1,11 +1,6 @@ package lbdmod import ( - "fmt" - "io/fs" - "os" - "path/filepath" - lbdsrc "miren.dev/runtime/third_party/lbd" ) @@ -19,32 +14,5 @@ func SourceVersion() string { // files next to the source, so this has to be a real writable directory rather // than a read-only mount of something we already have. func materializeSource(dir string) error { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("creating the build directory %s: %w", dir, err) - } - - return fs.WalkDir(lbdsrc.FS, "src", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - rel, err := filepath.Rel("src", path) - if err != nil { - return err - } - target := filepath.Join(dir, rel) - - if d.IsDir() { - return os.MkdirAll(target, 0755) - } - - data, err := lbdsrc.FS.ReadFile(path) - if err != nil { - return fmt.Errorf("reading embedded %s: %w", path, err) - } - if err := os.WriteFile(target, data, 0644); err != nil { - return fmt.Errorf("writing %s: %w", target, err) - } - return nil - }) + return materializeEmbed(lbdsrc.FS, "src", dir) } diff --git a/servers/build/lbd_builder.go b/servers/build/lbd_builder.go new file mode 100644 index 000000000..987f105fb --- /dev/null +++ b/servers/build/lbd_builder.go @@ -0,0 +1,103 @@ +package build + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/tonistiigi/fsutil" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/lbdmod" +) + +// lbdBuilderLock serializes the toolchain build within one coordinator. Two +// concurrent builds would both succeed -- the registry dedupes by manifest +// digest -- but they would each spend a full image build to get there. +var lbdBuilderLock sync.Mutex + +// EnsureLbdBuilderImage makes sure the lbd toolchain image is in the cluster +// registry and returns the reference nodes should pull. +// +// The image is a base plus a handful of build packages; it carries no lbd +// source. Building it here rather than publishing one means there is no +// released artifact to version and no public registry to depend on, and adding +// a builder for another distribution later costs only a Dockerfile. +// +// It is keyed by a content hash of the embedded Dockerfile and build script, so +// this is a no-op on every call after the first until that content changes. +func (b *Builder) EnsureLbdBuilderImage(ctx context.Context) (string, error) { + ref := lbdmod.BuilderImage(ocireg.Host) + + if b.present(ctx) { + return ref, nil + } + + lbdBuilderLock.Lock() + defer lbdBuilderLock.Unlock() + + // Another call may have finished the build while this one waited. + if b.present(ctx) { + return ref, nil + } + + if b.BuildKit == nil { + return "", fmt.Errorf("no buildkit available to build the lbd toolchain image") + } + + dir, err := os.MkdirTemp(b.TempDir, "lbd-builder-") + if err != nil { + return "", fmt.Errorf("creating a build context directory: %w", err) + } + defer os.RemoveAll(dir) + + if err := lbdmod.MaterializeBuilder(dir); err != nil { + return "", err + } + + // The dockerfile frontend takes its context from a real directory, and + // resolves "filename" relative to that root, so the Dockerfile has to sit + // inside the context we just wrote. + dfs, err := fsutil.NewFS(dir) + if err != nil { + return "", fmt.Errorf("opening the build context %s: %w", dir, err) + } + + bkc, err := b.BuildKit.Client(ctx) + if err != nil { + return "", fmt.Errorf("connecting to buildkit: %w", err) + } + defer bkc.Close() + + b.Log.Info("building the lbd toolchain image", "image", ref) + + bk := &Buildkit{Client: bkc, Log: b.Log, WorkloadIssuer: b.WorkloadIssuer} + res, err := bk.BuildImage(ctx, dfs, BuildStack{ + Stack: "dockerfile", + Input: lbdmod.BuilderDockerfile, + }, lbdmod.BuilderRepository, ref) + if err != nil { + return "", fmt.Errorf("building the lbd toolchain image: %w", err) + } + + b.Log.Info("built the lbd toolchain image", "image", ref, "digest", res.ManifestDigest) + return ref, nil +} + +// present reports whether the toolchain image for this content hash is already +// in the registry. An artifact's entity name is the tag it was pushed under, +// so the tag is the lookup key. +func (b *Builder) present(ctx context.Context) bool { + if b.ec == nil { + return false + } + + var artifact core_v1alpha.Artifact + if err := b.ec.Get(ctx, lbdmod.BuilderTag(), &artifact); err != nil { + return false + } + // An archived artifact has had, or is about to have, its blobs collected, + // so it cannot be pulled and has to be rebuilt. + return artifact.Status == core_v1alpha.ACTIVE +} diff --git a/servers/build/lbd_builder_test.go b/servers/build/lbd_builder_test.go new file mode 100644 index 000000000..a523396c0 --- /dev/null +++ b/servers/build/lbd_builder_test.go @@ -0,0 +1,67 @@ +package build + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" + "miren.dev/runtime/components/ocireg" + "miren.dev/runtime/pkg/entity/testutils" + "miren.dev/runtime/pkg/lbdmod" +) + +func TestEnsureLbdBuilderImageSkipsAnExistingImage(t *testing.T) { + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + // The artifact's entity name is the tag it was pushed under, so a prior + // build shows up under exactly this name. + _, err := inmem.Client.Create(ctx, lbdmod.BuilderTag(), + &core_v1alpha.Artifact{Status: core_v1alpha.ACTIVE}) + require.NoError(t, err) + + // BuildKit is deliberately nil: finding the image must short-circuit + // before anything tries to build, or every install would rebuild. + b := &Builder{Log: log, EAS: inmem.EAC, ec: entityserver.NewClient(log, inmem.EAC)} + + ref, err := b.EnsureLbdBuilderImage(ctx) + require.NoError(t, err) + assert.Equal(t, lbdmod.BuilderImage(ocireg.Host), ref) +} + +func TestEnsureLbdBuilderImageRebuildsAnArchivedImage(t *testing.T) { + // An archived artifact has had, or is about to have, its blobs collected, + // so it cannot be pulled. Treating it as present would hand nodes a + // reference that fails at pull time. + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + log := testutils.TestLogger(t) + + _, err := inmem.Client.Create(ctx, lbdmod.BuilderTag(), + &core_v1alpha.Artifact{Status: core_v1alpha.ARCHIVED}) + require.NoError(t, err) + + b := &Builder{Log: log, EAS: inmem.EAC, ec: entityserver.NewClient(log, inmem.EAC)} + + _, err = b.EnsureLbdBuilderImage(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no buildkit", "it should have tried to rebuild") +} + +func TestEnsureLbdBuilderImageNeedsBuildkit(t *testing.T) { + ctx := context.Background() + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + b := &Builder{Log: testutils.TestLogger(t), EAS: inmem.EAC, ec: entityserver.NewClient(testutils.TestLogger(t), inmem.EAC)} + + _, err := b.EnsureLbdBuilderImage(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "no buildkit available") +} From 7f0e9d265d9aa55c6ad971ff6e047273b146fc78 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Tue, 8 Sep 2026 16:06:02 -0700 Subject: [PATCH 06/15] Install the lbd module through the cluster, not the CLI With the toolchain image now living in the cluster registry, the CLI could no longer reach it: pulling needs a workload identity token, and the CLI holds none. That made the local install path a dead end, and it was the wrong shape anyway for a cluster with more than one node, since it could only ever install on the machine you happened to be typing on. So installation moves to the server, and the work splits across two machines for a reason. The toolchain image is built once on the coordinator, where BuildKit and the registry already are. The kernel module is built on the target node, because it has to be compiled against the kernel running there and loaded into it. The coordinator gains RunnerRegistration.InstallDiskAccelerator, which makes sure the image exists and then dials the node, following the same pattern the exec proxy uses. Each runner gains a NodeAdmin service; it had no admin surface to extend, only exec and sandbox metrics, and this is not about a sandbox. The CLI now names the node to install on, and status stays local and read-only. That lets the published image go: imagerefs.LbdBuilder, the release job that pushed it, and the registry lookup guarding that push are all gone. Verified across a coordinator and a separate runner: the image was built and pushed to the cluster registry, the runner pulled it over cluster.local with a coordinator-minted token, and compiled lbd.ko for its own kernel. That run also turned up a missing check -- a container can have /lib/modules mounted and no kmod, so depmod is confirmed before the build rather than after a minute of compiling. hack/dev-distributed could not bootstrap from a git worktree at all, which is how that verification nearly did not happen. Writing global git config from /src fails when .git points at a parent repo the peer cannot see, and it needs no repo, so it runs from / instead. --- .github/workflows/release.yml | 76 ------ api/nodeadmin/nodeadmin.go | 6 + api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go | 220 ++++++++++++++++++ api/nodeadmin/rpc.yml | 38 +++ api/runner/rpc.yml | 30 +++ api/runner/runner_v1alpha/rpc.gen.go | 197 ++++++++++++++++ cli/commands/commands.go | 6 +- cli/commands/disk_accelerator.go | 79 +++---- cli/commands/disk_accelerator_doc.go | 13 +- cli/commands/disk_accelerator_other.go | 8 +- components/coordinate/runner_endpoints.go | 10 + components/runner/lbd.go | 19 ++ components/runner/nodeadmin.go | 59 +++++ components/runner/runner.go | 6 + components/runner/storage.go | 4 +- docs/docs/command/disk-accelerator-install.md | 17 +- docs/docs/command/disk-accelerator.md | 13 +- docs/docs/disk-accelerator.md | 21 +- hack/dev-distributed | 8 +- pkg/imagerefs/imagerefs.go | 6 - pkg/lbdmod/build.go | 26 ++- pkg/lbdmod/build_test.go | 32 +++ pkg/rpc/service.go | 4 + servers/runner/disk_accelerator.go | 97 ++++++++ servers/runner/registration.go | 10 + 25 files changed, 830 insertions(+), 175 deletions(-) create mode 100644 api/nodeadmin/nodeadmin.go create mode 100644 api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go create mode 100644 api/nodeadmin/rpc.yml create mode 100644 components/runner/nodeadmin.go create mode 100644 servers/runner/disk_accelerator.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4afabe7cc..326f1efdb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -469,82 +469,6 @@ jobs: # this path makes worse. cache-to: ${{ needs.init.outputs.cache == 'true' && 'type=gha,mode=max' || '' }} - build-and-push-lbd-builder: - # The toolchain image that compiles the lbd kernel module on a node. It - # carries no module source, so its tag only moves when the toolchain does: - # this publishes the tag pkg/imagerefs pins, and does nothing if that tag - # already exists. Bumping imagerefs.LbdBuilder is what ships a new one. - needs: [init, test] - if: >- - !cancelled() && - (needs.test.result == 'success' || needs.test.result == 'skipped') - runs-on: depot-ubuntu-latest - permissions: - contents: read - id-token: write - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - ref: ${{ needs.init.outputs.sha }} - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - workload_identity_provider: ${{ secrets.WIF_PROVIDER }} - service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} - - - name: Configure Docker for Artifact Registry - run: gcloud auth configure-docker us-central1-docker.pkg.dev - - - name: Decide whether the builder image needs publishing - id: lbd-builder - run: | - set -euo pipefail - # The pinned reference is the single source of truth for the tag. - REF=$(grep -oE 'oci\.miren\.cloud/lbd-builder:[A-Za-z0-9._-]+' pkg/imagerefs/imagerefs.go | head -1) - if [ -z "$REF" ]; then - echo "could not find the lbd-builder reference in pkg/imagerefs/imagerefs.go" >&2 - exit 1 - fi - TAG="${REF##*:}" - IMAGE="us-central1-docker.pkg.dev/miren-cloud/miren-oci/lbd-builder:$TAG" - echo "image=$IMAGE" >> $GITHUB_OUTPUT - - # Republishing an existing tag would move a pinned reference under - # already-deployed clusters, so only a genuinely absent tag is - # published. An auth or transient failure must not read as absent: - # that is how a lookup blip turns into a moved tag, so anything - # other than a confirmed NOT_FOUND fails the job. - if lookup=$(gcloud artifacts docker images describe "$IMAGE" 2>&1); then - echo "$IMAGE already exists; nothing to publish" - echo "publish=false" >> $GITHUB_OUTPUT - elif printf '%s' "$lookup" | grep -qiE 'NOT_FOUND|was not found|does not exist'; then - echo "$IMAGE does not exist yet; publishing" - echo "publish=true" >> $GITHUB_OUTPUT - else - echo "could not determine whether $IMAGE exists:" >&2 - printf '%s\n' "$lookup" >&2 - exit 1 - fi - - - name: Set up Docker Buildx - if: steps.lbd-builder.outputs.publish == 'true' - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - - name: Build and push the lbd builder image - if: steps.lbd-builder.outputs.publish == 'true' - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - file: ./docker/Dockerfile.lbd-builder - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.lbd-builder.outputs.image }} - cache-from: type=gha - cache-to: ${{ needs.init.outputs.cache == 'true' && 'type=gha,mode=max' || '' }} - upload-to-miren: needs: [init, test, package, build-binaries] # Same reasoning as build-and-push-docker: tolerate `test` being skipped on diff --git a/api/nodeadmin/nodeadmin.go b/api/nodeadmin/nodeadmin.go new file mode 100644 index 000000000..936a432ae --- /dev/null +++ b/api/nodeadmin/nodeadmin.go @@ -0,0 +1,6 @@ +// Package nodeadmin carries the RPC the coordinator uses to ask one node to +// change something about itself. +package nodeadmin + +//go:generate mkdir -p nodeadmin_v1alpha +//go:generate go run ../../pkg/rpc/cmd/rpcgen -pkg nodeadmin_v1alpha -input rpc.yml -output nodeadmin_v1alpha/rpc.gen.go diff --git a/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go b/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go new file mode 100644 index 000000000..c0c6c3b1e --- /dev/null +++ b/api/nodeadmin/nodeadmin_v1alpha/rpc.gen.go @@ -0,0 +1,220 @@ +package nodeadmin_v1alpha + +import ( + "context" + "encoding/json" + + "github.com/fxamacker/cbor/v2" + rpc "miren.dev/runtime/pkg/rpc" +) + +type nodeAdminInstallDiskAcceleratorArgsData struct { + Image *string `cbor:"0,keyasint,omitempty" json:"image,omitempty"` + Force *bool `cbor:"1,keyasint,omitempty" json:"force,omitempty"` +} + +type NodeAdminInstallDiskAcceleratorArgs struct { + call rpc.Call + data nodeAdminInstallDiskAcceleratorArgsData +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) HasImage() bool { + return v.data.Image != nil +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) Image() string { + if v.data.Image == nil { + return "" + } + return *v.data.Image +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type nodeAdminInstallDiskAcceleratorResultsData struct { + KernelRelease *string `cbor:"0,keyasint,omitempty" json:"kernel_release,omitempty"` + LbdVersion *string `cbor:"1,keyasint,omitempty" json:"lbd_version,omitempty"` + Error *string `cbor:"2,keyasint,omitempty" json:"error,omitempty"` +} + +type NodeAdminInstallDiskAcceleratorResults struct { + call rpc.Call + data nodeAdminInstallDiskAcceleratorResultsData +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetKernelRelease(kernel_release string) { + v.data.KernelRelease = &kernel_release +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetLbdVersion(lbd_version string) { + v.data.LbdVersion = &lbd_version +} + +func (v *NodeAdminInstallDiskAcceleratorResults) SetError(error string) { + v.data.Error = &error +} + +func (v *NodeAdminInstallDiskAcceleratorResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *NodeAdminInstallDiskAcceleratorResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type NodeAdminInstallDiskAccelerator struct { + rpc.Call + args NodeAdminInstallDiskAcceleratorArgs + results NodeAdminInstallDiskAcceleratorResults +} + +func (t *NodeAdminInstallDiskAccelerator) Args() *NodeAdminInstallDiskAcceleratorArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *NodeAdminInstallDiskAccelerator) Results() *NodeAdminInstallDiskAcceleratorResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + +type NodeAdmin interface { + InstallDiskAccelerator(ctx context.Context, state *NodeAdminInstallDiskAccelerator) error +} + +type reexportNodeAdmin struct { + client rpc.Client +} + +func (reexportNodeAdmin) InstallDiskAccelerator(ctx context.Context, state *NodeAdminInstallDiskAccelerator) error { + panic("not implemented") +} + +func (t reexportNodeAdmin) CapabilityClient() rpc.Client { + return t.client +} + +func AdaptNodeAdmin(t NodeAdmin) *rpc.Interface { + methods := []rpc.Method{ + { + Name: "install_disk_accelerator", + InterfaceName: "NodeAdmin", + Index: 0, + Public: false, + Params: []string{"image", "force"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.InstallDiskAccelerator(ctx, &NodeAdminInstallDiskAccelerator{Call: call}) + }, + }, + } + + return rpc.NewInterface(methods, t) +} + +type NodeAdminClient struct { + rpc.Client +} + +func NewNodeAdminClient(client rpc.Client) *NodeAdminClient { + return &NodeAdminClient{Client: client} +} + +func (c NodeAdminClient) Export() NodeAdmin { + return reexportNodeAdmin{client: c.Client} +} + +type NodeAdminClientInstallDiskAcceleratorResults struct { + client rpc.Client + data nodeAdminInstallDiskAcceleratorResultsData +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasKernelRelease() bool { + return v.data.KernelRelease != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) KernelRelease() string { + if v.data.KernelRelease == nil { + return "" + } + return *v.data.KernelRelease +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasLbdVersion() bool { + return v.data.LbdVersion != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) LbdVersion() string { + if v.data.LbdVersion == nil { + return "" + } + return *v.data.LbdVersion +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) HasError() bool { + return v.data.Error != nil +} + +func (v *NodeAdminClientInstallDiskAcceleratorResults) Error() string { + if v.data.Error == nil { + return "" + } + return *v.data.Error +} + +func (v NodeAdminClient) InstallDiskAccelerator(ctx context.Context, image string, force bool) (*NodeAdminClientInstallDiskAcceleratorResults, error) { + args := NodeAdminInstallDiskAcceleratorArgs{} + args.data.Image = &image + args.data.Force = &force + + var ret nodeAdminInstallDiskAcceleratorResultsData + + err := v.Call(ctx, "install_disk_accelerator", &args, &ret) + if err != nil { + return nil, err + } + + return &NodeAdminClientInstallDiskAcceleratorResults{client: v.Client, data: ret}, nil +} diff --git a/api/nodeadmin/rpc.yml b/api/nodeadmin/rpc.yml new file mode 100644 index 000000000..ee4d98317 --- /dev/null +++ b/api/nodeadmin/rpc.yml @@ -0,0 +1,38 @@ +apiVersion: miren.dev/rpc/v1 +kind: IDL + +# Work the coordinator asks one specific node to do to itself. +# +# This is separate from the exec service because it is not about a sandbox: it +# acts on the host, and the coordinator picks the node rather than deriving it +# from a workload's placement. + +interfaces: + - name: NodeAdmin + methods: + - name: install_disk_accelerator + doc: | + Build and load the lbd kernel module on this node, so its disks can + use accelerator mode instead of loop devices. + + The module has to be compiled against the kernel actually running + here, which is why this happens on the node rather than centrally. + The toolchain image is built once by the coordinator and pulled from + the cluster registry. + parameters: + - name: image + type: string + doc: Toolchain image reference to build with + - name: force + type: bool + doc: Rebuild even when the installed module is already current + results: + - name: kernel_release + type: string + doc: Kernel the module was built for + - name: lbd_version + type: string + doc: lbd version that was installed + - name: error + type: string + doc: Error message if the install failed diff --git a/api/runner/rpc.yml b/api/runner/rpc.yml index 97474895b..57760f4f1 100644 --- a/api/runner/rpc.yml +++ b/api/runner/rpc.yml @@ -333,6 +333,36 @@ interfaces: type: string doc: Error message if issuance failed + - name: InstallDiskAccelerator + index: 13 + doc: | + Build and load the lbd kernel module on a runner, so its disks can + use accelerator mode instead of loop devices. + + The coordinator makes sure the toolchain image is in the cluster + registry, then asks the node to do the install. It runs on the node + because the module is compiled against the kernel running there. + parameters: + - name: query + type: string + doc: Runner to install on (name, ID, or short ID) + - name: force + type: bool + doc: Rebuild even when the installed module is already current + results: + - name: name + type: string + doc: Name of the runner installed on + - name: kernel_release + type: string + doc: Kernel the module was built for + - name: lbd_version + type: string + doc: lbd version that was installed + - name: error + type: string + doc: Error message if the install failed + types: - type: InviteInfo doc: Information about a runner invite diff --git a/api/runner/runner_v1alpha/rpc.gen.go b/api/runner/runner_v1alpha/rpc.gen.go index efef3153a..aff44e082 100644 --- a/api/runner/runner_v1alpha/rpc.gen.go +++ b/api/runner/runner_v1alpha/rpc.gen.go @@ -1547,6 +1547,98 @@ func (v *RunnerRegistrationIssueSystemWorkloadTokenResults) UnmarshalJSON(data [ return json.Unmarshal(data, &v.data) } +type runnerRegistrationInstallDiskAcceleratorArgsData struct { + Query *string `cbor:"0,keyasint,omitempty" json:"query,omitempty"` + Force *bool `cbor:"1,keyasint,omitempty" json:"force,omitempty"` +} + +type RunnerRegistrationInstallDiskAcceleratorArgs struct { + call rpc.Call + data runnerRegistrationInstallDiskAcceleratorArgsData +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) HasQuery() bool { + return v.data.Query != nil +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) Query() string { + if v.data.Query == nil { + return "" + } + return *v.data.Query +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) HasForce() bool { + return v.data.Force != nil +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) Force() bool { + if v.data.Force == nil { + return false + } + return *v.data.Force +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorArgs) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + +type runnerRegistrationInstallDiskAcceleratorResultsData struct { + Name *string `cbor:"0,keyasint,omitempty" json:"name,omitempty"` + KernelRelease *string `cbor:"1,keyasint,omitempty" json:"kernel_release,omitempty"` + LbdVersion *string `cbor:"2,keyasint,omitempty" json:"lbd_version,omitempty"` + Error *string `cbor:"3,keyasint,omitempty" json:"error,omitempty"` +} + +type RunnerRegistrationInstallDiskAcceleratorResults struct { + call rpc.Call + data runnerRegistrationInstallDiskAcceleratorResultsData +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetName(name string) { + v.data.Name = &name +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetKernelRelease(kernel_release string) { + v.data.KernelRelease = &kernel_release +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetLbdVersion(lbd_version string) { + v.data.LbdVersion = &lbd_version +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) SetError(error string) { + v.data.Error = &error +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) MarshalCBOR() ([]byte, error) { + return cbor.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) UnmarshalCBOR(data []byte) error { + return cbor.Unmarshal(data, &v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) MarshalJSON() ([]byte, error) { + return json.Marshal(v.data) +} + +func (v *RunnerRegistrationInstallDiskAcceleratorResults) UnmarshalJSON(data []byte) error { + return json.Unmarshal(data, &v.data) +} + type RunnerRegistrationCreateInvite struct { rpc.Call args RunnerRegistrationCreateInviteArgs @@ -1885,6 +1977,32 @@ func (t *RunnerRegistrationIssueSystemWorkloadToken) Results() *RunnerRegistrati return results } +type RunnerRegistrationInstallDiskAccelerator struct { + rpc.Call + args RunnerRegistrationInstallDiskAcceleratorArgs + results RunnerRegistrationInstallDiskAcceleratorResults +} + +func (t *RunnerRegistrationInstallDiskAccelerator) Args() *RunnerRegistrationInstallDiskAcceleratorArgs { + args := &t.args + if args.call != nil { + return args + } + args.call = t.Call + t.Call.Args(args) + return args +} + +func (t *RunnerRegistrationInstallDiskAccelerator) Results() *RunnerRegistrationInstallDiskAcceleratorResults { + results := &t.results + if results.call != nil { + return results + } + results.call = t.Call + t.Call.Results(results) + return results +} + type RunnerRegistration interface { CreateInvite(ctx context.Context, state *RunnerRegistrationCreateInvite) error Join(ctx context.Context, state *RunnerRegistrationJoin) error @@ -1899,6 +2017,7 @@ type RunnerRegistration interface { UncordonRunner(ctx context.Context, state *RunnerRegistrationUncordonRunner) error DrainRunner(ctx context.Context, state *RunnerRegistrationDrainRunner) error IssueSystemWorkloadToken(ctx context.Context, state *RunnerRegistrationIssueSystemWorkloadToken) error + InstallDiskAccelerator(ctx context.Context, state *RunnerRegistrationInstallDiskAccelerator) error } type reexportRunnerRegistration struct { @@ -1957,6 +2076,10 @@ func (reexportRunnerRegistration) IssueSystemWorkloadToken(ctx context.Context, panic("not implemented") } +func (reexportRunnerRegistration) InstallDiskAccelerator(ctx context.Context, state *RunnerRegistrationInstallDiskAccelerator) error { + panic("not implemented") +} + func (t reexportRunnerRegistration) CapabilityClient() rpc.Client { return t.client } @@ -2093,6 +2216,16 @@ func AdaptRunnerRegistration(t RunnerRegistration) *rpc.Interface { return t.IssueSystemWorkloadToken(ctx, &RunnerRegistrationIssueSystemWorkloadToken{Call: call}) }, }, + { + Name: "InstallDiskAccelerator", + InterfaceName: "RunnerRegistration", + Index: 13, + Public: false, + Params: []string{"query", "force"}, + Handler: func(ctx context.Context, call rpc.Call) error { + return t.InstallDiskAccelerator(ctx, &RunnerRegistrationInstallDiskAccelerator{Call: call}) + }, + }, } return rpc.NewInterface(methods, t) @@ -2845,3 +2978,67 @@ func (v RunnerRegistrationClient) IssueSystemWorkloadToken(ctx context.Context, return &RunnerRegistrationClientIssueSystemWorkloadTokenResults{client: v.Client, data: ret}, nil } + +type RunnerRegistrationClientInstallDiskAcceleratorResults struct { + client rpc.Client + data runnerRegistrationInstallDiskAcceleratorResultsData +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasName() bool { + return v.data.Name != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) Name() string { + if v.data.Name == nil { + return "" + } + return *v.data.Name +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasKernelRelease() bool { + return v.data.KernelRelease != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) KernelRelease() string { + if v.data.KernelRelease == nil { + return "" + } + return *v.data.KernelRelease +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasLbdVersion() bool { + return v.data.LbdVersion != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) LbdVersion() string { + if v.data.LbdVersion == nil { + return "" + } + return *v.data.LbdVersion +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) HasError() bool { + return v.data.Error != nil +} + +func (v *RunnerRegistrationClientInstallDiskAcceleratorResults) Error() string { + if v.data.Error == nil { + return "" + } + return *v.data.Error +} + +func (v RunnerRegistrationClient) InstallDiskAccelerator(ctx context.Context, query string, force bool) (*RunnerRegistrationClientInstallDiskAcceleratorResults, error) { + args := RunnerRegistrationInstallDiskAcceleratorArgs{} + args.data.Query = &query + args.data.Force = &force + + var ret runnerRegistrationInstallDiskAcceleratorResultsData + + err := v.Call(ctx, "InstallDiskAccelerator", &args, &ret) + if err != nil { + return nil, err + } + + return &RunnerRegistrationClientInstallDiskAcceleratorResults{client: v.Client, data: ret}, nil +} diff --git a/cli/commands/commands.go b/cli/commands/commands.go index c4b4c55b7..6d32dfac2 100644 --- a/cli/commands/commands.go +++ b/cli/commands/commands.go @@ -1422,12 +1422,12 @@ Warning: These commands are intended for advanced users and developers. They may )) d.Dispatch("disk accelerator install", Infer("disk accelerator install", "Build and load the lbd kernel module for this kernel", DiskAcceleratorInstall, WithExample(mflags.Example{ - Name: "Enable accelerator mode", - Body: "sudo miren disk accelerator install", + Name: "Enable accelerator mode on a runner", + Body: "miren disk accelerator install runner1", }), WithExample(mflags.Example{ Name: "Rebuild after a kernel upgrade", - Body: "sudo miren disk accelerator install --force", + Body: "miren disk accelerator install runner1 --force", }), )) d.Dispatch("disk accelerator uninstall", Infer("disk accelerator uninstall", "Unload and remove the lbd kernel module", DiskAcceleratorUninstall, diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go index 5ed627bb0..2c3cda6d2 100644 --- a/cli/commands/disk_accelerator.go +++ b/cli/commands/disk_accelerator.go @@ -3,13 +3,12 @@ package commands import ( - "context" "fmt" "time" - containerd "github.com/containerd/containerd/v2/client" + "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/pkg/lbdmod" - "miren.dev/runtime/pkg/lbdmod/ctrbuild" + "miren.dev/runtime/pkg/rpc" ) // DiskAcceleratorStatus reports whether accelerator mode can run on this host. @@ -51,47 +50,50 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { case status.Available() && !status.Stale(): return nil case status.Stale(): - ctx.Warn("The installed module no longer matches this host. Run: sudo miren disk accelerator install") + ctx.Warn("The installed module no longer matches this host. Run: miren disk accelerator install ") case status.Host.HeadersDir == "" && status.Host.CanFetchHeaders(): - ctx.Info("This host has no kernel headers; the builder will fetch them. Run: sudo miren disk accelerator install") + ctx.Info("This host has no kernel headers; the builder will fetch them. Run: miren disk accelerator install ") case status.Host.HeadersDir == "": ctx.Warn("This host has no kernel headers, which the build needs. %s", status.Host.InstallHint()) default: - ctx.Info("To enable accelerator mode, run: sudo miren disk accelerator install") + ctx.Info("To enable accelerator mode, run: miren disk accelerator install ") } return nil } -// DiskAcceleratorInstall compiles the lbd kernel module against the running -// kernel and loads it, so disks can use accelerator mode instead of falling -// back to loop devices. +// DiskAcceleratorInstall asks the cluster to build and load the lbd kernel +// module on a node, so its disks use accelerator mode instead of loop devices. +// +// This runs through the server rather than locally because the toolchain image +// lives in the cluster registry, and reaching it needs an identity the CLI does +// not hold. The coordinator builds the image if it is missing, then hands the +// work to the node, which is where the module has to be compiled anyway. func DiskAcceleratorInstall(ctx *Context, opts struct { - Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` - Image string `long:"image" description:"Override the builder image"` - Socket string `long:"socket" description:"Path to the containerd socket"` - DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` + ConfigCentric + + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` }) error { - cc, err := dialContainerd(ctx, opts.Socket) + client, err := ctx.RPCClient(rpc.ServiceRunner) if err != nil { return err } - defer cc.Close() + defer client.Close() - installer := &lbdmod.Installer{ - Log: ctx.Log, - Builder: ctrbuild.New(cc, ctx.Log), - Options: lbdmod.HostOptions(opts.DataPath), - Image: opts.Image, - } + rc := runner_v1alpha.NewRunnerRegistrationClient(client) + + ctx.Begin("Installing the lbd kernel module on %s", opts.Node) - ctx.Begin("Installing the lbd kernel module") - status, err := installer.Install(ctx, opts.Force) + res, err := rc.InstallDiskAccelerator(ctx, opts.Node, opts.Force) if err != nil { return err } + if res.Error() != "" { + return fmt.Errorf("%s", res.Error()) + } - ctx.Completed("Accelerator mode is ready on kernel %s", status.Host.KernelRelease) - ctx.Info("Restart the miren server to pick it up: sudo systemctl restart miren") + ctx.Completed("Accelerator mode is ready on %s, kernel %s", res.Name(), res.KernelRelease()) + ctx.Info("Restart that node's miren service to pick it up") return nil } @@ -115,33 +117,6 @@ func DiskAcceleratorUninstall(ctx *Context, opts struct { return nil } -// dialContainerd connects to the containerd miren runs its own containers on, -// preferring miren's socket over the system one. -func dialContainerd(ctx context.Context, socket string) (*containerd.Client, error) { - if socket == "" { - socket = defaultContainerdSocket() - } - - cc, err := containerd.New(socket) - if err != nil { - return nil, fmt.Errorf("could not reach containerd at %s, which the builder needs: %w", socket, err) - } - - // containerd.New does not connect, so without this a dead socket would - // surface much later as an opaque failure to pull the builder image. - serving, err := cc.IsServing(ctx) - if err != nil { - cc.Close() - return nil, fmt.Errorf("containerd at %s is not responding, which the builder needs: %w", socket, err) - } - if !serving { - cc.Close() - return nil, fmt.Errorf("containerd at %s answered but is not serving, which the builder needs", socket) - } - - return cc, nil -} - // acceleratorStatusJSON is the machine-readable shape of the status command. type acceleratorStatusJSON struct { Available bool `json:"available"` diff --git a/cli/commands/disk_accelerator_doc.go b/cli/commands/disk_accelerator_doc.go index a05fb990d..0b1aee9e2 100644 --- a/cli/commands/disk_accelerator_doc.go +++ b/cli/commands/disk_accelerator_doc.go @@ -10,16 +10,17 @@ write-ahead log in front of the disk. It is faster, and it is what continuous backup to Miren Cloud is built on. ` + "`lbd`" + ` is not part of the Linux kernel, so it has to be compiled for the -exact kernel your node is running. ` + "`miren disk accelerator install`" + ` does -that for you: it downloads a builder image, compiles the module against your -kernel's headers inside a container, then installs and loads the result. Nothing -but Miren has to be installed on the host beyond the kernel headers themselves. +exact kernel each node is running. ` + "`miren disk accelerator install`" + ` does +that: your cluster builds the toolchain image with the BuildKit and registry it +already runs, the named node pulls it from there, and the module is compiled and +loaded on that node. Nothing is downloaded from us, and there is no published +image to keep up to date. ## Getting started ` + "```" + `bash miren disk accelerator status # can this host run it? -sudo miren disk accelerator install # build and load the module +miren disk accelerator install runner1 # build and load it there sudo systemctl restart miren # pick up the new mode ` + "```" + ` @@ -38,7 +39,7 @@ sudo systemctl restart miren # pick up the new mode A module only loads on the kernel it was built for. Once a host has installed the module, Miren notices on startup that the running kernel has changed and rebuilds it. You can also do it by hand with -` + "`sudo miren disk accelerator install --force`" + `. +` + "`miren disk accelerator install --force`" + `. Until the module is back, disks fall back to universal mode. Nothing breaks; they are just slower.` diff --git a/cli/commands/disk_accelerator_other.go b/cli/commands/disk_accelerator_other.go index ee5c38930..3fe14d629 100644 --- a/cli/commands/disk_accelerator_other.go +++ b/cli/commands/disk_accelerator_other.go @@ -14,10 +14,10 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { // DiskAcceleratorInstall is not supported on non-Linux platforms func DiskAcceleratorInstall(ctx *Context, opts struct { - Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` - Image string `long:"image" description:"Override the builder image"` - Socket string `long:"socket" description:"Path to the containerd socket"` - DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` + ConfigCentric + + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` }) error { return fmt.Errorf("disk accelerator install is only available on Linux") } diff --git a/components/coordinate/runner_endpoints.go b/components/coordinate/runner_endpoints.go index d63eb2514..4d92cc5af 100644 --- a/components/coordinate/runner_endpoints.go +++ b/components/coordinate/runner_endpoints.go @@ -6,10 +6,12 @@ import ( "fmt" "path/filepath" + "miren.dev/runtime/api/entityserver" "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/api/sqlitebackup/sqlitebackup_v1alpha" "miren.dev/runtime/api/telemetry/telemetry_v1alpha" "miren.dev/runtime/pkg/rpc" + "miren.dev/runtime/servers/build" runnerserver "miren.dev/runtime/servers/runner" sqlitebackupsrv "miren.dev/runtime/servers/sqlitebackup" telemetrysrv "miren.dev/runtime/servers/telemetry" @@ -58,6 +60,14 @@ func (c *RunnerEndpoints) Start(context.Context) error { VictoriametricsAddress: c.VictoriametricsAddress, VictorialogsAddress: c.VictorialogsAddress, WorkloadIssuer: c.WorkloadIssuer, + LbdBuilder: &build.LbdToolchain{ + Log: c.Log, + BuildKit: c.BuildKit, + Issuer: c.WorkloadIssuer, + EC: entityserver.NewClient(c.Log, c.eac), + TempDir: c.TempDir, + }, + RPC: c.state, }) server.ExposeValue(rpc.ServiceRunner, runner_v1alpha.AdaptRunnerRegistration(runnerReg)) server.ExposeValue("dev.miren.runtime/telemetry", telemetry_v1alpha.AdaptTelemetry(telemetrysrv.NewServer(c.Log))) diff --git a/components/runner/lbd.go b/components/runner/lbd.go index 9457ae78b..9d569665c 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -99,3 +99,22 @@ func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { log.Warn("rebuilt the lbd kernel module but it is still not usable", "error", err) } } + +// lbdDeps gathers what accelerator mode needs from the runner. +func (r *SandboxHost) lbdDeps() lbdDeps { + return lbdDeps{ + CC: r.deps.CC, + Resolver: r.deps.Resolver, + WorkloadIssuer: r.deps.WorkloadIssuer, + DataPath: r.DataPath, + } +} + +func (s *NodeStorage) lbdDeps() lbdDeps { + return lbdDeps{ + CC: s.deps.CC, + Resolver: s.deps.Resolver, + WorkloadIssuer: s.access.deps.WorkloadIssuer, + DataPath: s.config.DataPath, + } +} diff --git a/components/runner/nodeadmin.go b/components/runner/nodeadmin.go new file mode 100644 index 000000000..450d27ddb --- /dev/null +++ b/components/runner/nodeadmin.go @@ -0,0 +1,59 @@ +package runner + +import ( + "context" + "log/slog" + + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/components/diskio" + "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/lbdmod/ctrbuild" +) + +// nodeAdminServer handles work the coordinator asks this node to do to itself. +// +// It lives on the runner rather than the coordinator because the work is about +// this host: a kernel module has to be compiled against the kernel actually +// running here, and loaded into it. +type nodeAdminServer struct { + log *slog.Logger + deps lbdDeps +} + +// InstallDiskAccelerator builds and loads the lbd kernel module on this node. +// +// Failures come back in the result rather than as an RPC error, so the +// operator sees why the install did not happen instead of a transport-level +// message. The call itself only fails when the node could not be reached. +func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodeadmin_v1alpha.NodeAdminInstallDiskAccelerator) error { + res := req.Results() + + installer := &lbdmod.Installer{ + Log: s.log, + Builder: ctrbuild.New(s.deps.CC, s.log, ctrbuild.WithClusterRegistry(&ctrbuild.ClusterRegistry{ + Resolver: s.deps.Resolver, + Issuer: s.deps.WorkloadIssuer, + })), + Options: lbdmod.HostOptions(s.deps.DataPath), + Image: req.Args().Image(), + } + + status, err := installer.Install(ctx, req.Args().Force()) + if err != nil { + s.log.Warn("installing the lbd kernel module failed", "error", err) + res.SetError(err.Error()) + return nil + } + + // The disk controller reads the mode once at startup, so a node that just + // gained accelerator mode keeps serving loop devices until it restarts. + // Say so rather than letting the operator discover it from a disk that + // came up the old way. + if err := diskio.EnsureLbdDevices(ctx, s.log); err != nil { + s.log.Warn("lbd installed but is not usable yet", "error", err) + } + + res.SetKernelRelease(status.Host.KernelRelease) + res.SetLbdVersion(lbdmod.SourceVersion()) + return nil +} diff --git a/components/runner/runner.go b/components/runner/runner.go index 139decfd2..1b3f083ec 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -541,6 +541,12 @@ func (r *SandboxHost) Start(ctx context.Context, eg ...*errgroup.Group) error { r.Log.Info("Registered exec server") + r.access.state.Server().ExposeValue(rpc.ServiceNodeAdmin, nodeadmin_v1alpha.AdaptNodeAdmin(&nodeAdminServer{ + log: r.Log.With("module", "nodeadmin"), + deps: r.lbdDeps(), + })) + r.Log.Info("Registered node admin server") + return nil } diff --git a/components/runner/storage.go b/components/runner/storage.go index a7e0e362a..1a1e743fc 100644 --- a/components/runner/storage.go +++ b/components/runner/storage.go @@ -71,9 +71,7 @@ func (s *NodeStorage) Start(ctx context.Context) error { if err := diskio.EnsureLoopDevices(log); err != nil { log.Warn("Loop devices not available, disk mounts will fail", "error", err) } - if err := diskio.EnsureLbdDevices(log); err != nil { - log.Warn("lbd devices not available, accelerator mode will not work", "error", err) - } + setupLbd(ctx, s.lbdDeps(), log) diskioState, err := diskio.LoadState(dataPath) if err != nil { diff --git a/docs/docs/command/disk-accelerator-install.md b/docs/docs/command/disk-accelerator-install.md index 29fe82eb0..b45e97cdc 100644 --- a/docs/docs/command/disk-accelerator-install.md +++ b/docs/docs/command/disk-accelerator-install.md @@ -11,15 +11,18 @@ Build and load the lbd kernel module for this kernel ## Usage ```bash -miren disk accelerator install [flags] +miren disk accelerator install [flags] ``` +## Arguments + +- `node` — Runner to install on (name, ID, or short ID) + ## Flags -- `--data-path` — Path to miren data (default: `/var/lib/miren`) +- `--cluster, -C` — Cluster name +- `--config` — Path to the config file - `--force, -f` — Rebuild even when the module is already current -- `--image` — Override the builder image -- `--socket` — Path to the containerd socket ## Global Options @@ -29,16 +32,16 @@ miren disk accelerator install [flags] ## Examples -**Enable accelerator mode:** +**Enable accelerator mode on a runner:** ```bash -sudo miren disk accelerator install +miren disk accelerator install runner1 ``` **Rebuild after a kernel upgrade:** ```bash -sudo miren disk accelerator install --force +miren disk accelerator install runner1 --force ``` ## See also diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md index 36ff6b397..94e10f443 100644 --- a/docs/docs/command/disk-accelerator.md +++ b/docs/docs/command/disk-accelerator.md @@ -18,16 +18,17 @@ write-ahead log in front of the disk. It is faster, and it is what continuous backup to Miren Cloud is built on. `lbd` is not part of the Linux kernel, so it has to be compiled for the -exact kernel your node is running. `miren disk accelerator install` does -that for you: it downloads a builder image, compiles the module against your -kernel's headers inside a container, then installs and loads the result. Nothing -but Miren has to be installed on the host beyond the kernel headers themselves. +exact kernel each node is running. `miren disk accelerator install` does +that: your cluster builds the toolchain image with the BuildKit and registry it +already runs, the named node pulls it from there, and the module is compiled and +loaded on that node. Nothing is downloaded from us, and there is no published +image to keep up to date. ## Getting started ```bash miren disk accelerator status # can this host run it? -sudo miren disk accelerator install # build and load the module +miren disk accelerator install runner1 # build and load it there sudo systemctl restart miren # pick up the new mode ``` @@ -46,7 +47,7 @@ sudo systemctl restart miren # pick up the new mode A module only loads on the kernel it was built for. Once a host has installed the module, Miren notices on startup that the running kernel has changed and rebuilds it. You can also do it by hand with -`sudo miren disk accelerator install --force`. +`miren disk accelerator install --force`. Until the module is back, disks fall back to universal mode. Nothing breaks; they are just slower. diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md index 6a7eb655b..b7b317260 100644 --- a/docs/docs/disk-accelerator.md +++ b/docs/docs/disk-accelerator.md @@ -25,15 +25,20 @@ node is running. One command does that. ## Minimum working example ```bash -miren disk accelerator status # can this host run it? -sudo miren disk accelerator install # build and load the module -sudo systemctl restart miren # pick up the new mode +miren disk accelerator status # can this host run it? +miren disk accelerator install runner1 # build and load the module there +sudo systemctl restart miren # on that node, to pick up the mode ``` -`install` downloads a builder image, compiles the module against your kernel's -headers inside a container, then installs and loads the result. The toolchain -lives in the image, so on Debian and Ubuntu there is nothing to install first — -if the host has no kernel headers, the builder fetches them for itself. +`install` names the node to install on, because the module has to be compiled +against the kernel that node is running. Your cluster builds the toolchain image +itself, using the same BuildKit and registry it already uses for your apps, and +the node pulls it from there. Nothing is downloaded from us, and there is no +image to keep up to date. + +The toolchain lives in that image, so on Debian and Ubuntu there is nothing to +install on the host first — if it has no kernel headers, the builder fetches +them for itself. Once the server restarts, new disks use accelerator mode. Existing disks keep whatever mode they were created with. @@ -98,7 +103,7 @@ the module it built, and rebuilds. You do not have to do anything, though you can force it by hand: ```bash -sudo miren disk accelerator install --force +miren disk accelerator install runner1 --force ``` This only happens on hosts that installed the module in the first place. A host diff --git a/hack/dev-distributed b/hack/dev-distributed index 2a3e48a31..ac7474bb5 100755 --- a/hack/dev-distributed +++ b/hack/dev-distributed @@ -34,7 +34,13 @@ bootstrap_peer() { # build VCS stamping fails the build. Unlike the standalone dev flow # (hack/dev.sh), which builds as the host user, this path builds as root, so # mark /src safe. Mirrors hack/package-release.sh. - peer_exec "$peer" git config --global --add safe.directory /src + # + # Run it from / rather than /src. In a git worktree /src/.git is a file + # pointing at the parent repo, which is not mounted in the peer, so git + # resolves the repo before writing global config and exits 128 on a path it + # cannot see. Writing global config needs no repo, so stepping outside the + # worktree sidesteps it. + peer_exec "$peer" bash -c "cd / && git config --global --add safe.directory /src" # Build binary peer_exec "$peer" make bin/miren # Symlinks diff --git a/pkg/imagerefs/imagerefs.go b/pkg/imagerefs/imagerefs.go index e7246563f..0eabb3097 100644 --- a/pkg/imagerefs/imagerefs.go +++ b/pkg/imagerefs/imagerefs.go @@ -38,12 +38,6 @@ const ( // Miren runtime server Miren = "oci.miren.cloud/miren:latest" - - // LbdBuilder carries the toolchain that compiles the lbd kernel module - // against a node's running kernel (docker/Dockerfile.lbd-builder). It - // holds no module source -- miren embeds that and mounts it in -- so the - // tag only moves when the toolchain itself needs to. - LbdBuilder = "oci.miren.cloud/lbd-builder:v1" ) // Base images for language stacks diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go index 86950b42f..e09630c43 100644 --- a/pkg/lbdmod/build.go +++ b/pkg/lbdmod/build.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "miren.dev/runtime/pkg/imagerefs" + "miren.dev/runtime/components/ocireg" ) const ( @@ -40,7 +40,8 @@ type Installer struct { // Options say where to read host state and keep the install record. Options Options - // Image overrides the builder image. Empty means imagerefs.LbdBuilder. + // Image is the toolchain image to build in. Empty falls back to the + // cluster registry reference, which is where the coordinator publishes it. Image string } @@ -56,7 +57,7 @@ func (i *Installer) image() string { if i.Image != "" { return i.Image } - return imagerefs.LbdBuilder + return BuilderImage(ocireg.Host) } // Install compiles the module against the running kernel and loads it. It is @@ -184,9 +185,28 @@ func (i *Installer) checkCanBuild(status Status) error { return fmt.Errorf("no container runtime to run the lbd builder in") } + if err := checkKernelTools(); err != nil { + return err + } + return i.checkCompilerAndHeaders(status) } +// checkKernelTools makes sure the commands that load a module are present. +// +// Checked before the build rather than after. The build takes minutes, and +// failing at the end on a missing binary wastes all of it and reads like the +// compile itself went wrong. Found on a container that had /lib/modules +// mounted but no kmod installed. +func checkKernelTools() error { + for _, tool := range []string{"depmod", "modprobe"} { + if _, err := exec.LookPath(tool); err != nil { + return fmt.Errorf("%s is not installed, and loading a kernel module needs it: install kmod", tool) + } + } + return nil +} + // checkCompilerAndHeaders covers the host conditions that make a build // pointless: firmware that will refuse the result, a toolchain we cannot // match, or no build tree to compile against. diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 9d0c78fb5..3aa63832e 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -422,3 +422,35 @@ func TestUninstallPathsWithNoMarker(t *testing.T) { // No install record means miren put nothing on this host. assert.Empty(t, uninstallPaths(nil)) } + +func TestCheckKernelToolsFailsBeforeAnExpensiveBuild(t *testing.T) { + // A container can have /lib/modules mounted and still lack kmod, which is + // how this was found. Catching it up front matters because the build takes + // minutes and the failure otherwise looks like the compile went wrong. + t.Setenv("PATH", t.TempDir()) + + err := checkKernelTools() + require.Error(t, err) + assert.Contains(t, err.Error(), "depmod is not installed") + assert.Contains(t, err.Error(), "install kmod", "the error has to say what to do about it") +} + +func TestCheckKernelToolsPassesWhenBothArePresent(t *testing.T) { + dir := t.TempDir() + for _, tool := range []string{"depmod", "modprobe"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, tool), []byte("#!/bin/sh\n"), 0755)) + } + t.Setenv("PATH", dir) + + require.NoError(t, checkKernelTools()) +} + +func TestCheckKernelToolsNeedsModprobeToo(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "depmod"), []byte("#!/bin/sh\n"), 0755)) + t.Setenv("PATH", dir) + + err := checkKernelTools() + require.Error(t, err) + assert.Contains(t, err.Error(), "modprobe is not installed") +} diff --git a/pkg/rpc/service.go b/pkg/rpc/service.go index 937c5b4e4..a399dca01 100644 --- a/pkg/rpc/service.go +++ b/pkg/rpc/service.go @@ -13,6 +13,10 @@ type ServiceID = string const ( ServiceRunner ServiceID = "dev.miren.runtime/runner" + // ServiceNodeAdmin is served by each runner, for work the coordinator asks + // one specific node to do to itself. + ServiceNodeAdmin ServiceID = "dev.miren.runtime/nodeadmin" + // ServiceSqliteBackup stores LTX transaction files replicated from // SQLite-provider disks on runners. ServiceSqliteBackup ServiceID = "dev.miren.runtime/sqlite-backup" diff --git a/servers/runner/disk_accelerator.go b/servers/runner/disk_accelerator.go new file mode 100644 index 000000000..3a1226951 --- /dev/null +++ b/servers/runner/disk_accelerator.go @@ -0,0 +1,97 @@ +package runner + +import ( + "context" + "fmt" + + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/api/runner/runner_v1alpha" + "miren.dev/runtime/pkg/rpc" +) + +// InstallDiskAccelerator builds and loads the lbd kernel module on a runner. +// +// The work splits across two machines. The toolchain image is built once, +// here, because the coordinator is where BuildKit and the registry live. The +// module itself is built on the target node, because it has to be compiled +// against the kernel running there and loaded into it. +// +// Failures come back in the result rather than as an RPC error, so an operator +// sees why the install did not happen. +func (s *RegistrationServer) InstallDiskAccelerator(ctx context.Context, req *runner_v1alpha.RunnerRegistrationInstallDiskAccelerator) error { + args := req.Args() + results := req.Results() + + if !args.HasQuery() || args.Query() == "" { + results.SetError("runner name or ID is required") + return nil + } + query := args.Query() + + if s.LbdBuilder == nil { + results.SetError("this cluster cannot build the lbd toolchain image, so accelerator mode is unavailable") + return nil + } + if s.RPC == nil { + results.SetError("no rpc state to reach the runner with") + return nil + } + + node, _, err := s.findNodeByQuery(ctx, query) + if err != nil { + s.Log.Error("failed to find runner", "query", query, "error", err) + results.SetError(err.Error()) + return nil + } + if node == nil { + results.SetError(fmt.Sprintf("runner %q not found", query)) + return nil + } + if node.ApiAddress == "" { + results.SetError(fmt.Sprintf("runner %q has no address to reach it on", query)) + return nil + } + + // Built before dialing: a node that pulls an image the cluster has not + // published yet fails with a registry error that says nothing about why. + image, err := s.LbdBuilder.EnsureLbdBuilderImage(ctx) + if err != nil { + s.Log.Error("failed to build the lbd toolchain image", "error", err) + results.SetError(fmt.Sprintf("building the lbd toolchain image: %v", err)) + return nil + } + + s.Log.Info("installing the lbd kernel module on a runner", + "node", node.ID, "address", node.ApiAddress, "image", image) + + cl, err := s.RPC.Connect(node.ApiAddress, string(rpc.ServiceNodeAdmin)) + if err != nil { + results.SetError(fmt.Sprintf("connecting to runner %q at %s: %v", query, node.ApiAddress, err)) + return nil + } + + nc := &nodeadmin_v1alpha.NodeAdminClient{Client: cl} + res, err := nc.InstallDiskAccelerator(ctx, image, args.HasForce() && args.Force()) + if err != nil { + results.SetError(fmt.Sprintf("installing on runner %q: %v", query, err)) + return nil + } + + if res.HasError() && res.Error() != "" { + results.SetError(res.Error()) + return nil + } + + results.SetName(node.Name) + results.SetKernelRelease(res.KernelRelease()) + results.SetLbdVersion(res.LbdVersion()) + return nil +} + +// LbdBuilderImageEnsurer builds the lbd toolchain image into the cluster +// registry if it is not already there. servers/build.Builder implements it; +// this is an interface so the runner registration server does not depend on +// the whole build server to ask one question of it. +type LbdBuilderImageEnsurer interface { + EnsureLbdBuilderImage(ctx context.Context) (string, error) +} diff --git a/servers/runner/registration.go b/servers/runner/registration.go index dd2b3e514..b146a14b1 100644 --- a/servers/runner/registration.go +++ b/servers/runner/registration.go @@ -53,6 +53,16 @@ type RegistrationServerConfig struct { // do not hold the cluster signing key, request tokens from the coordinator // through this server. May be nil when no issuer is configured. WorkloadIssuer *workloadidentity.Issuer + + // LbdBuilder builds the lbd toolchain image into the cluster registry, so + // a node has something to pull before it compiles the kernel module. Nil + // on a cluster with no BuildKit, where accelerator mode is unavailable. + LbdBuilder LbdBuilderImageEnsurer + + // RPC is how the coordinator reaches a specific runner. Installing the + // kernel module has to happen on the node itself, so unlike the rest of + // this server it is not enough to write an entity and wait. + RPC *rpc.State } type RegistrationServer struct { From 4d8cd21b75abbf9dd7487e75e6cf2322ee29f07b Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Thu, 10 Sep 2026 16:12:02 -0700 Subject: [PATCH 07/15] address review: close the node-admin authorization hole The install RPC took a caller-supplied image, ran its entrypoint, and installed the result as a kernel module on the host. That would be fine behind an authenticated boundary, and there is not one: the runner builds its RPC state with WithSkipVerify and no authenticator, so its listener accepts clients presenting no certificate at all and every caller arrives anonymous. The resolver also falls back to public registries for anything outside cluster.local. Together that meant anyone who could reach a runner API port could load arbitrary kernel code as root. Closed from both sides. The handler now requires the coordinator: not merely a certificate, since a registered runner holds a valid one too, but one whose subject is the coordinator API cert. And the image has to name this cluster own toolchain repository, so even a caller past the first gate cannot point the node somewhere else. The tag is left unchecked on purpose, because a coordinator on a newer miren carries a different content hash and asking a node to build with it is fine. The coordinator common name was a bare string in two places, which is how a check like this quietly stops matching. It is a shared constant now. Also from review: install no longer touches the local host, only makes an RPC, so leaving it behind a linux build tag stopped a macOS client from installing on a remote runner. It moves to an untagged file, with status and uninstall staying platform-split since they read /proc and load modules. The client returned by Connect was never closed. And a generated doc line said to restart miren without saying on which machine. --- cli/commands/disk_accelerator.go | 39 -------------------- cli/commands/disk_accelerator_doc.go | 2 +- cli/commands/disk_accelerator_install.go | 44 ++++++++++++++++++++++ cli/commands/disk_accelerator_other.go | 10 ----- components/coordinate/foundation.go | 2 +- components/runner/nodeadmin.go | 46 ++++++++++++++++++++++- components/runner/nodeadmin_test.go | 47 ++++++++++++++++++++++++ docs/docs/command/disk-accelerator.md | 2 +- pkg/lbdmod/builderimage.go | 19 ++++++++++ pkg/lbdmod/builderimage_test.go | 27 ++++++++++++++ pkg/rpc/authenticator.go | 6 +++ servers/runner/disk_accelerator.go | 1 + 12 files changed, 192 insertions(+), 53 deletions(-) create mode 100644 cli/commands/disk_accelerator_install.go create mode 100644 components/runner/nodeadmin_test.go diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go index 2c3cda6d2..aaf483337 100644 --- a/cli/commands/disk_accelerator.go +++ b/cli/commands/disk_accelerator.go @@ -3,12 +3,9 @@ package commands import ( - "fmt" "time" - "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/pkg/lbdmod" - "miren.dev/runtime/pkg/rpc" ) // DiskAcceleratorStatus reports whether accelerator mode can run on this host. @@ -61,42 +58,6 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { return nil } -// DiskAcceleratorInstall asks the cluster to build and load the lbd kernel -// module on a node, so its disks use accelerator mode instead of loop devices. -// -// This runs through the server rather than locally because the toolchain image -// lives in the cluster registry, and reaching it needs an identity the CLI does -// not hold. The coordinator builds the image if it is missing, then hands the -// work to the node, which is where the module has to be compiled anyway. -func DiskAcceleratorInstall(ctx *Context, opts struct { - ConfigCentric - - Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` - Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` -}) error { - client, err := ctx.RPCClient(rpc.ServiceRunner) - if err != nil { - return err - } - defer client.Close() - - rc := runner_v1alpha.NewRunnerRegistrationClient(client) - - ctx.Begin("Installing the lbd kernel module on %s", opts.Node) - - res, err := rc.InstallDiskAccelerator(ctx, opts.Node, opts.Force) - if err != nil { - return err - } - if res.Error() != "" { - return fmt.Errorf("%s", res.Error()) - } - - ctx.Completed("Accelerator mode is ready on %s, kernel %s", res.Name(), res.KernelRelease()) - ctx.Info("Restart that node's miren service to pick it up") - return nil -} - // DiskAcceleratorUninstall unloads the module and removes what the install put // on the host, including the record that would otherwise rebuild it after a // kernel upgrade. diff --git a/cli/commands/disk_accelerator_doc.go b/cli/commands/disk_accelerator_doc.go index 0b1aee9e2..222e9b68d 100644 --- a/cli/commands/disk_accelerator_doc.go +++ b/cli/commands/disk_accelerator_doc.go @@ -21,7 +21,7 @@ image to keep up to date. ` + "```" + `bash miren disk accelerator status # can this host run it? miren disk accelerator install runner1 # build and load it there -sudo systemctl restart miren # pick up the new mode +sudo systemctl restart miren # on that node, to pick up the mode ` + "```" + ` ## Requirements diff --git a/cli/commands/disk_accelerator_install.go b/cli/commands/disk_accelerator_install.go new file mode 100644 index 000000000..6cfdeef13 --- /dev/null +++ b/cli/commands/disk_accelerator_install.go @@ -0,0 +1,44 @@ +package commands + +import ( + "fmt" + + "miren.dev/runtime/api/runner/runner_v1alpha" + "miren.dev/runtime/pkg/rpc" +) + +// DiskAcceleratorInstall asks the cluster to build and load the lbd kernel +// module on a node, so its disks use accelerator mode instead of loop devices. +// +// This runs through the server rather than locally because the toolchain image +// lives in the cluster registry, and reaching it needs an identity the CLI does +// not hold. The coordinator builds the image if it is missing, then hands the +// work to the node, which is where the module has to be compiled anyway. +func DiskAcceleratorInstall(ctx *Context, opts struct { + ConfigCentric + + Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` + Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` +}) error { + client, err := ctx.RPCClient(rpc.ServiceRunner) + if err != nil { + return err + } + defer client.Close() + + rc := runner_v1alpha.NewRunnerRegistrationClient(client) + + ctx.Begin("Installing the lbd kernel module on %s", opts.Node) + + res, err := rc.InstallDiskAccelerator(ctx, opts.Node, opts.Force) + if err != nil { + return err + } + if res.Error() != "" { + return fmt.Errorf("%s", res.Error()) + } + + ctx.Completed("Accelerator mode is ready on %s, kernel %s", res.Name(), res.KernelRelease()) + ctx.Info("Restart that node's miren service to pick it up") + return nil +} diff --git a/cli/commands/disk_accelerator_other.go b/cli/commands/disk_accelerator_other.go index 3fe14d629..a1989c3bc 100644 --- a/cli/commands/disk_accelerator_other.go +++ b/cli/commands/disk_accelerator_other.go @@ -12,16 +12,6 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { return fmt.Errorf("disk accelerator status is only available on Linux") } -// DiskAcceleratorInstall is not supported on non-Linux platforms -func DiskAcceleratorInstall(ctx *Context, opts struct { - ConfigCentric - - Force bool `short:"f" long:"force" description:"Rebuild even when the module is already current"` - Node string `position:"0" usage:"Runner to install on (name, ID, or short ID)" required:"true"` -}) error { - return fmt.Errorf("disk accelerator install is only available on Linux") -} - // DiskAcceleratorUninstall is not supported on non-Linux platforms func DiskAcceleratorUninstall(ctx *Context, opts struct { DataPath string `long:"data-path" description:"Path to miren data" default:"/var/lib/miren"` diff --git a/components/coordinate/foundation.go b/components/coordinate/foundation.go index e62644b6f..dc5b02496 100644 --- a/components/coordinate/foundation.go +++ b/components/coordinate/foundation.go @@ -269,7 +269,7 @@ regen: c.Log.Info("generating new API cert", "path", cert) cc, err := c.authority.IssueCertificate(caauth.Options{ - CommonName: "miren-api", + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: 1 * year, IPs: ips, diff --git a/components/runner/nodeadmin.go b/components/runner/nodeadmin.go index 450d27ddb..8e0aa8a02 100644 --- a/components/runner/nodeadmin.go +++ b/components/runner/nodeadmin.go @@ -2,12 +2,14 @@ package runner import ( "context" + "fmt" "log/slog" "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" "miren.dev/runtime/components/diskio" "miren.dev/runtime/pkg/lbdmod" "miren.dev/runtime/pkg/lbdmod/ctrbuild" + "miren.dev/runtime/pkg/rpc" ) // nodeAdminServer handles work the coordinator asks this node to do to itself. @@ -28,6 +30,23 @@ type nodeAdminServer struct { func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodeadmin_v1alpha.NodeAdminInstallDiskAccelerator) error { res := req.Results() + if err := requireCoordinator(ctx); err != nil { + s.log.Warn("rejected a disk accelerator install", "error", err) + res.SetError(err.Error()) + return nil + } + + // Pinned to the cluster's own toolchain repository. The image's entrypoint + // runs here and its output is loaded into this kernel as root, so a + // reference pointing anywhere else is not something to act on even from a + // caller that got past the check above. + image := req.Args().Image() + if !lbdmod.IsBuilderImage(image) { + s.log.Warn("rejected a disk accelerator install naming a foreign image", "image", image) + res.SetError(fmt.Sprintf("%q is not this cluster's lbd toolchain image", image)) + return nil + } + installer := &lbdmod.Installer{ Log: s.log, Builder: ctrbuild.New(s.deps.CC, s.log, ctrbuild.WithClusterRegistry(&ctrbuild.ClusterRegistry{ @@ -35,7 +54,7 @@ func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodea Issuer: s.deps.WorkloadIssuer, })), Options: lbdmod.HostOptions(s.deps.DataPath), - Image: req.Args().Image(), + Image: image, } status, err := installer.Install(ctx, req.Args().Force()) @@ -57,3 +76,28 @@ func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodea res.SetLbdVersion(lbdmod.SourceVersion()) return nil } + +// requireCoordinator refuses anyone but the coordinator. +// +// The runner's API listener is built with rpc.WithSkipVerify and no +// authenticator, so it accepts clients that present no certificate at all and +// every caller arrives anonymous unless it brought one. That is tolerable for +// the services it already exposes; it is not tolerable here, where the handler +// pulls an image, runs it, and loads the result into the kernel as root. +// Anyone who could reach the port would otherwise have root on the node. +// +// The coordinator dials with its API certificate, so its subject is what +// separates it from everything else that can route to this port. +func requireCoordinator(ctx context.Context) error { + identity := rpc.IdentityFromContext(ctx) + if identity == nil || identity.Method == rpc.AuthMethodAnonymous { + return fmt.Errorf("installing a kernel module requires the coordinator's certificate, and this caller presented none") + } + if identity.Method != rpc.AuthMethodCert { + return fmt.Errorf("installing a kernel module requires a certificate, got %q", identity.Method) + } + if identity.Subject != rpc.CoordinatorCertSubject { + return fmt.Errorf("only the coordinator may install a kernel module, not %q", identity.Subject) + } + return nil +} diff --git a/components/runner/nodeadmin_test.go b/components/runner/nodeadmin_test.go new file mode 100644 index 000000000..7bee510cf --- /dev/null +++ b/components/runner/nodeadmin_test.go @@ -0,0 +1,47 @@ +package runner + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "miren.dev/runtime/pkg/rpc" +) + +func TestRequireCoordinatorRejectsAnAnonymousCaller(t *testing.T) { + // The runner's listener is built with WithSkipVerify and no + // authenticator, so an unauthenticated caller reaches the handler. This + // endpoint pulls an image, runs it, and loads the result into the kernel + // as root, so anonymous must not get through. + err := requireCoordinator(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "presented none") + + err = requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodAnonymous})) + require.Error(t, err) +} + +func TestRequireCoordinatorRejectsAnotherCertHolder(t *testing.T) { + // A registered runner holds a valid cluster certificate. Holding one is + // not the same as being the coordinator. + err := requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodCert, Subject: "runner-abc123"})) + require.Error(t, err) + assert.Contains(t, err.Error(), "only the coordinator") +} + +func TestRequireCoordinatorRejectsANonCertMethod(t *testing.T) { + // A bearer token or JWT carrying the right subject is still not the + // coordinator's certificate. + err := requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodJWT, Subject: rpc.CoordinatorCertSubject})) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a certificate") +} + +func TestRequireCoordinatorAcceptsTheCoordinator(t *testing.T) { + require.NoError(t, requireCoordinator(rpc.ContextWithIdentity(context.Background(), + &rpc.Identity{Method: rpc.AuthMethodCert, Subject: rpc.CoordinatorCertSubject}))) +} diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md index 94e10f443..c1cfe418c 100644 --- a/docs/docs/command/disk-accelerator.md +++ b/docs/docs/command/disk-accelerator.md @@ -29,7 +29,7 @@ image to keep up to date. ```bash miren disk accelerator status # can this host run it? miren disk accelerator install runner1 # build and load it there -sudo systemctl restart miren # pick up the new mode +sudo systemctl restart miren # on that node, to pick up the mode ``` ## Requirements diff --git a/pkg/lbdmod/builderimage.go b/pkg/lbdmod/builderimage.go index 54969ed89..8bf554c8c 100644 --- a/pkg/lbdmod/builderimage.go +++ b/pkg/lbdmod/builderimage.go @@ -9,8 +9,10 @@ import ( "os" "path/filepath" "sort" + "strings" "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/components/ocireg" ) // builderFS holds the toolchain image definition. It is embedded rather than @@ -62,6 +64,23 @@ func BuilderImage(registryHost string) string { return fmt.Sprintf("%s/%s:%s", registryHost, BuilderRepository, BuilderTag()) } +// IsBuilderImage reports whether a reference names the cluster's own lbd +// toolchain image. +// +// This is an authorization check rather than a convenience. A node asked to +// build runs whatever the image's entrypoint says and then loads the result +// into its own kernel as root, so the reference has to be pinned to the one +// repository this cluster publishes to. +// +// The tag is deliberately not checked. A coordinator running a newer miren +// carries a different content hash, and asking a node to build with it is +// legitimate; what must not be legitimate is pointing the node somewhere else +// entirely. +func IsBuilderImage(ref string) bool { + prefix := ocireg.Host + "/" + BuilderRepository + ":" + return strings.HasPrefix(ref, prefix) && len(ref) > len(prefix) +} + // MaterializeBuilder writes the toolchain definition into dir, which then // becomes the BuildKit context. It has to reach a real directory: fsutil.NewFS // only takes a path, and the repo has no in-memory build context. diff --git a/pkg/lbdmod/builderimage_test.go b/pkg/lbdmod/builderimage_test.go index b80fbbe51..43afda6eb 100644 --- a/pkg/lbdmod/builderimage_test.go +++ b/pkg/lbdmod/builderimage_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/components/ocireg" "miren.dev/runtime/pkg/entity" ) @@ -100,3 +101,29 @@ func TestMaterializeBuilderIsRepeatable(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, "clobbered", string(data)) } + +func TestIsBuilderImageRejectsForeignReferences(t *testing.T) { + // This gates what a node will pull, run, and load into its kernel, so it + // has to reject anything outside the cluster's own toolchain repository. + for _, ref := range []string{ + "docker.io/library/ubuntu:24.04", + "evil.example.com/miren-system/lbd-builder:v1", + "cluster.local:5000/someapp:latest", + // A prefix match on the host alone is not enough. + "cluster.local:5000/miren-system/lbd-builder-evil:v1", + // No tag at all. + "cluster.local:5000/" + BuilderRepository, + "cluster.local:5000/" + BuilderRepository + ":", + "", + } { + assert.False(t, IsBuilderImage(ref), "should have rejected %q", ref) + } +} + +func TestIsBuilderImageAcceptsOurOwn(t *testing.T) { + assert.True(t, IsBuilderImage(BuilderImage(ocireg.Host))) + + // A coordinator on a newer miren carries a different content hash, and + // asking a node to build with it is legitimate. + assert.True(t, IsBuilderImage(ocireg.Host+"/"+BuilderRepository+":miren-system-lbd-builder-0000000000000000")) +} diff --git a/pkg/rpc/authenticator.go b/pkg/rpc/authenticator.go index a37a44735..e82e2f89a 100644 --- a/pkg/rpc/authenticator.go +++ b/pkg/rpc/authenticator.go @@ -45,6 +45,12 @@ const ( AuthMethodSigned AuthMethod = "signed" // ed25519-signed request over a message transport ) +// CoordinatorCertSubject is the common name on the certificate the coordinator +// dials with. A runner uses it to tell the coordinator apart from anything else +// that can reach its API, which matters because the runner's listener does not +// require a client certificate at all. +const CoordinatorCertSubject = "miren-api" + // Identity represents an authenticated caller type Identity struct { // Subject is the primary identifier (cert CN, JWT subject, etc.) diff --git a/servers/runner/disk_accelerator.go b/servers/runner/disk_accelerator.go index 3a1226951..29e241fa1 100644 --- a/servers/runner/disk_accelerator.go +++ b/servers/runner/disk_accelerator.go @@ -69,6 +69,7 @@ func (s *RegistrationServer) InstallDiskAccelerator(ctx context.Context, req *ru results.SetError(fmt.Sprintf("connecting to runner %q at %s: %v", query, node.ApiAddress, err)) return nil } + defer cl.Close() nc := &nodeadmin_v1alpha.NodeAdminClient{Client: cl} res, err := nc.InstallDiskAccelerator(ctx, image, args.HasForce() && args.Force()) From d190faaf39eafbe285c4aa1149c7f997be7d2be9 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Sat, 12 Sep 2026 15:33:12 -0700 Subject: [PATCH 08/15] address review: let a waiting build give up, and say where status looks The toolchain build lock was a sync.Mutex, which cannot be cancelled. An image build takes minutes, so a second caller waiting behind one pinned its RPC handler with no way out. It is a 1-buffered channel now, selected on alongside ctx.Done(). The quick-start also read as though status checked the node you were about to install to. It does not: install names a node, while status and uninstall act on the host you run them on. --- cli/commands/disk_accelerator_doc.go | 9 +++++--- docs/docs/command/disk-accelerator.md | 9 +++++--- docs/docs/disk-accelerator.md | 4 +++- servers/build/lbd_builder.go | 15 ++++++++++---- servers/build/lbd_builder_test.go | 30 +++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/cli/commands/disk_accelerator_doc.go b/cli/commands/disk_accelerator_doc.go index 222e9b68d..3fa8d086d 100644 --- a/cli/commands/disk_accelerator_doc.go +++ b/cli/commands/disk_accelerator_doc.go @@ -19,11 +19,14 @@ image to keep up to date. ## Getting started ` + "```" + `bash -miren disk accelerator status # can this host run it? -miren disk accelerator install runner1 # build and load it there -sudo systemctl restart miren # on that node, to pick up the mode +miren disk accelerator install runner1 # build and load it on runner1 +sudo systemctl restart miren # on runner1, to pick up the mode ` + "```" + ` +` + "`install`" + ` names the node to install on, but ` + "`status`" + ` and +` + "`uninstall`" + ` read and change the host you run them on. To check a node +before or after installing, run ` + "`status`" + ` on that node. + ## Requirements - The kernel headers for your running kernel. On Debian and Ubuntu the builder diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md index c1cfe418c..d99bfaac1 100644 --- a/docs/docs/command/disk-accelerator.md +++ b/docs/docs/command/disk-accelerator.md @@ -27,11 +27,14 @@ image to keep up to date. ## Getting started ```bash -miren disk accelerator status # can this host run it? -miren disk accelerator install runner1 # build and load it there -sudo systemctl restart miren # on that node, to pick up the mode +miren disk accelerator install runner1 # build and load it on runner1 +sudo systemctl restart miren # on runner1, to pick up the mode ``` +`install` names the node to install on, but `status` and +`uninstall` read and change the host you run them on. To check a node +before or after installing, run `status` on that node. + ## Requirements - The kernel headers for your running kernel. On Debian and Ubuntu the builder diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md index b7b317260..0417a9c29 100644 --- a/docs/docs/disk-accelerator.md +++ b/docs/docs/disk-accelerator.md @@ -25,7 +25,6 @@ node is running. One command does that. ## Minimum working example ```bash -miren disk accelerator status # can this host run it? miren disk accelerator install runner1 # build and load the module there sudo systemctl restart miren # on that node, to pick up the mode ``` @@ -70,6 +69,9 @@ network at all. ## Checking what is going on +`status` reads the host you run it on, not the node you last installed to. To +see how a runner is doing, run it on that runner. + ```bash miren disk accelerator status ``` diff --git a/servers/build/lbd_builder.go b/servers/build/lbd_builder.go index 987f105fb..074288b27 100644 --- a/servers/build/lbd_builder.go +++ b/servers/build/lbd_builder.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "sync" "github.com/tonistiigi/fsutil" "miren.dev/runtime/api/core/core_v1alpha" @@ -15,7 +14,11 @@ import ( // lbdBuilderLock serializes the toolchain build within one coordinator. Two // concurrent builds would both succeed -- the registry dedupes by manifest // digest -- but they would each spend a full image build to get there. -var lbdBuilderLock sync.Mutex +// +// It is a channel rather than a sync.Mutex so a caller can give up: an image +// build takes minutes, and the RPC handler waiting behind one has to stay +// cancellable. +var lbdBuilderLock = make(chan struct{}, 1) // EnsureLbdBuilderImage makes sure the lbd toolchain image is in the cluster // registry and returns the reference nodes should pull. @@ -34,8 +37,12 @@ func (b *Builder) EnsureLbdBuilderImage(ctx context.Context) (string, error) { return ref, nil } - lbdBuilderLock.Lock() - defer lbdBuilderLock.Unlock() + select { + case lbdBuilderLock <- struct{}{}: + case <-ctx.Done(): + return "", fmt.Errorf("waiting for another lbd toolchain build to finish: %w", ctx.Err()) + } + defer func() { <-lbdBuilderLock }() // Another call may have finished the build while this one waited. if b.present(ctx) { diff --git a/servers/build/lbd_builder_test.go b/servers/build/lbd_builder_test.go index a523396c0..7b3fc876e 100644 --- a/servers/build/lbd_builder_test.go +++ b/servers/build/lbd_builder_test.go @@ -3,6 +3,7 @@ package build import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -54,6 +55,35 @@ func TestEnsureLbdBuilderImageRebuildsAnArchivedImage(t *testing.T) { assert.Contains(t, err.Error(), "no buildkit", "it should have tried to rebuild") } +func TestEnsureLbdBuilderImageGivesUpWhenTheCallerDoes(t *testing.T) { + // A toolchain build takes minutes, so a second caller has to be able to + // walk away rather than pinning an RPC handler until the first one lands. + inmem, cleanup := testutils.NewInMemEntityServer(t) + defer cleanup() + + lbdBuilderLock <- struct{}{} + defer func() { <-lbdBuilderLock }() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + log := testutils.TestLogger(t) + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} + + done := make(chan error, 1) + go func() { + _, err := b.EnsureLbdBuilderImage(ctx) + done <- err + }() + + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("it blocked on the build lock instead of honouring the cancelled context") + } +} + func TestEnsureLbdBuilderImageNeedsBuildkit(t *testing.T) { ctx := context.Background() inmem, cleanup := testutils.NewInMemEntityServer(t) From 00c0360dd984c74beeb4dcfbdbc82f778864e8aa Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 05:16:24 +0000 Subject: [PATCH 09/15] Authenticate coordinator calls on runner RPC listener Verify client certificates against the cluster CA before allowing node-admin installs. Report device initialization failures to the install caller instead of claiming readiness. --- components/runner/nodeadmin.go | 14 ++--- components/runner/nodeadmin_test.go | 89 +++++++++++++++++++++++++++-- components/runner/runner.go | 29 ++++++++-- 3 files changed, 114 insertions(+), 18 deletions(-) diff --git a/components/runner/nodeadmin.go b/components/runner/nodeadmin.go index 8e0aa8a02..d922a0ce8 100644 --- a/components/runner/nodeadmin.go +++ b/components/runner/nodeadmin.go @@ -70,6 +70,8 @@ func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodea // came up the old way. if err := diskio.EnsureLbdDevices(ctx, s.log); err != nil { s.log.Warn("lbd installed but is not usable yet", "error", err) + res.SetError(fmt.Sprintf("lbd installed but is not usable: %v", err)) + return nil } res.SetKernelRelease(status.Host.KernelRelease) @@ -79,15 +81,9 @@ func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodea // requireCoordinator refuses anyone but the coordinator. // -// The runner's API listener is built with rpc.WithSkipVerify and no -// authenticator, so it accepts clients that present no certificate at all and -// every caller arrives anonymous unless it brought one. That is tolerable for -// the services it already exposes; it is not tolerable here, where the handler -// pulls an image, runs it, and loads the result into the kernel as root. -// Anyone who could reach the port would otherwise have root on the node. -// -// The coordinator dials with its API certificate, so its subject is what -// separates it from everything else that can route to this port. +// The listener verifies client certificates against the cluster CA. The +// coordinator dials with its API certificate, whose subject separates it from +// other cluster members that can also reach this port. func requireCoordinator(ctx context.Context) error { identity := rpc.IdentityFromContext(ctx) if identity == nil || identity.Method == rpc.AuthMethodAnonymous { diff --git a/components/runner/nodeadmin_test.go b/components/runner/nodeadmin_test.go index 7bee510cf..5ec37925d 100644 --- a/components/runner/nodeadmin_test.go +++ b/components/runner/nodeadmin_test.go @@ -2,18 +2,19 @@ package runner import ( "context" + "log/slog" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" + "miren.dev/runtime/clientconfig" + "miren.dev/runtime/pkg/caauth" "miren.dev/runtime/pkg/rpc" ) func TestRequireCoordinatorRejectsAnAnonymousCaller(t *testing.T) { - // The runner's listener is built with WithSkipVerify and no - // authenticator, so an unauthenticated caller reaches the handler. This - // endpoint pulls an image, runs it, and loads the result into the kernel - // as root, so anonymous must not get through. err := requireCoordinator(context.Background()) require.Error(t, err) assert.Contains(t, err.Error(), "presented none") @@ -45,3 +46,83 @@ func TestRequireCoordinatorAcceptsTheCoordinator(t *testing.T) { require.NoError(t, requireCoordinator(rpc.ContextWithIdentity(context.Background(), &rpc.Identity{Method: rpc.AuthMethodCert, Subject: rpc.CoordinatorCertSubject}))) } + +func TestNodeAdminAuthenticatesCoordinatorOverWire(t *testing.T) { + ca, err := caauth.New(caauth.Options{CommonName: "cluster-ca", Organization: "miren", ValidFor: time.Hour}) + require.NoError(t, err) + runnerCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: "runner", Organization: "miren", ValidFor: time.Hour, DNSNames: []string{"localhost"}, + }) + require.NoError(t, err) + coordinatorCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + otherCert, err := ca.IssueCertificate(caauth.Options{ + CommonName: "another-runner", Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + foreignCA, err := caauth.New(caauth.Options{CommonName: "cluster-ca", Organization: "miren", ValidFor: time.Hour}) + require.NoError(t, err) + forgedCert, err := foreignCA.IssueCertificate(caauth.Options{ + CommonName: rpc.CoordinatorCertSubject, Organization: "miren", ValidFor: time.Hour, + }) + require.NoError(t, err) + + for _, insecure := range []bool{false, true} { + name := "verified-outbound" + if insecure { + name = "insecure-outbound" + } + t.Run(name, func(t *testing.T) { + cfg := clientconfig.NewConfig() + cfg.SetCluster("cluster", &clientconfig.ClusterConfig{ + Hostname: "localhost:0", CACert: string(ca.GetCACertificate()), + ClientCert: string(runnerCert.CertPEM), ClientKey: string(runnerCert.KeyPEM), Insecure: insecure, + }) + require.NoError(t, cfg.SetActiveCluster("cluster")) + access, err := NewClusterAccess(slog.Default(), RunnerDeps{}, RunnerConfig{ + Id: "runner", DataPath: t.TempDir(), ListenAddress: "localhost:0", Config: cfg, + }) + require.NoError(t, err) + server, err := access.newRPCState(t.Context()) + require.NoError(t, err) + defer server.Close() + server.Server().ExposeValue(rpc.ServiceNodeAdmin, nodeadmin_v1alpha.AdaptNodeAdmin(&nodeAdminServer{log: slog.Default()})) + + call := func(cert, key []byte) (*nodeadmin_v1alpha.NodeAdminClientInstallDiskAcceleratorResults, error) { + opts := []rpc.StateOption{rpc.WithSkipVerify} + if cert != nil { + opts = append(opts, rpc.WithCertPEMs(cert, key)) + } + client, err := rpc.NewState(t.Context(), opts...) + if err != nil { + return nil, err + } + defer client.Close() + cl, err := client.Connect(server.ListenAddr(), string(rpc.ServiceNodeAdmin)) + if err != nil { + return nil, err + } + defer cl.Close() + return nodeadmin_v1alpha.NewNodeAdminClient(cl).InstallDiskAccelerator(t.Context(), "foreign/image:tag", false) + } + + // The foreign image stops authorized callers before any host work. + result, err := call(coordinatorCert.CertPEM, coordinatorCert.KeyPEM) + require.NoError(t, err) + assert.Contains(t, result.Error(), "not this cluster's lbd toolchain image") + + result, err = call(otherCert.CertPEM, otherCert.KeyPEM) + require.NoError(t, err) + assert.Contains(t, result.Error(), "only the coordinator") + + _, err = call(forgedCert.CertPEM, forgedCert.KeyPEM) + require.Error(t, err) + + _, err = call(nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "401") + }) + } +} diff --git a/components/runner/runner.go b/components/runner/runner.go index 1b3f083ec..31dd46b72 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -444,11 +444,7 @@ func (r *ClusterAccess) Start(ctx context.Context) (retErr error) { ) r.Log.Info("establishing cluster access", "listen", r.ListenAddress, "distributed", r.Config != nil) - if r.Config == nil { - rs, err = rpc.NewState(ctx, rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress), rpc.WithSkipVerify) - } else { - rs, err = r.Config.State(ctx, rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress)) - } + rs, err = r.newRPCState(ctx) if err != nil { return err } @@ -495,6 +491,29 @@ func (r *ClusterAccess) Server() *rpc.Server { return r.state.Server() } +func (r *ClusterAccess) newRPCState(ctx context.Context) (*rpc.State, error) { + opts := []rpc.StateOption{ + rpc.WithLogger(r.Log), rpc.WithBindAddr(r.ListenAddress), + rpc.WithAuthenticator(&rpc.LocalOnlyAuthenticator{}), + } + if r.Config == nil { + return rpc.NewState(ctx, append(opts, rpc.WithSkipVerify)...) + } + + cluster, err := r.Config.GetActiveCluster() + if err != nil { + return nil, fmt.Errorf("runner cluster CA: %w", err) + } + if cluster.CACert == "" { + return nil, fmt.Errorf("runner cluster CA is required to authenticate coordinator requests") + } + // Config.State supplies the runner's own certificate for outbound calls. + // Apply the CA last so even an insecure outbound configuration cannot turn + // off verification of certificates presented to this listener. + opts = append(opts, rpc.WithCertificateVerification([]byte(cluster.CACert))) + return r.Config.State(ctx, opts...) +} + func (r *ClusterAccess) Close() error { var errs []error for _, closer := range r.closers { From ade170aa39e37f6815cd2e4efa66e7fa43a6644d Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 05:48:41 +0000 Subject: [PATCH 10/15] Keep startup rebuilds on published toolchains and tighten runner auth Remember the toolchain image that succeeded on each node so a newer runner can rebuild after a kernel upgrade before its coordinator has published a new builder. Restrict node-admin image inputs to system lbd-builder tags, fail closed without a cluster CA, and share the coordinator certificate subject with issuance. --- components/runner/nodeadmin.go | 4 ++-- components/runner/nodeadmin_test.go | 16 ++++++++++++++++ components/runner/runner.go | 15 +++++---------- pkg/lbdmod/build.go | 4 ++++ pkg/lbdmod/build_test.go | 22 ++++++++++++++++++++++ pkg/lbdmod/builderimage.go | 16 +++++++--------- pkg/lbdmod/builderimage_test.go | 4 ++++ pkg/lbdmod/status.go | 5 +++++ pkg/lbdmod/status_test.go | 1 + pkg/rpc/authenticator.go | 4 ++-- 10 files changed, 68 insertions(+), 23 deletions(-) diff --git a/components/runner/nodeadmin.go b/components/runner/nodeadmin.go index d922a0ce8..407519072 100644 --- a/components/runner/nodeadmin.go +++ b/components/runner/nodeadmin.go @@ -36,9 +36,9 @@ func (s *nodeAdminServer) InstallDiskAccelerator(ctx context.Context, req *nodea return nil } - // Pinned to the cluster's own toolchain repository. The image's entrypoint + // Pinned to the cluster's toolchain tag family. The image's entrypoint // runs here and its output is loaded into this kernel as root, so a - // reference pointing anywhere else is not something to act on even from a + // reference pointing at any other tag is not something to act on even from a // caller that got past the check above. image := req.Args().Image() if !lbdmod.IsBuilderImage(image) { diff --git a/components/runner/nodeadmin_test.go b/components/runner/nodeadmin_test.go index 5ec37925d..68a13a834 100644 --- a/components/runner/nodeadmin_test.go +++ b/components/runner/nodeadmin_test.go @@ -47,6 +47,22 @@ func TestRequireCoordinatorAcceptsTheCoordinator(t *testing.T) { &rpc.Identity{Method: rpc.AuthMethodCert, Subject: rpc.CoordinatorCertSubject}))) } +func TestRunnerListenerNeedsAClusterCA(t *testing.T) { + access, err := NewClusterAccess(slog.Default(), RunnerDeps{}, RunnerConfig{ + Id: "runner", DataPath: t.TempDir(), ListenAddress: "localhost:0", + }) + require.NoError(t, err) + _, err = access.newRPCState(t.Context()) + require.ErrorContains(t, err, "cluster config is required") + + cfg := clientconfig.NewConfig() + cfg.SetCluster("cluster", &clientconfig.ClusterConfig{Hostname: "localhost:0"}) + require.NoError(t, cfg.SetActiveCluster("cluster")) + access.Config = cfg + _, err = access.newRPCState(t.Context()) + require.ErrorContains(t, err, "cluster CA is required") +} + func TestNodeAdminAuthenticatesCoordinatorOverWire(t *testing.T) { ca, err := caauth.New(caauth.Options{CommonName: "cluster-ca", Organization: "miren", ValidFor: time.Hour}) require.NoError(t, err) diff --git a/components/runner/runner.go b/components/runner/runner.go index 31dd46b72..1b89f32ed 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -54,9 +54,8 @@ type RunnerConfig struct { Workers int `json:"workers" cbor:"workers" yaml:"workers"` DataPath string `json:"data_path" cbor:"data_path" yaml:"data_path"` - // Optional RPC configuration for advanced setups - // If not provided, a default insecure connection will be used - // to connect to the server address. + // RPC configuration supplies the cluster CA used to authenticate callers + // on the runner listener, as well as its outbound coordinator connection. Config *clientconfig.Config `json:"config" cbor:"config" yaml:"config"` // Optional cloud authentication configuration for disk replication @@ -443,7 +442,7 @@ func (r *ClusterAccess) Start(ctx context.Context) (retErr error) { client *rpc.NetworkClient ) - r.Log.Info("establishing cluster access", "listen", r.ListenAddress, "distributed", r.Config != nil) + r.Log.Info("establishing cluster access", "listen", r.ListenAddress) rs, err = r.newRPCState(ctx) if err != nil { return err @@ -459,11 +458,7 @@ func (r *ClusterAccess) Start(ctx context.Context) (retErr error) { r.state = nil } }() - if r.Config == nil { - client, err = rs.Connect("", "entities") - } else { - client, err = rs.Client("entities") - } + client, err = rs.Client("entities") if err != nil { return err } @@ -497,7 +492,7 @@ func (r *ClusterAccess) newRPCState(ctx context.Context) (*rpc.State, error) { rpc.WithAuthenticator(&rpc.LocalOnlyAuthenticator{}), } if r.Config == nil { - return rpc.NewState(ctx, append(opts, rpc.WithSkipVerify)...) + return nil, fmt.Errorf("runner cluster config is required to authenticate coordinator requests") } cluster, err := r.Config.GetActiveCluster() diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go index e09630c43..2deff5fe3 100644 --- a/pkg/lbdmod/build.go +++ b/pkg/lbdmod/build.go @@ -124,6 +124,7 @@ func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { marker := Marker{ LbdVersion: SourceVersion(), KernelRelease: status.Host.KernelRelease, + BuilderImage: i.image(), ModulePath: modulePath(status.Host.KernelRelease), LbdctlPath: filepath.Join(lbdctlInstallDir, "lbdctl"), BuiltAt: time.Now().UTC(), @@ -167,6 +168,9 @@ func (i *Installer) EnsureCurrent(ctx context.Context) (bool, error) { return false, nil } + if i.Image == "" { + i.Image = status.Marker.BuilderImage + } i.Log.Info("rebuilding the lbd kernel module", "reason", status.Explain()) if _, err := i.Install(ctx, false); err != nil { return false, err diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 3aa63832e..1e1a364fb 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -369,6 +369,28 @@ func TestEnsureCurrentRebuildsAfterAKernelUpgrade(t *testing.T) { "a stale module must trigger a rebuild attempt, not silence") } +func TestEnsureCurrentUsesThePublishedImageFromTheInstallRecord(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + previousImage := "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-previous" + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), KernelRelease: "6.8.0-45-generic", BuilderImage: previousImage, + })) + + i := testInstaller(t, root, dataPath) + // No builder is needed to verify image selection: the install will fail + // its preflight check before it could touch the host kernel. + _, err := i.EnsureCurrent(t.Context()) + require.Error(t, err) + assert.Equal(t, previousImage, i.image(), "a new binary's toolchain tag may not have been published") + + // An explicit image from a coordinator install must still take precedence. + i.Image = "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-new" + _, err = i.EnsureCurrent(t.Context()) + require.Error(t, err) + assert.Equal(t, "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-new", i.image()) +} + func TestBuildFailedErrorQuotesTheOutput(t *testing.T) { err := &BuildFailedError{ExitCode: 2, Output: "error: no kernel headers for 6.8.0-51-generic"} assert.Contains(t, err.Error(), "exit 2") diff --git a/pkg/lbdmod/builderimage.go b/pkg/lbdmod/builderimage.go index 8bf554c8c..a99b40d0b 100644 --- a/pkg/lbdmod/builderimage.go +++ b/pkg/lbdmod/builderimage.go @@ -64,20 +64,18 @@ func BuilderImage(registryHost string) string { return fmt.Sprintf("%s/%s:%s", registryHost, BuilderRepository, BuilderTag()) } -// IsBuilderImage reports whether a reference names the cluster's own lbd -// toolchain image. +// IsBuilderImage reports whether a reference names a cluster lbd toolchain tag. // // This is an authorization check rather than a convenience. A node asked to // build runs whatever the image's entrypoint says and then loads the result -// into its own kernel as root, so the reference has to be pinned to the one -// repository this cluster publishes to. +// into its own kernel as root. The registry resolves manifests by tag alone, +// so checking the repository without the tag prefix would also accept an app +// image under a forged repository name. // -// The tag is deliberately not checked. A coordinator running a newer miren -// carries a different content hash, and asking a node to build with it is -// legitimate; what must not be legitimate is pointing the node somewhere else -// entirely. +// The content hash is deliberately not checked: a newer coordinator may +// publish a different version of the toolchain for this node to build with. func IsBuilderImage(ref string) bool { - prefix := ocireg.Host + "/" + BuilderRepository + ":" + prefix := ocireg.Host + "/" + BuilderRepository + ":" + core_v1alpha.SystemArtifactPrefix + "lbd-builder-" return strings.HasPrefix(ref, prefix) && len(ref) > len(prefix) } diff --git a/pkg/lbdmod/builderimage_test.go b/pkg/lbdmod/builderimage_test.go index 43afda6eb..19ba29e89 100644 --- a/pkg/lbdmod/builderimage_test.go +++ b/pkg/lbdmod/builderimage_test.go @@ -111,6 +111,10 @@ func TestIsBuilderImageRejectsForeignReferences(t *testing.T) { "cluster.local:5000/someapp:latest", // A prefix match on the host alone is not enough. "cluster.local:5000/miren-system/lbd-builder-evil:v1", + // The registry ignores the repository on lookup, so an app tag can + // masquerade behind the toolchain repository name. + ocireg.Host + "/" + BuilderRepository + ":app-version-123", + ocireg.Host + "/" + BuilderRepository + ":" + core_v1alpha.SystemArtifactPrefix + "lbd-builder-", // No tag at all. "cluster.local:5000/" + BuilderRepository, "cluster.local:5000/" + BuilderRepository + ":", diff --git a/pkg/lbdmod/status.go b/pkg/lbdmod/status.go index 578ba591f..d6fcfa78c 100644 --- a/pkg/lbdmod/status.go +++ b/pkg/lbdmod/status.go @@ -35,6 +35,11 @@ type Marker struct { // for one release will not load on another. KernelRelease string `json:"kernel_release"` + // BuilderImage is the cluster image used for this install. Retaining the + // reference lets startup rebuild without asking the coordinator to publish + // the current binary's potentially newer toolchain tag first. + BuilderImage string `json:"builder_image,omitempty"` + // ModulePath is where the built module was installed. ModulePath string `json:"module_path"` diff --git a/pkg/lbdmod/status_test.go b/pkg/lbdmod/status_test.go index 548565fb5..cc529d3df 100644 --- a/pkg/lbdmod/status_test.go +++ b/pkg/lbdmod/status_test.go @@ -198,6 +198,7 @@ func TestMarkerRoundTrip(t *testing.T) { want := Marker{ LbdVersion: "v0.0.0-20260824210626-be4cec661034", KernelRelease: testRelease, + BuilderImage: "cluster.local:5000/miren-system/lbd-builder:miren-system-lbd-builder-oldhash", ModulePath: modulePath(testRelease), LbdctlPath: "/usr/local/bin/lbdctl", BuiltAt: time.Now().UTC().Truncate(time.Second), diff --git a/pkg/rpc/authenticator.go b/pkg/rpc/authenticator.go index e82e2f89a..c23152d4a 100644 --- a/pkg/rpc/authenticator.go +++ b/pkg/rpc/authenticator.go @@ -47,8 +47,8 @@ const ( // CoordinatorCertSubject is the common name on the certificate the coordinator // dials with. A runner uses it to tell the coordinator apart from anything else -// that can reach its API, which matters because the runner's listener does not -// require a client certificate at all. +// that can reach its API. The listener accepts certless connections but +// authenticates non-public calls with a verified client certificate. const CoordinatorCertSubject = "miren-api" // Identity represents an authenticated caller From 925a3b280275e6cd167806c6c5528a9536d2ced6 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 06:10:19 +0000 Subject: [PATCH 11/15] Adapt lbd startup and generated docs to current main --- cli/commands/disk_accelerator.go | 12 ++++++---- components/diskio/disk_ops_linux.go | 11 ++++++--- components/runner/runner.go | 1 + docs/docs/command/disk-accelerator-install.md | 2 +- docs/docs/command/disk-accelerator-status.md | 2 +- .../command/disk-accelerator-uninstall.md | 2 +- docs/docs/command/disk-accelerator.md | 8 +++---- docs/docs/command/disk.md | 5 ++-- docs/docs/commands.md | 6 ++++- docs/docs/disk-accelerator.md | 4 ++-- docs/docs/disks.md | 2 +- servers/build/lbd_builder.go | 23 +++++++++++++++---- servers/build/lbd_builder_test.go | 6 ++--- 13 files changed, 55 insertions(+), 29 deletions(-) diff --git a/cli/commands/disk_accelerator.go b/cli/commands/disk_accelerator.go index aaf483337..0aedcf70e 100644 --- a/cli/commands/disk_accelerator.go +++ b/cli/commands/disk_accelerator.go @@ -6,6 +6,7 @@ import ( "time" "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/ui" ) // DiskAcceleratorStatus reports whether accelerator mode can run on this host. @@ -23,7 +24,7 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { return PrintJSON(newAcceleratorStatusJSON(status)) } - rows := [][]string{ + rows := []ui.Row{ {"Available", yesNo(status.Available())}, {"State", status.Explain()}, {"Kernel", status.Host.KernelRelease}, @@ -36,12 +37,13 @@ func DiskAcceleratorStatus(ctx *Context, opts struct { } if status.Marker != nil { rows = append(rows, - []string{"Installed version", status.Marker.LbdVersion}, - []string{"Built for kernel", status.Marker.KernelRelease}, - []string{"Built at", status.Marker.BuiltAt.Local().Format(time.RFC3339)}, + ui.Row{"Installed version", status.Marker.LbdVersion}, + ui.Row{"Built for kernel", status.Marker.KernelRelease}, + ui.Row{"Built at", status.Marker.BuiltAt.Local().Format(time.RFC3339)}, ) } - ctx.DisplayTable([]string{"", ""}, rows) + table := ui.NewTable(ui.WithColumns(ui.AutoSizeColumns([]string{"", ""}, rows, nil)), ui.WithRows(rows)) + ctx.Printf("%s\n", table.Render()) switch { case status.Available() && !status.Stale(): diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 5fabc9cdd..736b61a27 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -20,6 +20,9 @@ import ( "miren.dev/runtime/pkg/lbdmod" ) +// Bound modprobe on the runner startup path. +const modprobeTimeout = 30 * time.Second + const ( loopCtlGetFree = 0x4C82 loopClrFd = 0x4C01 @@ -640,12 +643,14 @@ func ensureLoopDeviceNode(log *slog.Logger, index int) error { // wedged, and lbdctl can be installed on a host whose module never loaded. // What settles it is the same probe accelerator mode itself relies on, so a // node never selects accelerator mode it cannot serve. -func EnsureLbdDevices(log *slog.Logger) error { - if out, err := exec.Command("modprobe", lbdmod.ModuleName).CombinedOutput(); err != nil { +func EnsureLbdDevices(ctx context.Context, log *slog.Logger) error { + ctx, cancel := context.WithTimeout(ctx, modprobeTimeout) + defer cancel() + if out, err := exec.CommandContext(ctx, "modprobe", lbdmod.ModuleName).CombinedOutput(); err != nil { log.Debug("modprobe lbd failed", "error", err, "output", strings.TrimSpace(string(out))) } - status, err := lbdmod.Probe(lbdmod.Options{}) + status, err := lbdmod.Probe(lbdmod.HostOptions("")) if err != nil { return err } diff --git a/components/runner/runner.go b/components/runner/runner.go index 1b89f32ed..f9e6fb331 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -20,6 +20,7 @@ import ( "miren.dev/runtime/api/exec/exec_v1alpha" "miren.dev/runtime/api/metric/metric_v1alpha" "miren.dev/runtime/api/network/network_v1alpha" + "miren.dev/runtime/api/nodeadmin/nodeadmin_v1alpha" "miren.dev/runtime/api/runner/runner_v1alpha" "miren.dev/runtime/api/secret/secret_v1alpha" "miren.dev/runtime/api/sqlitebackup/sqlitebackup_v1alpha" diff --git a/docs/docs/command/disk-accelerator-install.md b/docs/docs/command/disk-accelerator-install.md index b45e97cdc..6887c11ac 100644 --- a/docs/docs/command/disk-accelerator-install.md +++ b/docs/docs/command/disk-accelerator-install.md @@ -46,4 +46,4 @@ miren disk accelerator install runner1 --force ## See also -- [`miren disk accelerator`](/command/disk-accelerator) +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator-status.md b/docs/docs/command/disk-accelerator-status.md index 8ae8b3950..82cb1d270 100644 --- a/docs/docs/command/disk-accelerator-status.md +++ b/docs/docs/command/disk-accelerator-status.md @@ -36,4 +36,4 @@ miren disk accelerator status ## See also -- [`miren disk accelerator`](/command/disk-accelerator) +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator-uninstall.md b/docs/docs/command/disk-accelerator-uninstall.md index 778a9911d..4e3655ebd 100644 --- a/docs/docs/command/disk-accelerator-uninstall.md +++ b/docs/docs/command/disk-accelerator-uninstall.md @@ -34,4 +34,4 @@ sudo miren disk accelerator uninstall ## See also -- [`miren disk accelerator`](/command/disk-accelerator) +- [`miren disk accelerator`](./disk-accelerator.md) diff --git a/docs/docs/command/disk-accelerator.md b/docs/docs/command/disk-accelerator.md index d99bfaac1..8b18f05be 100644 --- a/docs/docs/command/disk-accelerator.md +++ b/docs/docs/command/disk-accelerator.md @@ -63,10 +63,10 @@ miren disk accelerator [flags] ## Subcommands -- [`miren disk accelerator install`](/command/disk-accelerator-install) — Build and load the lbd kernel module for this kernel -- [`miren disk accelerator status`](/command/disk-accelerator-status) — Show whether accelerator mode can run on this host -- [`miren disk accelerator uninstall`](/command/disk-accelerator-uninstall) — Unload and remove the lbd kernel module +- [`miren disk accelerator install`](./disk-accelerator-install.md) — Build and load the lbd kernel module for this kernel +- [`miren disk accelerator status`](./disk-accelerator-status.md) — Show whether accelerator mode can run on this host +- [`miren disk accelerator uninstall`](./disk-accelerator-uninstall.md) — Unload and remove the lbd kernel module ## See also -- [`miren disk`](/command/disk) +- [`miren disk`](./disk.md) diff --git a/docs/docs/command/disk.md b/docs/docs/command/disk.md index b31b81d5d..db909da97 100644 --- a/docs/docs/command/disk.md +++ b/docs/docs/command/disk.md @@ -1,12 +1,12 @@ --- title: "miren disk" sidebar_label: "disk" -description: "Disk backup and recovery" +description: "Disk backup, recovery, and acceleration" --- # miren disk -Disk backup and recovery +Disk backup, recovery, and acceleration ## Usage @@ -16,6 +16,7 @@ miren disk [flags] ## Subcommands +- [`miren disk accelerator`](./disk-accelerator.md) — Faster block-device disks via the lbd kernel module - [`miren disk backup`](./disk-backup.md) — Backup a disk to a snapshot file - [`miren disk list-deleted`](./disk-list-deleted.md) — List deleted disks available for recovery - [`miren disk restore`](./disk-restore.md) — Restore a disk from a snapshot file diff --git a/docs/docs/commands.md b/docs/docs/commands.md index 3fff56408..42d12993a 100644 --- a/docs/docs/commands.md +++ b/docs/docs/commands.md @@ -114,7 +114,11 @@ Complete reference for all `miren` CLI commands. | Command | Description | |---------|-------------| -| [`miren disk`](./command/disk.md) | Disk backup and recovery | +| [`miren disk`](./command/disk.md) | Disk backup, recovery, and acceleration | +| [`miren disk accelerator`](./command/disk-accelerator.md) | Faster block-device disks via the lbd kernel module | +| [`miren disk accelerator install`](./command/disk-accelerator-install.md) | Build and load the lbd kernel module for this kernel | +| [`miren disk accelerator status`](./command/disk-accelerator-status.md) | Show whether accelerator mode can run on this host | +| [`miren disk accelerator uninstall`](./command/disk-accelerator-uninstall.md) | Unload and remove the lbd kernel module | | [`miren disk backup`](./command/disk-backup.md) | Backup a disk to a snapshot file | | [`miren disk list-deleted`](./command/disk-list-deleted.md) | List deleted disks available for recovery | | [`miren disk restore`](./command/disk-restore.md) | Restore a disk from a snapshot file | diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md index 0417a9c29..11b621f58 100644 --- a/docs/docs/disk-accelerator.md +++ b/docs/docs/disk-accelerator.md @@ -6,7 +6,7 @@ keywords: [accelerator, lbd, kernel module, disk performance, disk mode, loop de # Disk Accelerator -Miren serves [Miren Disks](/disks#miren-disks) in one of two modes, and picks +Miren serves [Miren Disks](./disks.md#miren-disks) in one of two modes, and picks between them on its own. **Universal mode** is the default and works everywhere. It backs each disk with a @@ -131,7 +131,7 @@ disk_mode = "universal" # or "accelerator", or "auto" (the default) `universal` forces loop devices even where the module is loaded. `accelerator` insists on `lbd`, and disks will fail to attach if it is not there — useful when you would rather find out loudly than quietly run slower. See -[Server Configuration](/server-config). +[Server Configuration](./server-config.md). ## Turning it off diff --git a/docs/docs/disks.md b/docs/docs/disks.md index 5a6d17aab..b090245fd 100644 --- a/docs/docs/disks.md +++ b/docs/docs/disks.md @@ -103,7 +103,7 @@ Miren Disks live on your server. Back up important data with `miren disk backup` Miren Disks provide managed persistent storage for your applications. Disks are provisioned with a specific size and filesystem, support exclusive leasing for data consistency, and persist across app restarts and redeployments. -They work out of the box on any Linux host. For faster writes, [accelerator mode](/disk-accelerator) builds and loads a Miren kernel module tuned for this job; it is one command, and disks fall back to the default cleanly if you skip it. +They work out of the box on any Linux host. For faster writes, [accelerator mode](./disk-accelerator.md) builds and loads a Miren kernel module tuned for this job; it is one command, and disks fall back to the default cleanly if you skip it. ### Why Use Disks? diff --git a/servers/build/lbd_builder.go b/servers/build/lbd_builder.go index 074288b27..401ff5b96 100644 --- a/servers/build/lbd_builder.go +++ b/servers/build/lbd_builder.go @@ -3,12 +3,15 @@ package build import ( "context" "fmt" + "log/slog" "os" "github.com/tonistiigi/fsutil" "miren.dev/runtime/api/core/core_v1alpha" + "miren.dev/runtime/api/entityserver" "miren.dev/runtime/components/ocireg" "miren.dev/runtime/pkg/lbdmod" + "miren.dev/runtime/pkg/workloadidentity" ) // lbdBuilderLock serializes the toolchain build within one coordinator. Two @@ -20,6 +23,16 @@ import ( // cancellable. var lbdBuilderLock = make(chan struct{}, 1) +// LbdToolchain builds the image before workload control starts; it cannot +// depend on the app builder, which is initialized after runner endpoints. +type LbdToolchain struct { + Log *slog.Logger + BuildKit BuildKitProvider + Issuer *workloadidentity.Issuer + EC *entityserver.Client + TempDir string +} + // EnsureLbdBuilderImage makes sure the lbd toolchain image is in the cluster // registry and returns the reference nodes should pull. // @@ -30,7 +43,7 @@ var lbdBuilderLock = make(chan struct{}, 1) // // It is keyed by a content hash of the embedded Dockerfile and build script, so // this is a no-op on every call after the first until that content changes. -func (b *Builder) EnsureLbdBuilderImage(ctx context.Context) (string, error) { +func (b *LbdToolchain) EnsureLbdBuilderImage(ctx context.Context) (string, error) { ref := lbdmod.BuilderImage(ocireg.Host) if b.present(ctx) { @@ -79,7 +92,7 @@ func (b *Builder) EnsureLbdBuilderImage(ctx context.Context) (string, error) { b.Log.Info("building the lbd toolchain image", "image", ref) - bk := &Buildkit{Client: bkc, Log: b.Log, WorkloadIssuer: b.WorkloadIssuer} + bk := &Buildkit{Client: bkc, Log: b.Log, WorkloadIssuer: b.Issuer} res, err := bk.BuildImage(ctx, dfs, BuildStack{ Stack: "dockerfile", Input: lbdmod.BuilderDockerfile, @@ -95,13 +108,13 @@ func (b *Builder) EnsureLbdBuilderImage(ctx context.Context) (string, error) { // present reports whether the toolchain image for this content hash is already // in the registry. An artifact's entity name is the tag it was pushed under, // so the tag is the lookup key. -func (b *Builder) present(ctx context.Context) bool { - if b.ec == nil { +func (b *LbdToolchain) present(ctx context.Context) bool { + if b.EC == nil { return false } var artifact core_v1alpha.Artifact - if err := b.ec.Get(ctx, lbdmod.BuilderTag(), &artifact); err != nil { + if err := b.EC.Get(ctx, lbdmod.BuilderTag(), &artifact); err != nil { return false } // An archived artifact has had, or is about to have, its blobs collected, diff --git a/servers/build/lbd_builder_test.go b/servers/build/lbd_builder_test.go index 7b3fc876e..ef580402c 100644 --- a/servers/build/lbd_builder_test.go +++ b/servers/build/lbd_builder_test.go @@ -28,7 +28,7 @@ func TestEnsureLbdBuilderImageSkipsAnExistingImage(t *testing.T) { // BuildKit is deliberately nil: finding the image must short-circuit // before anything tries to build, or every install would rebuild. - b := &Builder{Log: log, EAS: inmem.EAC, ec: entityserver.NewClient(log, inmem.EAC)} + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} ref, err := b.EnsureLbdBuilderImage(ctx) require.NoError(t, err) @@ -48,7 +48,7 @@ func TestEnsureLbdBuilderImageRebuildsAnArchivedImage(t *testing.T) { &core_v1alpha.Artifact{Status: core_v1alpha.ARCHIVED}) require.NoError(t, err) - b := &Builder{Log: log, EAS: inmem.EAC, ec: entityserver.NewClient(log, inmem.EAC)} + b := &LbdToolchain{Log: log, EC: entityserver.NewClient(log, inmem.EAC)} _, err = b.EnsureLbdBuilderImage(ctx) require.Error(t, err) @@ -89,7 +89,7 @@ func TestEnsureLbdBuilderImageNeedsBuildkit(t *testing.T) { inmem, cleanup := testutils.NewInMemEntityServer(t) defer cleanup() - b := &Builder{Log: testutils.TestLogger(t), EAS: inmem.EAC, ec: entityserver.NewClient(testutils.TestLogger(t), inmem.EAC)} + b := &LbdToolchain{Log: testutils.TestLogger(t), EC: entityserver.NewClient(testutils.TestLogger(t), inmem.EAC)} _, err := b.EnsureLbdBuilderImage(ctx) require.Error(t, err) From 91d656f07fa3347794c984c7c1ca92afa601f0cf Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 17:12:37 +0000 Subject: [PATCH 12/15] Prepare runner network and build dependencies before lbd rebuild --- .../distributedrunner/boot_node_storage.go | 8 +- .../distributedrunner/boot_sandbox_host.go | 94 ++++++++++++------- .../boot_sandbox_host_test.go | 26 +++-- components/distributedrunner/startup.go | 7 +- components/distributedrunner/startup_test.go | 1 + components/runner/lbd.go | 4 + components/runner/lbd_test.go | 24 +++++ components/runner/runner.go | 44 ++++----- components/server/boot_node_storage.go | 15 ++- components/server/startup.go | 2 +- 10 files changed, 151 insertions(+), 74 deletions(-) create mode 100644 components/runner/lbd_test.go diff --git a/components/distributedrunner/boot_node_storage.go b/components/distributedrunner/boot_node_storage.go index 82c1ef0f2..950e002aa 100644 --- a/components/distributedrunner/boot_node_storage.go +++ b/components/distributedrunner/boot_node_storage.go @@ -15,17 +15,19 @@ type nodeStorageBoot struct { output boot.Output[*runner.NodeStorage] } -func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], telemetry boot.Output[telemetryBootOutput]) *nodeStorageBoot { +func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], telemetry boot.Output[telemetryBootOutput], containerd boot.Output[containerdBootOutput], networkDeps boot.Output[runner.RunnerDeps]) *nodeStorageBoot { b := &nodeStorageBoot{} - b.component, b.output = boot.Provide2("node-storage", access, telemetry, b.start, + b.component, b.output = boot.Provide4("node-storage", access, telemetry, containerd, networkDeps, b.start, boot.WithStop(b.stop, 0)) return b } -func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, telemetry telemetryBootOutput) (*runner.NodeStorage, error) { +func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, telemetry telemetryBootOutput, containerd containerdBootOutput, networkDeps runner.RunnerDeps) (*runner.NodeStorage, error) { var err error b.value, err = runner.NewNodeStorage(access.access, runner.RunnerDeps{ MetricsWriter: telemetry.metricsWriter, + CC: containerd.Client, + Resolver: networkDeps.Resolver, }, access.config) if err != nil { return nil, err diff --git a/components/distributedrunner/boot_sandbox_host.go b/components/distributedrunner/boot_sandbox_host.go index 48cf43b29..4e2b7786b 100644 --- a/components/distributedrunner/boot_sandbox_host.go +++ b/components/distributedrunner/boot_sandbox_host.go @@ -38,6 +38,35 @@ type sandboxHostBoot struct { output boot.Output[*runner.SandboxHost] } +// networkDepsBoot maps the cluster registry before storage can rebuild lbd, +// and shares the same resolver with the later sandbox host. +type networkDepsBoot struct { + component *boot.Component + inputs sandboxHostBootInputs + output boot.Output[runner.RunnerDeps] +} + +func newNetworkDepsBoot(inputs sandboxHostBootInputs, access boot.Output[clusterAccessBootOutput]) *networkDepsBoot { + b := &networkDepsBoot{inputs: inputs} + b.component, b.output = boot.Provide1("runner-network-deps", access, b.start) + return b +} + +func (b *networkDepsBoot) start(ctx context.Context, access clusterAccessBootOutput) (runner.RunnerDeps, error) { + var deps runner.RunnerDeps + if err := b.inputs.prepareNetworkDeps(&deps, access.access.CoordinatorInternalIP()); err != nil { + return runner.RunnerDeps{}, err + } + deps.EtcdEndpoints = append([]string(nil), b.inputs.etcdEndpoints...) + deps.EtcdPrefix = b.inputs.etcdPrefix + if len(deps.EtcdEndpoints) > 0 { + if err := runner.InitializeDistributedNetwork(ctx, b.inputs.log, b.inputs.dataPath, &deps, b.inputs.group); err != nil { + return runner.RunnerDeps{}, fmt.Errorf("initializing network before node storage: %w", err) + } + } + return deps, nil +} + func sandboxHostInputs(options StartOptions) sandboxHostBootInputs { return sandboxHostBootInputs{ log: options.Log, @@ -59,10 +88,11 @@ func newSandboxHostBoot( storage boot.Output[*runner.NodeStorage], containerd boot.Output[containerdBootOutput], telemetry boot.Output[telemetryBootOutput], + networkDeps boot.Output[runner.RunnerDeps], ) *sandboxHostBoot { b := &sandboxHostBoot{inputs: inputs} - b.component, b.output = boot.Provide4( - "sandbox-host", access, storage, containerd, telemetry, b.start, + b.component, b.output = boot.Provide5( + "sandbox-host", access, storage, containerd, telemetry, networkDeps, b.start, boot.WithStop(b.stop, 0), ) return b @@ -74,27 +104,19 @@ func (b *sandboxHostBoot) start( storage *runner.NodeStorage, containerd containerdBootOutput, telemetry telemetryBootOutput, + dependencies runner.RunnerDeps, ) (*runner.SandboxHost, error) { - dependencies := runner.RunnerDeps{ - CC: containerd.Client, - Namespace: containerd.Namespace, - Bridge: "rt0", - Tempdir: os.TempDir(), - - DisableLocalNet: true, - LogsMaintainer: observability.NewLogsMaintainer(), - LogWriter: telemetry.logWriter, - StatusMon: observability.NewStatusMonitor(b.inputs.log), - SandboxMetrics: telemetry.sandboxMetrics, - MetricsWriter: telemetry.metricsWriter, - ServicePrefixes: b.inputs.servicePrefixes, - - EtcdEndpoints: append([]string(nil), b.inputs.etcdEndpoints...), - EtcdPrefix: b.inputs.etcdPrefix, - } - if err := b.prepareNetworkDeps(&dependencies, access.access.CoordinatorInternalIP()); err != nil { - return nil, err - } + dependencies.CC = containerd.Client + dependencies.Namespace = containerd.Namespace + dependencies.Bridge = "rt0" + dependencies.Tempdir = os.TempDir() + dependencies.DisableLocalNet = true + dependencies.LogsMaintainer = observability.NewLogsMaintainer() + dependencies.LogWriter = telemetry.logWriter + dependencies.StatusMon = observability.NewStatusMonitor(b.inputs.log) + dependencies.SandboxMetrics = telemetry.sandboxMetrics + dependencies.MetricsWriter = telemetry.metricsWriter + dependencies.ServicePrefixes = b.inputs.servicePrefixes var err error b.value, err = runner.NewSandboxHost(access.access, storage, dependencies, access.config) @@ -107,54 +129,54 @@ func (b *sandboxHostBoot) start( return b.value, nil } -func (b *sandboxHostBoot) prepareNetworkDeps(deps *runner.RunnerDeps, coordinatorInternalIP netip.Addr) error { +func (i sandboxHostBootInputs) prepareNetworkDeps(deps *runner.RunnerDeps, coordinatorInternalIP netip.Addr) error { resolver, hostMapper := netresolve.NewLocalResolver() deps.Resolver = resolver if coordinatorInternalIP.Is4() { if err := hostMapper.SetHost("cluster.local", coordinatorInternalIP); err != nil { return fmt.Errorf("mapping cluster registry: %w", err) } - b.inputs.log.Info("mapped cluster.local to coordinator WireGuard gateway", "addr", coordinatorInternalIP) + i.log.Info("mapped cluster.local to coordinator WireGuard gateway", "addr", coordinatorInternalIP) } - coordinatorHost, coordinatorPort, splitErr := net.SplitHostPort(b.inputs.coordinator) + coordinatorHost, coordinatorPort, splitErr := net.SplitHostPort(i.coordinator) if splitErr != nil { - b.inputs.log.Warn("in-cluster API access disabled: coordinator address has no usable host and port", - "coordinator", b.inputs.coordinator, "error", splitErr) + i.log.Warn("in-cluster API access disabled: coordinator address has no usable host and port", + "coordinator", i.coordinator, "error", splitErr) } else if coordinatorAddr, err := resolveHost(coordinatorHost); err != nil { - b.inputs.log.Warn("could not resolve coordinator address", "host", coordinatorHost, "error", err) + i.log.Warn("could not resolve coordinator address", "host", coordinatorHost, "error", err) } else { // Sandboxes reach the API on the coordinator rather than the local bridge // router. This must be an IP because sandbox DNS resolves app.miren names // and nothing else, so the coordinator hostname would not resolve there. deps.ApiAddress = net.JoinHostPort(coordinatorAddr.String(), coordinatorPort) - deps.CACert = []byte(b.inputs.caCert) - b.inputs.log.Info("sandboxes will reach the cluster API at", "address", deps.ApiAddress) + deps.CACert = []byte(i.caCert) + i.log.Info("sandboxes will reach the cluster API at", "address", deps.ApiAddress) if !coordinatorInternalIP.IsValid() { // Older coordinators serve the registry on the same address as the API. if err := hostMapper.SetHost("cluster.local", coordinatorAddr); err != nil { return fmt.Errorf("mapping legacy cluster registry: %w", err) } - b.inputs.log.Warn("coordinator did not advertise an internal address; using its API address for registry pulls", "addr", coordinatorAddr) + i.log.Warn("coordinator did not advertise an internal address; using its API address for registry pulls", "addr", coordinatorAddr) } } - if b.inputs.clientCert == "" || b.inputs.clientKey == "" || b.inputs.caCert == "" { + if i.clientCert == "" || i.clientKey == "" || i.caCert == "" { return nil } - etcdCertsDir := filepath.Join(b.inputs.dataPath, "etcd-certs") + etcdCertsDir := filepath.Join(i.dataPath, "etcd-certs") if err := os.MkdirAll(etcdCertsDir, 0700); err != nil { return fmt.Errorf("creating etcd certs directory: %w", err) } deps.EtcdTLSCertFile = filepath.Join(etcdCertsDir, "client.crt") deps.EtcdTLSKeyFile = filepath.Join(etcdCertsDir, "client.key") deps.EtcdTLSCAFile = filepath.Join(etcdCertsDir, "ca.crt") - if err := os.WriteFile(deps.EtcdTLSCertFile, []byte(b.inputs.clientCert), 0644); err != nil { + if err := os.WriteFile(deps.EtcdTLSCertFile, []byte(i.clientCert), 0644); err != nil { return fmt.Errorf("writing etcd client cert: %w", err) } - if err := os.WriteFile(deps.EtcdTLSKeyFile, []byte(b.inputs.clientKey), 0600); err != nil { + if err := os.WriteFile(deps.EtcdTLSKeyFile, []byte(i.clientKey), 0600); err != nil { return fmt.Errorf("writing etcd client key: %w", err) } - if err := os.WriteFile(deps.EtcdTLSCAFile, []byte(b.inputs.caCert), 0644); err != nil { + if err := os.WriteFile(deps.EtcdTLSCAFile, []byte(i.caCert), 0644); err != nil { return fmt.Errorf("writing etcd CA cert: %w", err) } return nil diff --git a/components/distributedrunner/boot_sandbox_host_test.go b/components/distributedrunner/boot_sandbox_host_test.go index 70d49b551..0d048c7b8 100644 --- a/components/distributedrunner/boot_sandbox_host_test.go +++ b/components/distributedrunner/boot_sandbox_host_test.go @@ -8,14 +8,28 @@ import ( "github.com/stretchr/testify/require" "miren.dev/runtime/components/runner" + "miren.dev/runtime/pkg/boot" ) +func TestNetworkDepsReadyBeforeNodeStorage(t *testing.T) { + access := &runner.ClusterAccess{} + boot := newNetworkDepsBoot(sandboxHostBootInputs{ + log: testLogger(), coordinator: "198.51.100.9:8443", + }, boot.ResolvedOutput(clusterAccessBootOutput{access: access})) + deps, err := boot.start(t.Context(), clusterAccessBootOutput{access: access}) + require.NoError(t, err) + require.NotNil(t, deps.Resolver) + addr, err := deps.Resolver.LookupHost("cluster.local") + require.NoError(t, err) + require.Equal(t, netip.MustParseAddr("198.51.100.9"), addr) +} + func TestRegistryResolvesOverWireGuardNotPublicCoordinator(t *testing.T) { - boot := &sandboxHostBoot{inputs: sandboxHostBootInputs{ + inputs := sandboxHostBootInputs{ log: testLogger(), coordinator: "198.51.100.9:8443", - }} + } var deps runner.RunnerDeps - require.NoError(t, boot.prepareNetworkDeps(&deps, netip.MustParseAddr("10.8.42.1"))) + require.NoError(t, inputs.prepareNetworkDeps(&deps, netip.MustParseAddr("10.8.42.1"))) addr, err := deps.Resolver.LookupHost("cluster.local") require.NoError(t, err) require.Equal(t, netip.MustParseAddr("10.8.42.1"), addr) @@ -23,11 +37,11 @@ func TestRegistryResolvesOverWireGuardNotPublicCoordinator(t *testing.T) { } func TestLegacyCoordinatorRegistryUsesAPIAddress(t *testing.T) { - boot := &sandboxHostBoot{inputs: sandboxHostBootInputs{ + inputs := sandboxHostBootInputs{ log: testLogger(), coordinator: "198.51.100.9:8443", - }} + } var deps runner.RunnerDeps - require.NoError(t, boot.prepareNetworkDeps(&deps, netip.Addr{})) + require.NoError(t, inputs.prepareNetworkDeps(&deps, netip.Addr{})) addr, err := deps.Resolver.LookupHost("cluster.local") require.NoError(t, err) require.Equal(t, netip.MustParseAddr("198.51.100.9"), addr) diff --git a/components/distributedrunner/startup.go b/components/distributedrunner/startup.go index f12e6ed4e..9967a77ef 100644 --- a/components/distributedrunner/startup.go +++ b/components/distributedrunner/startup.go @@ -16,6 +16,7 @@ type startup struct { runtime *Runtime containerd *containerdcomp.Boot clusterAccess *clusterAccessBoot + networkDeps *networkDepsBoot lifecycle *lifecycleBoot nodeStorage *nodeStorageBoot telemetry *telemetryBoot @@ -28,15 +29,17 @@ type startup struct { func newStartup(runtime *Runtime, options StartOptions) *startup { containerd := containerdcomp.NewBoot("containerd", containerdBootConfig(options)) clusterAccess := newClusterAccessBoot(clusterAccessInputs(options)) + networkDeps := newNetworkDepsBoot(sandboxHostInputs(options), clusterAccess.output) lifecycle := newLifecycleBoot(options.Log, runtime.instance, clusterAccess.output) telemetry := newTelemetryBoot(telemetryInputs(options), clusterAccess.output) - nodeStorage := newNodeStorageBoot(clusterAccess.output, telemetry.output) + nodeStorage := newNodeStorageBoot(clusterAccess.output, telemetry.output, containerd.Output, networkDeps.output) sandboxHost := newSandboxHostBoot( sandboxHostInputs(options), clusterAccess.output, nodeStorage.output, containerd.Output, telemetry.output, + networkDeps.output, ) storageAgent := runnercomp.NewStorageAgentBoot(nodeStorage.output, sandboxHost.component, 0) sandboxAgent := runnercomp.NewSandboxAgentBoot(sandboxHost.output, 0) @@ -46,6 +49,7 @@ func newStartup(runtime *Runtime, options StartOptions) *startup { runtime: runtime, containerd: containerd, clusterAccess: clusterAccess, + networkDeps: networkDeps, lifecycle: lifecycle, nodeStorage: nodeStorage, telemetry: telemetry, @@ -60,6 +64,7 @@ func (s *startup) addComponents() error { components := []*boot.Component{ s.containerd.Component, s.clusterAccess.component, + s.networkDeps.component, s.lifecycle.component, s.nodeStorage.component, s.telemetry.component, diff --git a/components/distributedrunner/startup_test.go b/components/distributedrunner/startup_test.go index c08ef5948..9d692f371 100644 --- a/components/distributedrunner/startup_test.go +++ b/components/distributedrunner/startup_test.go @@ -43,6 +43,7 @@ func TestSandboxHostDoesNotStartAfterContainerdFailure(t *testing.T) { boot.ResolvedOutput((*runner.NodeStorage)(nil)), containerdOutput, boot.ResolvedOutput(telemetryBootOutput{}), + boot.ResolvedOutput(runner.RunnerDeps{}), ) graph := boot.NewGraph() diff --git a/components/runner/lbd.go b/components/runner/lbd.go index 9d569665c..8cfb58d2a 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -54,6 +54,10 @@ func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { if err := diskio.EnsureLbdDevices(ctx, log); err == nil { return } + if deps.CC == nil || deps.Resolver == nil { + log.Warn("cannot rebuild the lbd kernel module without containerd and a cluster registry resolver; disks will use loop devices") + return + } // The toolchain image lives in the cluster registry and nowhere public, so // the pull needs the cluster's own address mapping and a registry token. diff --git a/components/runner/lbd_test.go b/components/runner/lbd_test.go new file mode 100644 index 000000000..ad0f6a998 --- /dev/null +++ b/components/runner/lbd_test.go @@ -0,0 +1,24 @@ +package runner + +import ( + "testing" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/stretchr/testify/require" + "miren.dev/runtime/components/netresolve" +) + +func TestNodeStorageLbdDeps(t *testing.T) { + client := &containerd.Client{} + resolver, _ := netresolve.NewLocalResolver() + dataPath := t.TempDir() + storage, err := NewNodeStorage(&ClusterAccess{}, RunnerDeps{ + CC: client, Resolver: resolver, + }, RunnerConfig{DataPath: dataPath}) + require.NoError(t, err) + + deps := storage.lbdDeps() + require.Same(t, client, deps.CC) + require.Same(t, resolver, deps.Resolver) + require.Equal(t, dataPath, deps.DataPath) +} diff --git a/components/runner/runner.go b/components/runner/runner.go index f9e6fb331..12b44fae8 100644 --- a/components/runner/runner.go +++ b/components/runner/runner.go @@ -529,8 +529,8 @@ func (r *SandboxHost) Start(ctx context.Context, eg ...*errgroup.Group) error { r.Log.Info("starting sandbox host", "id", r.Id) // Initialize Flannel/WireGuard network if distributed runner configuration is provided - if len(r.deps.EtcdEndpoints) > 0 { - if err := r.initializeNetwork(ctx, eg...); err != nil { + if len(r.deps.EtcdEndpoints) > 0 && r.deps.Subnet == nil { + if err := InitializeDistributedNetwork(ctx, r.Log, r.DataPath, &r.deps, eg...); err != nil { return fmt.Errorf("failed to initialize network: %w", err) } } @@ -754,28 +754,28 @@ func (r *ClusterAccess) setupRemoteSecrets(rs *rpc.State) error { return nil } -// initializeNetwork sets up the Flannel network for distributed runners. -// This is only called when EtcdEndpoints are configured (distributed runner mode). -func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Group) error { - r.Log.Info("Initializing distributed runner network", - "etcd_endpoints", r.deps.EtcdEndpoints, - "etcd_prefix", r.deps.EtcdPrefix) +// InitializeDistributedNetwork joins the mesh before storage tries to pull +// the lbd builder image from the coordinator's WireGuard-routed registry. +func InitializeDistributedNetwork(ctx context.Context, log *slog.Logger, dataPath string, deps *RunnerDeps, eg ...*errgroup.Group) error { + log.Info("Initializing distributed runner network", + "etcd_endpoints", deps.EtcdEndpoints, + "etcd_prefix", deps.EtcdPrefix) grungeOpts := grunge.NetworkOptions{ - EtcdEndpoints: r.deps.EtcdEndpoints, - EtcdPrefix: r.deps.EtcdPrefix, - PrevIPv4: r.deps.IPv4Routable, + EtcdEndpoints: deps.EtcdEndpoints, + EtcdPrefix: deps.EtcdPrefix, + PrevIPv4: deps.IPv4Routable, } // Add TLS config if provided - if r.deps.EtcdTLSCertFile != "" && r.deps.EtcdTLSKeyFile != "" && r.deps.EtcdTLSCAFile != "" { - r.Log.Info("Using etcd TLS", "cert", r.deps.EtcdTLSCertFile, "ca", r.deps.EtcdTLSCAFile) - grungeOpts.TLSCertFile = r.deps.EtcdTLSCertFile - grungeOpts.TLSKeyFile = r.deps.EtcdTLSKeyFile - grungeOpts.TLSCAFile = r.deps.EtcdTLSCAFile + if deps.EtcdTLSCertFile != "" && deps.EtcdTLSKeyFile != "" && deps.EtcdTLSCAFile != "" { + log.Info("Using etcd TLS", "cert", deps.EtcdTLSCertFile, "ca", deps.EtcdTLSCAFile) + grungeOpts.TLSCertFile = deps.EtcdTLSCertFile + grungeOpts.TLSKeyFile = deps.EtcdTLSKeyFile + grungeOpts.TLSCAFile = deps.EtcdTLSCAFile } - gn, err := grunge.NewNetwork(r.Log, grungeOpts) + gn, err := grunge.NewNetwork(log, grungeOpts) if err != nil { return fmt.Errorf("failed to create grunge network: %w", err) } @@ -799,18 +799,18 @@ func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Gro if localGroup { go func() { if err := runGroup.Wait(); err != nil { - r.Log.Error("network errgroup failed", "error", err) + log.Error("network errgroup failed", "error", err) } }() } // Update deps with the leased IP and subnet lease := gn.Lease() - r.deps.IPv4Routable = lease.IPv4() + deps.IPv4Routable = lease.IPv4() // Initialize netdb subnet from the flannel lease so the sandbox // controller can allocate IPs within this runner's subnet. - ndb, err := netdb.New(filepath.Join(r.DataPath, "net.db")) + ndb, err := netdb.New(filepath.Join(dataPath, "net.db")) if err != nil { return fmt.Errorf("failed to open netdb: %w", err) } @@ -818,9 +818,9 @@ func (r *SandboxHost) initializeNetwork(ctx context.Context, eg ...*errgroup.Gro if err != nil { return fmt.Errorf("failed to create subnet from lease: %w", err) } - r.deps.Subnet = subnet + deps.Subnet = subnet - r.Log.Info("Joined Flannel network", "ipv4", lease.IPv4().String()) + log.Info("Joined Flannel network", "ipv4", lease.IPv4().String()) return nil } diff --git a/components/server/boot_node_storage.go b/components/server/boot_node_storage.go index 443695d3f..9293ebe23 100644 --- a/components/server/boot_node_storage.go +++ b/components/server/boot_node_storage.go @@ -6,6 +6,7 @@ import ( "context" "miren.dev/runtime/components/coordinate" + "miren.dev/runtime/components/netresolve" "miren.dev/runtime/components/runner" "miren.dev/runtime/pkg/boot" ) @@ -14,18 +15,20 @@ type nodeStorageBoot struct { component *boot.Component value *runner.NodeStorage output boot.Output[*runner.NodeStorage] + resolver netresolve.Resolver } -func newNodeStorageBoot(access boot.Output[clusterAccessBootOutput], registration boot.Output[registrationBootOutput], observability boot.Output[observabilityBootOutput]) *nodeStorageBoot { - b := &nodeStorageBoot{} - b.component, b.output = boot.Provide3( - "node-storage", access, registration, observability, b.start, +func newNodeStorageBoot(resolver netresolve.Resolver, mapping *boot.Component, access boot.Output[clusterAccessBootOutput], registration boot.Output[registrationBootOutput], observability boot.Output[observabilityBootOutput], containerd boot.Output[containerdBootOutput]) *nodeStorageBoot { + b := &nodeStorageBoot{resolver: resolver} + b.component, b.output = boot.Provide4( + "node-storage", access, registration, observability, containerd, b.start, + boot.DependsOn(mapping), boot.WithStop(b.stop, runnerComponentStopTimeout), ) return b } -func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, registration registrationBootOutput, observability observabilityBootOutput) (*runner.NodeStorage, error) { +func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOutput, registration registrationBootOutput, observability observabilityBootOutput, containerd containerdBootOutput) (*runner.NodeStorage, error) { config := access.config cloudAuth := registration.cloudAuth if cloudAuth.Enabled { @@ -37,6 +40,8 @@ func (b *nodeStorageBoot) start(ctx context.Context, access clusterAccessBootOut b.value, err = runner.NewNodeStorage(access.access, runner.RunnerDeps{ IsCoordinator: true, MetricsWriter: observability.operationalMetrics, + CC: containerd.Client, + Resolver: b.resolver, }, config) if err != nil { return nil, err diff --git a/components/server/startup.go b/components/server/startup.go index 311522d2d..c7ee3d678 100644 --- a/components/server/startup.go +++ b/components/server/startup.go @@ -140,7 +140,7 @@ func newStartup(runtime *Runtime, options StartOptions) *startup { observability.output, runnerEndpoints.component, ) - nodeStorage := newNodeStorageBoot(clusterAccess.output, registration.output, observability.output) + nodeStorage := newNodeStorageBoot(resolver, registryHostMapping.component, clusterAccess.output, registration.output, observability.output, containerd.Output) sandboxHost := newSandboxHostBoot( sandboxHostInputs(options, resolver, serverPort(options.Log, address)), clusterAccess.output, From 88fea00e7716228c02d3cd359fcb1b4a09c481ed Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 18:34:18 +0000 Subject: [PATCH 13/15] Require an install record before skipping LBD setup --- pkg/lbdmod/build.go | 4 ++-- pkg/lbdmod/build_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go index 2deff5fe3..0877b2be2 100644 --- a/pkg/lbdmod/build.go +++ b/pkg/lbdmod/build.go @@ -71,7 +71,7 @@ func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { return status, err } - if !force && status.Available() && !status.Stale() { + if !force && status.Marker != nil && status.Available() && !status.Stale() { i.Log.Info("lbd is already installed and current", "kernel", status.Host.KernelRelease) return status, nil } @@ -93,7 +93,7 @@ func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { // Another process may have finished the very build this one was about to // start while we waited to be let in. if !force { - if current, err := Probe(i.Options); err == nil && current.Available() && !current.Stale() { + if current, err := Probe(i.Options); err == nil && current.Marker != nil && current.Available() && !current.Stale() { i.Log.Info("another process installed lbd while this one waited", "kernel", current.Host.KernelRelease) return current, nil diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 1e1a364fb..2e1073db2 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -103,6 +103,34 @@ func TestCheckCanBuildAcceptsAGoodHost(t *testing.T) { require.NoError(t, testInstaller(t, root, t.TempDir()).checkCompilerAndHeaders(status)) } +func TestInstallDoesNotSkipALoadedModuleWithoutAnInstallRecord(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 0 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + status, err := Probe(i.Options) + require.NoError(t, err) + require.True(t, status.Available()) + require.Nil(t, status.Marker) + + // A load followed by a failed verification leaves no marker. It must not + // report the next install as successful without recording opt-in for rebuilds. + _, err = i.Install(t.Context(), false) + require.Error(t, err, "a missing builder must prevent completing the install") + + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: SourceVersion(), KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + _, err = i.Install(t.Context(), false) + require.NoError(t, err, "a completed install can skip the build") +} + func TestSecureBootDetection(t *testing.T) { // No EFI at all: not enforcing, rather than guessing yes and blocking a // host that simply is not using EFI. From 6bdbc553625b30a40cd003b31aa82e9c55970956 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 20:49:49 +0000 Subject: [PATCH 14/15] Rebuild stale LBD releases and create control nodes without udev --- components/diskio/disk_ops_linux.go | 9 +++++ components/runner/lbd.go | 20 ++++++----- pkg/lbdmod/build.go | 3 ++ pkg/lbdmod/build_test.go | 26 ++++++++++++++ pkg/lbdmod/probe.go | 54 +++++++++++++++++++++++++++++ pkg/lbdmod/status_test.go | 33 ++++++++++++++++++ 6 files changed, 136 insertions(+), 9 deletions(-) diff --git a/components/diskio/disk_ops_linux.go b/components/diskio/disk_ops_linux.go index 66cc8c481..0dac8b7ca 100644 --- a/components/diskio/disk_ops_linux.go +++ b/components/diskio/disk_ops_linux.go @@ -689,6 +689,15 @@ func EnsureLbdDevices(ctx context.Context, log *slog.Logger) error { if err != nil { return err } + if status.Loaded && !status.ControlDevicePresent { + if err := lbdmod.EnsureControlDevice(); err != nil { + return err + } + status, err = lbdmod.Probe(lbdmod.HostOptions("")) + if err != nil { + return err + } + } if !status.Available() { return errors.New(status.Explain()) } diff --git a/components/runner/lbd.go b/components/runner/lbd.go index 8cfb58d2a..60abbaf1f 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -51,11 +51,11 @@ const rebuildTimeout = 10 * time.Minute // Neither failing nor timing out is fatal. Universal mode works everywhere, so // the worst case is slower disks, not a runner that will not start. func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { - if err := diskio.EnsureLbdDevices(ctx, log); err == nil { - return - } + ready := diskio.EnsureLbdDevices(ctx, log) == nil if deps.CC == nil || deps.Resolver == nil { - log.Warn("cannot rebuild the lbd kernel module without containerd and a cluster registry resolver; disks will use loop devices") + if !ready { + log.Warn("cannot rebuild the lbd kernel module without containerd and a cluster registry resolver; disks will use loop devices") + } return } @@ -83,19 +83,21 @@ func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { rebuilt, err := installer.EnsureCurrent(ctx) if errors.Is(err, context.DeadlineExceeded) { - log.Warn("gave up rebuilding the lbd kernel module, disks will use loop devices", + log.Warn("gave up rebuilding the lbd kernel module", "timeout", rebuildTimeout, + "existing_module_usable", ready, "retry_with", "miren disk accelerator install") return } if err != nil { - log.Warn("could not rebuild the lbd kernel module, disks will use loop devices", "error", err) + log.Warn("could not rebuild the lbd kernel module", "error", err, "existing_module_usable", ready) return } if !rebuilt { - // Nothing to rebuild: this host never installed the module. - log.Info("accelerator mode is not enabled on this host, disks will use loop devices", - "enable_with", "miren disk accelerator install") + if !ready { + log.Info("accelerator mode is not enabled on this host, disks will use loop devices", + "enable_with", "miren disk accelerator install") + } return } diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go index 0877b2be2..44aa1747b 100644 --- a/pkg/lbdmod/build.go +++ b/pkg/lbdmod/build.go @@ -379,6 +379,9 @@ func (i *Installer) load(ctx context.Context, host Host) error { if out, err := exec.CommandContext(ctx, "modprobe", ModuleName).CombinedOutput(); err != nil { return fmt.Errorf("modprobe %s failed: %w: %s", ModuleName, err, strings.TrimSpace(string(out))) } + if err := EnsureControlDevice(); err != nil { + return fmt.Errorf("lbd control device: %w", err) + } if err := os.MkdirAll(filepath.Dir(modulesLoadConf), 0755); err != nil { return fmt.Errorf("creating %s: %w", filepath.Dir(modulesLoadConf), err) diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 2e1073db2..709efb02d 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -372,6 +372,32 @@ func TestEnsureCurrentSkipsAHealthyHost(t *testing.T) { assert.False(t, builder.called) } +func TestEnsureCurrentRebuildsALoadedModuleFromAnOlderRelease(t *testing.T) { + root := ubuntuRoot(t) + dataPath := t.TempDir() + writeFile(t, root, "proc/modules", "lbd 65536 0 - Live 0x0000000000000000\n") + writeFile(t, root, ControlDevice, "") + writeFile(t, root, modulePath(testRelease), "") + writeFile(t, root, "usr/local/bin/lbdctl", "") + require.NoError(t, writeMarker(dataPath, Marker{ + LbdVersion: "older-lbd", KernelRelease: testRelease, + ModulePath: modulePath(testRelease), + })) + + i := testInstaller(t, root, dataPath) + i.Options.SearchPath = []string{"/usr/local/bin"} + status, err := Probe(i.Options) + require.NoError(t, err) + require.True(t, status.Available()) + require.True(t, status.Stale()) + + // It must attempt an install even though the old module still works. + // The fixture has no real builder, so an error is expected, not a no-op. + rebuilt, err := i.EnsureCurrent(t.Context()) + require.False(t, rebuilt) + require.Error(t, err) +} + func TestEnsureCurrentRebuildsAfterAKernelUpgrade(t *testing.T) { root := ubuntuRoot(t) dataPath := t.TempDir() diff --git a/pkg/lbdmod/probe.go b/pkg/lbdmod/probe.go index d4dfd2649..29709a096 100644 --- a/pkg/lbdmod/probe.go +++ b/pkg/lbdmod/probe.go @@ -1,10 +1,17 @@ package lbdmod import ( + "errors" + "fmt" "os" "os/exec" "path/filepath" "slices" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" lbdsrc "miren.dev/runtime/third_party/lbd" ) @@ -35,6 +42,53 @@ type Options struct { // have to remember to add it. const systemReleasePath = "/var/lib/miren/release" +// EnsureControlDevice creates the misc device node when /dev is a private +// tmpfs without udev. The module must already be loaded so sysfs can supply +// its assigned device number. +func EnsureControlDevice() error { + return ensureControlDevice("/sys/class/misc/lbd-control/dev", ControlDevice) +} + +func ensureControlDevice(sysDev, devicePath string) error { + data, err := os.ReadFile(sysDev) + if err != nil { + return fmt.Errorf("read %s: %w", sysDev, err) + } + majorStr, minorStr, ok := strings.Cut(strings.TrimSpace(string(data)), ":") + if !ok { + return fmt.Errorf("invalid device number in %s: %q", sysDev, strings.TrimSpace(string(data))) + } + major, err := strconv.ParseUint(majorStr, 10, 32) + if err != nil { + return fmt.Errorf("invalid major number in %s: %w", sysDev, err) + } + minor, err := strconv.ParseUint(minorStr, 10, 32) + if err != nil { + return fmt.Errorf("invalid minor number in %s: %w", sysDev, err) + } + want := unix.Mkdev(uint32(major), uint32(minor)) + verify := func() error { + fi, err := os.Stat(devicePath) + if err != nil { + return err + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok || fi.Mode()&os.ModeCharDevice == 0 || uint64(st.Rdev) != want { + return fmt.Errorf("%s is not the expected character device %d:%d", devicePath, major, minor) + } + return nil + } + if err := verify(); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := unix.Mknod(devicePath, unix.S_IFCHR|0660, int(want)); err != nil && !errors.Is(err, unix.EEXIST) { + return fmt.Errorf("mknod %s (%d:%d): %w", devicePath, major, minor, err) + } + return verify() +} + // HostOptions builds the options for inspecting this host. dataPath is where // miren keeps its data; empty means DefaultDataPath. // diff --git a/pkg/lbdmod/status_test.go b/pkg/lbdmod/status_test.go index cc529d3df..2f8453a45 100644 --- a/pkg/lbdmod/status_test.go +++ b/pkg/lbdmod/status_test.go @@ -1,13 +1,16 @@ package lbdmod import ( + "errors" "os" "path/filepath" + "syscall" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" ) const testRelease = "6.8.0-51-generic" @@ -108,6 +111,36 @@ func TestLoadedWithoutControlDeviceIsNotAvailable(t *testing.T) { assert.Contains(t, status.Explain(), ControlDevice+" is missing") } +func TestEnsureControlDevice(t *testing.T) { + dir := t.TempDir() + sysDev := filepath.Join(dir, "sysdev") + device := filepath.Join(dir, "lbd-control") + require.NoError(t, os.WriteFile(sysDev, []byte("10:263\n"), 0644)) + + // Wrong existing nodes must never be overwritten, even without mknod privileges. + require.NoError(t, os.WriteFile(device, []byte("not a device"), 0644)) + require.ErrorContains(t, ensureControlDevice(sysDev, device), "not the expected character device") + require.NoError(t, os.Remove(device)) + require.NoError(t, os.WriteFile(sysDev, []byte("bad-number"), 0644)) + require.ErrorContains(t, ensureControlDevice(sysDev, device), "invalid device number") + require.NoError(t, os.WriteFile(sysDev, []byte("10:263\n"), 0644)) + + err := ensureControlDevice(sysDev, device) + if errors.Is(err, os.ErrPermission) { + t.Skip("mknod requires CAP_MKNOD") + } + require.NoError(t, err) + info, err := os.Stat(device) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeCharDevice) + require.Equal(t, unix.Mkdev(10, 263), uint64(info.Sys().(*syscall.Stat_t).Rdev)) + require.NoError(t, ensureControlDevice(sysDev, device), "an existing matching node is retained") + + require.NoError(t, os.WriteFile(sysDev, []byte("10:264\n"), 0644)) + err = ensureControlDevice(sysDev, device) + require.ErrorContains(t, err, "not the expected character device") +} + func TestStaleAfterAKernelUpgrade(t *testing.T) { root := ubuntuRoot(t) dataPath := t.TempDir() From 5603e4ebc751bd99b114a66f0432787b14b19b72 Mon Sep 17 00:00:00 2001 From: Evan Phoenix Date: Fri, 25 Sep 2026 21:49:05 +0000 Subject: [PATCH 15/15] Defer unattended upgrades of working LBD modules --- components/runner/lbd.go | 19 +++++++++++-------- docs/docs/disk-accelerator.md | 5 +++++ pkg/lbdmod/build.go | 12 ++++++++---- pkg/lbdmod/build_test.go | 11 +++++++---- pkg/lbdmod/status.go | 6 +++--- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/components/runner/lbd.go b/components/runner/lbd.go index 60abbaf1f..aa93b8480 100644 --- a/components/runner/lbd.go +++ b/components/runner/lbd.go @@ -82,15 +82,18 @@ func setupLbd(ctx context.Context, deps lbdDeps, log *slog.Logger) { defer cancel() rebuilt, err := installer.EnsureCurrent(ctx) - if errors.Is(err, context.DeadlineExceeded) { - log.Warn("gave up rebuilding the lbd kernel module", - "timeout", rebuildTimeout, - "existing_module_usable", ready, - "retry_with", "miren disk accelerator install") - return - } if err != nil { - log.Warn("could not rebuild the lbd kernel module", "error", err, "existing_module_usable", ready) + // A failed swap can unload the old module; the pre-build result is not + // evidence that accelerator mode is still available. + status, probeErr := lbdmod.Probe(lbdmod.HostOptions(deps.DataPath)) + usable := probeErr == nil && status.Available() + if errors.Is(err, context.DeadlineExceeded) { + log.Warn("gave up rebuilding the lbd kernel module", + "timeout", rebuildTimeout, "module_usable", usable, + "retry_with", "miren disk accelerator install") + } else { + log.Warn("could not rebuild the lbd kernel module", "error", err, "module_usable", usable) + } return } if !rebuilt { diff --git a/docs/docs/disk-accelerator.md b/docs/docs/disk-accelerator.md index 11b621f58..a8bde9af1 100644 --- a/docs/docs/disk-accelerator.md +++ b/docs/docs/disk-accelerator.md @@ -112,6 +112,11 @@ This only happens on hosts that installed the module in the first place. A host that never turned accelerator mode on never pays for an unattended compile at startup. +When a Miren upgrade bundles a newer `lbd` but the installed module still +works, startup leaves it running. Replacing a working module automatically +could strand disks if the new one fails to load. Drain disks from the node, +then run `miren disk accelerator install ` to upgrade it deliberately. + The startup rebuild is capped at ten minutes. If it runs over, the server logs a warning, carries on booting, and the node stays on loop devices until you run `install` yourself. Accelerator mode is a speed-up, so a build that will not diff --git a/pkg/lbdmod/build.go b/pkg/lbdmod/build.go index 44aa1747b..a2eae5bc5 100644 --- a/pkg/lbdmod/build.go +++ b/pkg/lbdmod/build.go @@ -147,9 +147,9 @@ func (i *Installer) Install(ctx context.Context, force bool) (Status, error) { return after, nil } -// EnsureCurrent rebuilds the module when this host has installed it before but -// what is on disk no longer fits -- almost always because the kernel was -// upgraded, which leaves a module that cannot load. +// EnsureCurrent rebuilds an installed module when it is no longer usable. +// An older but working module is left alone: swapping it unattended after a +// miren upgrade could strand disks if the new module fails to load. // // A host with no install record is left alone: it never opted into accelerator // mode, so it should not pay for an unattended compile at startup. It reports @@ -160,7 +160,11 @@ func (i *Installer) EnsureCurrent(ctx context.Context) (bool, error) { return false, err } - if status.Available() && !status.Stale() { + if status.Available() { + if status.Stale() { + i.Log.Warn("lbd is usable but out of date; defer the upgrade until disks can be detached", + "reason", status.Explain(), "upgrade_with", "miren disk accelerator install ") + } return false, nil } diff --git a/pkg/lbdmod/build_test.go b/pkg/lbdmod/build_test.go index 709efb02d..c0789e804 100644 --- a/pkg/lbdmod/build_test.go +++ b/pkg/lbdmod/build_test.go @@ -372,7 +372,7 @@ func TestEnsureCurrentSkipsAHealthyHost(t *testing.T) { assert.False(t, builder.called) } -func TestEnsureCurrentRebuildsALoadedModuleFromAnOlderRelease(t *testing.T) { +func TestEnsureCurrentDefersAnOlderWorkingModule(t *testing.T) { root := ubuntuRoot(t) dataPath := t.TempDir() writeFile(t, root, "proc/modules", "lbd 65536 0 - Live 0x0000000000000000\n") @@ -386,16 +386,19 @@ func TestEnsureCurrentRebuildsALoadedModuleFromAnOlderRelease(t *testing.T) { i := testInstaller(t, root, dataPath) i.Options.SearchPath = []string{"/usr/local/bin"} + builder := &fakeBuilder{} + i.Builder = builder status, err := Probe(i.Options) require.NoError(t, err) require.True(t, status.Available()) require.True(t, status.Stale()) - // It must attempt an install even though the old module still works. - // The fixture has no real builder, so an error is expected, not a no-op. + // A successful compile does not guarantee the new module will load. Keep the + // working version until an operator can drain disks and upgrade explicitly. rebuilt, err := i.EnsureCurrent(t.Context()) + require.NoError(t, err) require.False(t, rebuilt) - require.Error(t, err) + require.False(t, builder.called) } func TestEnsureCurrentRebuildsAfterAKernelUpgrade(t *testing.T) { diff --git a/pkg/lbdmod/status.go b/pkg/lbdmod/status.go index d6fcfa78c..8b60ab58a 100644 --- a/pkg/lbdmod/status.go +++ b/pkg/lbdmod/status.go @@ -85,8 +85,8 @@ func (s Status) Available() bool { // Stale reports whether this host installed the module before but what is on // disk no longer fits -- almost always because the kernel was upgraded, but -// also when miren itself now carries a newer lbd. Callers use this to decide -// whether to rebuild without being asked. +// also when miren itself now carries a newer lbd. A usable old module is +// reported as stale but is not swapped out unattended. func (s Status) Stale() bool { return s.staleReason() != "" } @@ -120,7 +120,7 @@ func (s Status) Explain() string { switch { // A loaded module can still be the wrong one -- most often after miren // was upgraded to a build carrying a newer lbd. Saying only that it is - // loaded would read as healthy while a rebuild is pending. + // loaded would hide that an operator should arrange an upgrade. case s.Available() && stale != "": return "lbd is loaded but " + stale case s.Available():