From 144c84e1eaca49410e28e3f2ded2b018ec843cca Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Tue, 4 Aug 2026 15:25:39 +0200 Subject: [PATCH 01/51] update deps --- Cargo.toml | 48 ++++++++++++++++++++++++------------------------ src/backup.rs | 6 +----- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d45a915..6a6a766 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,30 +14,30 @@ categories = ["command-line-utilities"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-trait = "0.1.88" -chrono = "0.4.41" -dyn-clone = "1.0.19" -futures = "0.3.31" -glob = "0.3.2" -log = "0.4.27" -regex = "1.11.1" +async-trait = "0.1.91" +chrono = "0.4.45" +dyn-clone = "1.0.20" +futures = "0.3.33" +glob = "0.3.4" +log = "0.4.33" +regex = "1.13.1" scopeguard = "1.2.0" -serde = { version = "1.0.219", features = ["derive"] } -shellexpand = "3.1.1" +serde = { version = "1.0.229", features = ["derive"] } +shellexpand = "3.1.2" stderrlog = "0.6.0" structopt = "0.3.26" -tempfile = "3.20.0" -tokio = { version = "1.45.1", features = ["full"] } -toml = "0.8.23" -url = { version = "2.5.4", features = ["serde"] } -which = "8.0.0" -aws-config = "1.6.3" -aws-credential-types = { version = "1.2.3", features = ["hardcoded-credentials"] } -aws-sdk-s3 = "1.91.0" -async-compression = { version = "0.4.23", default-features = false, features = ["tokio", "gzip"] } -astral-tokio-tar = "0.5.6" -tokio-cron-scheduler = "0.14.0" -cron = "0.15.0" -uuid = { version = "1.17.0", features = ["v4"] } -aws-types = "1.3.7" -croner = "2.1.0" +tempfile = "3.27.0" +tokio = { version = "1.53.1", features = ["full"] } +toml = "1.1.4" +url = { version = "2.5.8", features = ["serde"] } +which = "8.0.5" +aws-config = "1.10.1" +aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } +aws-sdk-s3 = "1.140.0" +async-compression = { version = "0.4.43", default-features = false, features = ["tokio", "gzip"] } +astral-tokio-tar = "0.6.4" +tokio-cron-scheduler = "0.15.1" +cron = "0.17.0" +uuid = { version = "1.24.0", features = ["v4"] } +aws-types = "1.5.0" +croner = "3.0.1" diff --git a/src/backup.rs b/src/backup.rs index e43782c..17344c2 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -501,11 +501,7 @@ mod tests { let cron_str = result.unwrap(); // Same configuration of tokio-cron-scheduler assert!( - Cron::new(&cron_str) - .with_seconds_required() - .with_dom_and_dow() - .parse() - .is_ok(), + Cron::from_str(&cron_str).is_ok(), "Invalid croner expression '{}' for when string '{}'", cron_str, when From 05b4d3ae7de474ba8a1258517162f36542940b88 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:09:30 +0200 Subject: [PATCH 02/51] basic zfs snapshot + send to file --- README.md | 8 ++ src/config.rs | 7 ++ src/services/mod.rs | 1 + src/services/service.rs | 4 + src/services/zfs.rs | 183 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 203 insertions(+) create mode 100644 src/services/zfs.rs diff --git a/README.md b/README.md index 2352300..474a6dc 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,14 @@ When configuring the backups, the field **when** accepts configuration strings i container_name = "docker_postgres_1" command = "pg_dumpall -c -U postgres" # dump to stdout always +[zfs] + [zfs.root] + snapshot_name = "root-fs" + dataset = "zroot" + [zfs.storage] + snapshot_name = "storage-fs" + dataset = "storage" + # mapping services to remote [backup] # Compress the DB dump and upload it to aws diff --git a/src/config.rs b/src/config.rs index d5fe230..ae6afef 100644 --- a/src/config.rs +++ b/src/config.rs @@ -66,6 +66,12 @@ pub struct DockerConfig { pub command: String, } +#[derive(Serialize, Deserialize)] +pub struct ZfsConfig { + pub dataset: String, + pub snapshot_name: String, +} + #[derive(Serialize, Deserialize)] pub struct FoldersConfig { pub pattern: String, @@ -98,6 +104,7 @@ pub struct Config { pub folders: Option>, pub postgres: Option>, pub docker: Option>, + pub zfs: Option>, // mapping pub backup: HashMap, } diff --git a/src/services/mod.rs b/src/services/mod.rs index 93b2905..25ff6ec 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -15,4 +15,5 @@ pub mod docker; pub mod folders; pub mod postgresql; +pub mod zfs; pub mod service; diff --git a/src/services/service.rs b/src/services/service.rs index 0a94319..8651253 100644 --- a/src/services/service.rs +++ b/src/services/service.rs @@ -38,6 +38,10 @@ impl Drop for Dump { #[async_trait] pub trait Service: DynClone { + + // dump executes the dump command and creates the file/files to backup. Those files will be listed by list(). async fn dump(&self) -> Result>; + + // list returns the list of the paths that should be uploaded to the Remote - those are created by dump(). async fn list(&self) -> Vec; } diff --git a/src/services/zfs.rs b/src/services/zfs.rs new file mode 100644 index 0000000..845edba --- /dev/null +++ b/src/services/zfs.rs @@ -0,0 +1,183 @@ +// src/services/zfs.rs +// Copyright 2024 Minimal ZFS Service +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; +use std::path::PathBuf; +use std::string::String; +use std::vec::Vec; + +use crate::config::ZfsConfig; +use crate::services::service::{Dump, Service}; + +use which::which; + +use async_trait::async_trait; +use tokio::{fs::metadata, fs::File, io}; + +use std::process::Stdio; +use tokio::process::Command; + +#[derive(Clone)] +pub struct Zfs { + pub name: String, + pub cmd: PathBuf, + pub args: Vec, +} + +#[derive(Debug)] +pub enum Error { + CommandNotFound(which::Error), + RuntimeError(io::Error), + ZfsError(String), +} + +impl std::error::Error for Error {} +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::CommandNotFound(error) => write!(f, "Command not found: {}", error), + Error::RuntimeError(error) => write!(f, "Runtime error: {}", error), + Error::ZfsError(error) => write!(f, "ZFS error: {}", error), + } + } +} + +impl Zfs { + pub async fn new(config: ZfsConfig, name: &str) -> Result { + let cmd = match which("zfs") { + Err(error) => return Err(Error::CommandNotFound(error)), + Ok(cmd) => cmd, + }; + + // Verify zfs is working and the current user is in the allow list for executing + // snapshot and send + let child = Command::new(&cmd) + .args(vec![String::from("allow"), config.dataset.clone()]) + // capture output to variable to check if the user is there + .stdout(Stdio::piped()).spawn(); + + if let Err(error) = child { + return Err(Error::RuntimeError(error)); + } + + let child = child.unwrap(); + let output = child.wait_with_output().await; + + if let Err(err) = output { + return Err(Error::RuntimeError(err)) + } + let output = output.unwrap(); + let status = output.status; + if !status.success() { + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}`", config.dataset))); + } + + let stdout = String::from_utf8(output.stdout).unwrap(); + if stdout.is_empty() { + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}", config.dataset))); + } + + // Check if the string user $USER send,snapshot is in the stdout + let needle = format!("user {} send,snapshot", std::env::var("USER").unwrap()); + if !stdout.contains(&needle) { + return Err(Error::ZfsError(format!("\"{}\" not found in output of `zfs allow {}`", needle, config.dataset))); + } + + // If here, the current user is in the allow list for zfs send and snapshot + + let mut args: Vec = vec![ + String::from("snapshot"), + String::from("-r"), + format!("{}@{}", config.dataset.clone().trim(),config.snapshot_name.trim()), + ]; + args.extend(config.snapshot_name.split_whitespace().map(String::from)); + + Ok(Zfs { + name: String::from(name), + args, + cmd, + }) + } +} + +#[async_trait] +impl Service for Zfs { + async fn list(&self) -> Vec { + let dest = std::env::current_dir() + .unwrap() + .join(PathBuf::from(format!("{}.snapshot", self.name))); + + if metadata(&dest).await.is_ok() { + return vec![dest]; + } + return vec![]; + } + + async fn dump(&self) -> Result> { + // Current date in ISO format + let date = chrono::Utc::now().format("%Y%m%d-%H%M%S"); + + let mut args = self.args.clone(); + let checkpoint = args.last_mut().unwrap(); + checkpoint.push_str(&format!("-{}", date)); + + let checkpoint_name = checkpoint.clone(); + + // Step 1, execute the checkpoint (atomic, immediate action) + let status = Command::new(&self.cmd) + .args(args) + .stdout(Stdio::null()) + .status() + .await; + if status.is_err() { + return Err(Error::RuntimeError(status.err().unwrap()).into()); + } + + // Step 2, send the checkpoint to a local file, named: name-date.snapshot + let dest = std::env::current_dir() + .unwrap() + .join(PathBuf::from(format!("{}-{}.snapshot", self.name, date + ))); + + let parent = dest.parent().unwrap(); + if !parent.exists() { + return Err(Error::RuntimeError(io::Error::other(format!( + "Folder {} does not exist.", + parent.display() + ))) + .into()); + } + + let dest_file = File::create(&dest).await?; + + let send_args = vec![ + self.args[0].to_string(), + String::from("send"), + String::from("-R"), + String::from("-v"), + checkpoint_name.to_string(), + ]; + + match Command::new(&self.cmd) + .args(&send_args) + .stdout(Stdio::from(dest_file.try_into_std().unwrap())) + .status() + .await + { + Ok(_) => Ok(Dump { path: Some(dest) }), + Err(error) => Err(Error::RuntimeError(error).into()), + } + } +} \ No newline at end of file From abfeff1df1c5953d3933d93fab6bbd4b836d18ff Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:12:47 +0200 Subject: [PATCH 03/51] cargo fmt --- src/services/mod.rs | 2 +- src/services/service.rs | 1 - src/services/zfs.rs | 21 ++++++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/services/mod.rs b/src/services/mod.rs index 25ff6ec..bcc2c8c 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -15,5 +15,5 @@ pub mod docker; pub mod folders; pub mod postgresql; -pub mod zfs; pub mod service; +pub mod zfs; diff --git a/src/services/service.rs b/src/services/service.rs index 8651253..45b1ae2 100644 --- a/src/services/service.rs +++ b/src/services/service.rs @@ -38,7 +38,6 @@ impl Drop for Dump { #[async_trait] pub trait Service: DynClone { - // dump executes the dump command and creates the file/files to backup. Those files will be listed by list(). async fn dump(&self) -> Result>; diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 845edba..08232bb 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -66,7 +66,8 @@ impl Zfs { let child = Command::new(&cmd) .args(vec![String::from("allow"), config.dataset.clone()]) // capture output to variable to check if the user is there - .stdout(Stdio::piped()).spawn(); + .stdout(Stdio::piped()) + .spawn(); if let Err(error) = child { return Err(Error::RuntimeError(error)); @@ -76,7 +77,7 @@ impl Zfs { let output = child.wait_with_output().await; if let Err(err) = output { - return Err(Error::RuntimeError(err)) + return Err(Error::RuntimeError(err)); } let output = output.unwrap(); let status = output.status; @@ -92,7 +93,10 @@ impl Zfs { // Check if the string user $USER send,snapshot is in the stdout let needle = format!("user {} send,snapshot", std::env::var("USER").unwrap()); if !stdout.contains(&needle) { - return Err(Error::ZfsError(format!("\"{}\" not found in output of `zfs allow {}`", needle, config.dataset))); + return Err(Error::ZfsError(format!( + "\"{}\" not found in output of `zfs allow {}`", + needle, config.dataset + ))); } // If here, the current user is in the allow list for zfs send and snapshot @@ -100,7 +104,11 @@ impl Zfs { let mut args: Vec = vec![ String::from("snapshot"), String::from("-r"), - format!("{}@{}", config.dataset.clone().trim(),config.snapshot_name.trim()), + format!( + "{}@{}", + config.dataset.clone().trim(), + config.snapshot_name.trim() + ), ]; args.extend(config.snapshot_name.split_whitespace().map(String::from)); @@ -148,8 +156,7 @@ impl Service for Zfs { // Step 2, send the checkpoint to a local file, named: name-date.snapshot let dest = std::env::current_dir() .unwrap() - .join(PathBuf::from(format!("{}-{}.snapshot", self.name, date - ))); + .join(PathBuf::from(format!("{}-{}.snapshot", self.name, date))); let parent = dest.parent().unwrap(); if !parent.exists() { @@ -180,4 +187,4 @@ impl Service for Zfs { Err(error) => Err(Error::RuntimeError(error).into()), } } -} \ No newline at end of file +} From ea400d4c1fda664345ff097b3ed59e4f93fbd33f Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:26:18 +0200 Subject: [PATCH 04/51] ci: github actions update --- .github/workflows/checks.yml | 40 ++++++++++++------------------------ 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8417bdb..3563e89 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,17 +8,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v7 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.17 - name: Run cargo check - uses: actions-rs/cargo@v1 + uses: clechasseur/rs-cargo@v5.0.6 with: command: check @@ -27,18 +23,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v7 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.17 - name: Run cargo test - uses: actions-rs/cargo@v1 - continue-on-error: true + uses: lechasseur/rs-cargo@v5.0.6 with: command: test @@ -47,24 +38,19 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v7 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: rustfmt, clippy + - name: Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1.17 - name: Run cargo fmt - uses: actions-rs/cargo@v1 + uses: clechasseur/rs-cargo@v5.0.6 with: command: fmt args: --all -- --check - name: Run cargo clippy - uses: actions-rs/cargo@v1 + uses: clechasseur/rs-cargo@v5.0.6 with: command: clippy args: -- -D warnings \ No newline at end of file From 04d2f4ec197f8130ba27718e4d63fac5874d4b56 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:31:09 +0200 Subject: [PATCH 05/51] version bump --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6a6a766..4982289 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bacup" -version = "0.1.16" +version = "0.2.0" description = "An easy-to-use backup tool designed for servers." readme = "README.md" authors = ["Paolo Galeone "] From b13292be3ab1bae33798d37964899d51b704f33e Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:31:15 +0200 Subject: [PATCH 06/51] ci: fix package name --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3563e89..dea6a9c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -29,7 +29,7 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1.17 - name: Run cargo test - uses: lechasseur/rs-cargo@v5.0.6 + uses: clechasseur/rs-cargo@v5.0.6 with: command: test From 116b1e6676b8c85945883d47947becfd82a076b1 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 15:33:13 +0200 Subject: [PATCH 07/51] clippy --- src/backup.rs | 2 +- src/remotes/git.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 17344c2..572e0cc 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -308,7 +308,7 @@ impl Backup { let keep_last = inst.keep_last; // First call dump, to trigger the dump service if present - info!("[{}] Calling dump...", &name); + info!("[{}] Calling dump...", name); let dump = match service.dump().await { Err(error) => { error!("{}", Error::GeneralError(error)); diff --git a/src/remotes/git.rs b/src/remotes/git.rs index f7cfcdc..7d5dc67 100644 --- a/src/remotes/git.rs +++ b/src/remotes/git.rs @@ -130,7 +130,7 @@ impl Git { } let url = format!( "ssh://{}@{}:{}/{}", - &self.config.username, &self.config.host, &self.config.port, &self.config.repository + self.config.username, self.config.host, self.config.port, self.config.repository ); let status = Command::new(&self.git_cmd) @@ -140,7 +140,7 @@ impl Git { return Err(Error::RuntimeError(io::Error::other(format!( "Unable to execute {} clone {} --depth 1", self.git_cmd.display(), - &url + url )))); } From 353c34f361edc0e066299b06efd55699cead643a Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 16:02:51 +0200 Subject: [PATCH 08/51] zfs support --- src/bin/bacup.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index dc2386b..853159f 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -34,6 +34,7 @@ use bacup::services::folders::Folder; use bacup::services::postgresql::PostgreSql; use bacup::services::service::Service; +use bacup::services::zfs::Zfs; use log::*; use structopt::StructOpt; @@ -174,6 +175,19 @@ async fn main() -> Result<(), i32> { None => warn!("No Docker to backup."), } + match config.zfs { + Some(zfs) => { + for (service_name, instance_config) in zfs { + let key = format!("zfs.{}", service_name); + services.insert( + key, + Box::new(Zfs::new(instance_config, &service_name).await.unwrap()), + ); + } + } + None => warn!("No Docker to backup."), + } + let mut backup: HashMap> = HashMap::new(); for (backup_name, config) in config.backup { if !services.contains_key(&config.what) { From bdd6fc3724ea6f75fe3a80e710343d96e3e9d1ec Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 6 Aug 2026 17:00:08 +0200 Subject: [PATCH 09/51] zfs commands: fix args --- src/services/zfs.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 08232bb..8016e6c 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -101,7 +101,7 @@ impl Zfs { // If here, the current user is in the allow list for zfs send and snapshot - let mut args: Vec = vec![ + let args: Vec = vec![ String::from("snapshot"), String::from("-r"), format!( @@ -110,7 +110,6 @@ impl Zfs { config.snapshot_name.trim() ), ]; - args.extend(config.snapshot_name.split_whitespace().map(String::from)); Ok(Zfs { name: String::from(name), @@ -170,7 +169,6 @@ impl Service for Zfs { let dest_file = File::create(&dest).await?; let send_args = vec![ - self.args[0].to_string(), String::from("send"), String::from("-R"), String::from("-v"), From 5c04043cd4c56ad1b6ac7c33a50e665b62434ce9 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 13:57:32 +0200 Subject: [PATCH 10/51] update --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4982289..c27c279 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ url = { version = "2.5.8", features = ["serde"] } which = "8.0.5" aws-config = "1.10.1" aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } -aws-sdk-s3 = "1.140.0" +aws-sdk-s3 = "1.141.0" async-compression = { version = "0.4.43", default-features = false, features = ["tokio", "gzip"] } astral-tokio-tar = "0.6.4" tokio-cron-scheduler = "0.15.1" From d04f1aaa7b977475582dcc6ee3239b5dda270c96 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 13:57:36 +0200 Subject: [PATCH 11/51] logs --- src/services/zfs.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 8016e6c..1da3155 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -18,6 +18,8 @@ use std::path::PathBuf; use std::string::String; use std::vec::Vec; +use log::info; + use crate::config::ZfsConfig; use crate::services::service::{Dump, Service}; @@ -63,18 +65,13 @@ impl Zfs { // Verify zfs is working and the current user is in the allow list for executing // snapshot and send - let child = Command::new(&cmd) + let output = Command::new(&cmd) .args(vec![String::from("allow"), config.dataset.clone()]) // capture output to variable to check if the user is there .stdout(Stdio::piped()) - .spawn(); - - if let Err(error) = child { - return Err(Error::RuntimeError(error)); - } - - let child = child.unwrap(); - let output = child.wait_with_output().await; + .stderr(Stdio::piped()) + .output() + .await; if let Err(err) = output { return Err(Error::RuntimeError(err)); @@ -82,7 +79,7 @@ impl Zfs { let output = output.unwrap(); let status = output.status; if !status.success() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}`", config.dataset))); + return Err(Error::RuntimeError(io::Error::other(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}`", config.dataset)))); } let stdout = String::from_utf8(output.stdout).unwrap(); @@ -143,6 +140,7 @@ impl Service for Zfs { let checkpoint_name = checkpoint.clone(); // Step 1, execute the checkpoint (atomic, immediate action) + info!("Executing: {} {:?}", self.cmd.display(), args); let status = Command::new(&self.cmd) .args(args) .stdout(Stdio::null()) @@ -175,6 +173,12 @@ impl Service for Zfs { checkpoint_name.to_string(), ]; + info!( + "Executing: {} {:?} > {}", + self.cmd.display(), + send_args, + dest.display() + ); match Command::new(&self.cmd) .args(&send_args) .stdout(Stdio::from(dest_file.try_into_std().unwrap())) From 741798cb0e81afaf65b2cfd4702bcd1a83be874d Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 14:24:12 +0200 Subject: [PATCH 12/51] better handling of failed commands --- src/services/docker.rs | 20 +++++++++++++++----- src/services/zfs.rs | 33 +++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/services/docker.rs b/src/services/docker.rs index ce5082b..e03e4ac 100644 --- a/src/services/docker.rs +++ b/src/services/docker.rs @@ -123,14 +123,24 @@ impl Service for Docker { let dest_file = File::create(&dest).await?; - match Command::new(&self.cmd) + let status = Command::new(&self.cmd) .args(&self.args) .stdout(Stdio::from(dest_file.try_into_std().unwrap())) .status() - .await - { - Ok(_) => Ok(Dump { path: Some(dest) }), - Err(error) => Err(Error::RuntimeError(error).into()), + .await; + if let Err(error) = status { + return Err(Error::RuntimeError(error).into()); + } + let status = status?; + match status.success() { + true => Ok(Dump { path: Some(dest) }), + false => Err(Error::RuntimeError(io::Error::other(format!( + "{} {:?} failed with exit code {}", + self.cmd.display(), + self.args, + status.code().unwrap() + ))) + .into()), } } } diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 1da3155..49c6d6b 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -65,8 +65,10 @@ impl Zfs { // Verify zfs is working and the current user is in the allow list for executing // snapshot and send + let args = vec![String::from("allow"), config.dataset.clone()]; + info!("Executing {} {:?}", cmd.display(), args); let output = Command::new(&cmd) - .args(vec![String::from("allow"), config.dataset.clone()]) + .args(args) // capture output to variable to check if the user is there .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -79,16 +81,16 @@ impl Zfs { let output = output.unwrap(); let status = output.status; if !status.success() { - return Err(Error::RuntimeError(io::Error::other(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}`", config.dataset)))); + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER hold,send,snapshot {}`", config.dataset))); } let stdout = String::from_utf8(output.stdout).unwrap(); if stdout.is_empty() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER snapshot,send {}", config.dataset))); + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER hold,send,snapshot {}", config.dataset))); } - // Check if the string user $USER send,snapshot is in the stdout - let needle = format!("user {} send,snapshot", std::env::var("USER").unwrap()); + // Check if the string user $USER hold,send,snapshot is in the stdout + let needle = format!("user {} hold,send,snapshot", std::env::var("USER").unwrap()); if !stdout.contains(&needle) { return Err(Error::ZfsError(format!( "\"{}\" not found in output of `zfs allow {}`", @@ -179,14 +181,25 @@ impl Service for Zfs { send_args, dest.display() ); - match Command::new(&self.cmd) + + let status = Command::new(&self.cmd) .args(&send_args) .stdout(Stdio::from(dest_file.try_into_std().unwrap())) .status() - .await - { - Ok(_) => Ok(Dump { path: Some(dest) }), - Err(error) => Err(Error::RuntimeError(error).into()), + .await; + if let Err(error) = status { + return Err(Error::RuntimeError(error).into()); + } + let status = status?; + match status.success() { + true => Ok(Dump { path: Some(dest) }), + false => Err(Error::RuntimeError(io::Error::other(format!( + "{} {:?} failed with exit code {}", + self.cmd.display(), + self.args, + status.code().unwrap() + ))) + .into()), } } } From 791309224454054f8e251158e7510479ba8e2f22 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 14:50:12 +0200 Subject: [PATCH 13/51] correct dump (fixed filename) - automatically dropped via Drop --- src/services/zfs.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 49c6d6b..33e47cc 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -132,12 +132,11 @@ impl Service for Zfs { } async fn dump(&self) -> Result> { - // Current date in ISO format - let date = chrono::Utc::now().format("%Y%m%d-%H%M%S"); - let mut args = self.args.clone(); let checkpoint = args.last_mut().unwrap(); - checkpoint.push_str(&format!("-{}", date)); + + // Current date in ISO format + checkpoint.push_str(&format!("-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))); let checkpoint_name = checkpoint.clone(); @@ -152,10 +151,10 @@ impl Service for Zfs { return Err(Error::RuntimeError(status.err().unwrap()).into()); } - // Step 2, send the checkpoint to a local file, named: name-date.snapshot + // Step 2, send the checkpoint to a local file, named: name.snapshot let dest = std::env::current_dir() .unwrap() - .join(PathBuf::from(format!("{}-{}.snapshot", self.name, date))); + .join(PathBuf::from(format!("{}.snapshot", self.name))); let parent = dest.parent().unwrap(); if !parent.exists() { From e07f54b85ffbe8f3bba0b2c0349e4c8e0f6d09d3 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 16:57:55 +0200 Subject: [PATCH 14/51] use streaming instead of loading file to memory --- Cargo.toml | 2 +- src/remotes/remote.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c27c279..182e432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,9 +35,9 @@ aws-config = "1.10.1" aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-s3 = "1.141.0" async-compression = { version = "0.4.43", default-features = false, features = ["tokio", "gzip"] } -astral-tokio-tar = "0.6.4" tokio-cron-scheduler = "0.15.1" cron = "0.17.0" uuid = { version = "1.24.0", features = ["v4"] } aws-types = "1.5.0" croner = "3.0.1" +astral-tokio-tar = "0.6.4" diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index 0b7eff8..3443922 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -30,7 +30,7 @@ use crate::remotes::aws::Error as AWSError; use tempfile::NamedTempFile; use tokio::fs; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::io::AsyncWriteExt; use log::info; @@ -104,21 +104,21 @@ pub trait Remote: DynClone + Send + Sync { Self: Sized, { info!("Compressing file {}...", path.display()); - let mut content: Vec = vec![]; let mut file = match fs::File::open(path).await { Ok(file) => file, Err(error) => return Err(Error::LocalError(error)), }; - file.read_to_end(&mut content).await?; + let mut encoder = GzipEncoder::new(Vec::new()); - let mut e = GzipEncoder::new(Vec::new()); - e.write_all(&content).await?; - e.flush().await?; - e.shutdown().await?; + // Stream the file contents directly into the encoder without loading it all into RAM + tokio::io::copy(&mut file, &mut encoder).await?; + + encoder.flush().await?; + encoder.shutdown().await?; info!("Compression of file {} done.", path.display()); - Ok(e.into_inner()) + Ok(encoder.into_inner()) } fn remote_archive_path(&self, remote_path: &Path) -> PathBuf { From 8ca4911a6c3f9764b3424125256784008be1a867 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Fri, 7 Aug 2026 17:20:41 +0200 Subject: [PATCH 15/51] misc: systemdfile env var + working directory clarification --- README.md | 5 +++-- misc/systemd/bacup@.service | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 474a6dc..2189c59 100644 --- a/README.md +++ b/README.md @@ -170,10 +170,11 @@ sudo cp misc/systemd/bacup@.service /usr/lib/systemd/system/ then, the service can be enabled/started in the usual systemd way: ``` -sudo systemctl start bacup@$USER.service -sudo systemctl enable bacup@$USER.service +sudo systemctl enable --now bacup@$USER.service ``` +**Note**: the working directory is important if you plan to back-up big files. The files are created in that directory before being uploaded, so set it to a location where you have the write right and enough space. + ## Remote configuration Configuring the remotes is straightforward. Every remote have a different way of getting the access code, here we try to share some useful reference. diff --git a/misc/systemd/bacup@.service b/misc/systemd/bacup@.service index 90f3dc0..dd854a3 100644 --- a/misc/systemd/bacup@.service +++ b/misc/systemd/bacup@.service @@ -9,6 +9,7 @@ Restart=on-failure User=%I Type=simple WorkingDirectory=/home/%I/.bacup/ +Environment="CONF_FILE=/home/%I/.bacup/config.toml" ExecStart=/home/%I/.cargo/bin/bacup -vv [Install] From 54e34884c632a92ec623e49a35510994e034854e Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sat, 8 Aug 2026 10:38:04 +0200 Subject: [PATCH 16/51] remotes: use files, do not compress in memory --- src/remotes/remote.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index 3443922..f9a56b8 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -24,9 +24,6 @@ use chrono::Utc; use async_compression::tokio::write::GzipEncoder; use dyn_clone::DynClone; - -use crate::remotes::aws::Error as AWSError; - use tempfile::NamedTempFile; use tokio::fs; @@ -34,10 +31,12 @@ use tokio::io::AsyncWriteExt; use log::info; +use super::aws::AwsError; + #[derive(Debug)] pub enum Error { LocalError(std::io::Error), - RemoteError(AWSError), + RemoteError(AwsError), CompressionError, NotADirectory, } @@ -48,8 +47,8 @@ impl From for Error { } } -impl From for Error { - fn from(error: AWSError) -> Self { +impl From for Error { + fn from(error: AwsError) -> Self { Error::RemoteError(error) } } @@ -61,7 +60,7 @@ impl fmt::Display for Error { Error::LocalError(error) => write!(f, "Local (IO) error: {}", error), Error::CompressionError => write!(f, "Unable to compress the file/folder"), Error::NotADirectory => write!(f, "The specified file is not a directory"), - Error::RemoteError(error) => write!(f, "Remote error: {}", error), + Error::RemoteError(error) => write!(f, "Remote error: {:?}", error), } } } @@ -82,7 +81,7 @@ pub trait Remote: DynClone + Send + Sync { Self: Sized, { info!("Compressing folder {}", path.display()); - let archive_path = NamedTempFile::new()?; + let archive_path = NamedTempFile::new_in(std::env::current_dir().unwrap())?; let file = fs::File::create(&archive_path).await?; let encoder = GzipEncoder::new(file); @@ -99,26 +98,28 @@ pub trait Remote: DynClone + Send + Sync { Ok(archive_path) } - async fn compress_file(&self, path: &Path) -> Result, Error> + async fn compress_file(&self, path: &Path) -> Result where Self: Sized, { info!("Compressing file {}...", path.display()); - let mut file = match fs::File::open(path).await { + let mut input_file = match fs::File::open(path).await { Ok(file) => file, Err(error) => return Err(Error::LocalError(error)), }; - let mut encoder = GzipEncoder::new(Vec::new()); + let archive_path = NamedTempFile::new_in(std::env::current_dir().unwrap())?; + let file = fs::File::create(&archive_path).await?; + let mut encoder = GzipEncoder::new(file); // Stream the file contents directly into the encoder without loading it all into RAM - tokio::io::copy(&mut file, &mut encoder).await?; + tokio::io::copy(&mut input_file, &mut encoder).await?; encoder.flush().await?; encoder.shutdown().await?; info!("Compression of file {} done.", path.display()); - Ok(encoder.into_inner()) + Ok(archive_path) } fn remote_archive_path(&self, remote_path: &Path) -> PathBuf { From c33755391a15f7dc719361568bf56dc48c06df29 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sat, 8 Aug 2026 10:38:23 +0200 Subject: [PATCH 17/51] aws multipart upload + copyright update --- Cargo.toml | 2 + src/bin/bacup.rs | 2 +- src/config.rs | 2 +- src/lib.rs | 2 +- src/remotes/aws.rs | 199 ++++++++++++++++++++++++++++++++----- src/remotes/git.rs | 12 +-- src/remotes/localhost.rs | 8 +- src/remotes/mod.rs | 2 +- src/remotes/ssh.rs | 49 +++++---- src/services/docker.rs | 2 +- src/services/folders.rs | 2 +- src/services/mod.rs | 2 +- src/services/postgresql.rs | 2 +- src/services/service.rs | 2 +- src/services/zfs.rs | 3 +- 15 files changed, 225 insertions(+), 66 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 182e432..4458d82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,3 +41,5 @@ uuid = { version = "1.24.0", features = ["v4"] } aws-types = "1.5.0" croner = "3.0.1" astral-tokio-tar = "0.6.4" +get-file-size = "0.1.2" +aws-smithy-runtime-api = "1.14.0" diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index 853159f..b7bd107 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/config.rs b/src/config.rs index ae6afef..bd9ba9b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/lib.rs b/src/lib.rs index ece3331..5cdaf4f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index f8679d2..13206b1 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,8 +13,9 @@ // limitations under the License. use aws_credential_types::provider::SharedCredentialsProvider; -use aws_sdk_s3::primitives::ByteStream; -pub use aws_sdk_s3::{Client, Error}; +use aws_sdk_s3::primitives::{ByteStream, Length}; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::Client; use aws_types::region::Region; use crate::config::AwsConfig; @@ -27,6 +28,8 @@ use tokio::io::AsyncReadExt; use async_trait::async_trait; +use std::io; + #[derive(Clone)] pub struct AwsBucket { name: String, @@ -39,15 +42,38 @@ struct Bucket { bucket_name: String, } +#[derive(Debug)] +pub enum AwsError { + RemoteError(aws_sdk_s3::Error), + LocalError(io::Error), + GenericError(String), +} + +impl From for AwsError { + fn from(err: io::Error) -> Self { + AwsError::LocalError(err) + } +} + +impl From for AwsError { + fn from(err: aws_sdk_s3::Error) -> Self { + AwsError::RemoteError(err) + } +} + impl Bucket { - pub async fn list(&self, prefix: &str) -> Result, Error> { + pub async fn list(&self, prefix: &str) -> Result, AwsError> { let response = self .client .list_objects_v2() .bucket(&self.bucket_name) .prefix(prefix.trim_start_matches('/')) .send() - .await?; + .await; + if response.is_err() { + return Err(AwsError::RemoteError(response.err().unwrap().into())); + } + let response = response.unwrap(); let mut ret: Vec = vec![]; for res in response.contents.iter() { for object in res { @@ -57,31 +83,157 @@ impl Bucket { Ok(ret) } - pub async fn put_object(&self, remote_path: &str, content: Vec) -> Result<(), Error> { - self.client - .put_object() - .bucket(&self.bucket_name) - .key(remote_path.trim_start_matches('/')) - .body(ByteStream::from(content)) - .send() - .await?; + pub async fn put_object(&self, remote_path: &str, path: &Path) -> Result<(), AwsError> { + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + use get_file_size::GetFileSize; + const CHUNK_SIZE: u64 = 1024 * 1024 * 1024; // 1024 MiB + const MAX_CHUNKS: u64 = 10000; // 1024 * 10000 ~= 9.7 TiB max + + let file_size = path.file_size().await.unwrap_or_default(); + + let remote_path = remote_path.trim_start_matches('/'); + + if file_size <= CHUNK_SIZE { + // Just read the file and upload to bytes. + // Suppose CHUNK_SIZE free memory available. + let mut content: Vec = vec![]; + let mut file = File::open(path).await?; + file.read_to_end(&mut content).await?; + let response = self + .client + .put_object() + .bucket(&self.bucket_name) + .key(remote_path) + .body(ByteStream::from(content)) + .send() + .await; + + if response.is_err() { + return Err(AwsError::RemoteError(response.err().unwrap().into())); + } + } else { + // Multipart upload + + let multipart_upload_res = self + .client + .create_multipart_upload() + .bucket(&self.bucket_name) + .key(remote_path) + .send() + .await; + if multipart_upload_res.is_err() { + return Err(AwsError::RemoteError( + multipart_upload_res.err().unwrap().into(), + )); + } + let multipart_upload_res = multipart_upload_res.unwrap(); + + let upload_id = multipart_upload_res + .upload_id() + .ok_or(AwsError::GenericError( + "Missing upload_id after CreateMultipartUpload".to_string(), + ))?; + + let mut chunk_count = (file_size / CHUNK_SIZE) + 1; + let mut size_of_last_chunk = file_size % CHUNK_SIZE; + if size_of_last_chunk == 0 { + size_of_last_chunk = CHUNK_SIZE; + chunk_count -= 1; + } + + if chunk_count > MAX_CHUNKS { + return Err(AwsError::GenericError(format!( + "Too many chunks: {} > {}", + chunk_count, MAX_CHUNKS + ))); + } + + let mut upload_parts: Vec = Vec::new(); + + for chunk_index in 0..chunk_count { + let this_chunk = if chunk_count - 1 == chunk_index { + size_of_last_chunk + } else { + CHUNK_SIZE + }; + let stream = ByteStream::read_from() + .path(path) + .offset(chunk_index * CHUNK_SIZE) + .length(Length::Exact(this_chunk)) + .build() + .await + .unwrap(); + + // Chunk index needs to start at 0, but part numbers start at 1. + let part_number = (chunk_index as i32) + 1; + let upload_part_res = self + .client + .upload_part() + .key(remote_path) + .bucket(&self.bucket_name) + .upload_id(upload_id) + .body(stream) + .part_number(part_number) + .send() + .await; + + if upload_part_res.is_err() { + return Err(AwsError::RemoteError( + upload_part_res.err().unwrap().into_service_error().into(), + )); + } + let upload_part_res = upload_part_res.unwrap(); + + upload_parts.push( + CompletedPart::builder() + .e_tag(upload_part_res.e_tag.unwrap_or_default()) + .part_number(part_number) + .build(), + ); + } + + let completed_multipart_upload = + CompletedMultipartUpload::builder() + .set_parts(Some(upload_parts)) + .build(); + + let complete_multipart_upload_res = self + .client + .complete_multipart_upload() + .bucket(&self.bucket_name) + .key(remote_path) + .multipart_upload(completed_multipart_upload) + .upload_id(upload_id) + .send() + .await; + if complete_multipart_upload_res.is_err() { + return Err(AwsError::RemoteError( + complete_multipart_upload_res.err().unwrap().into(), + )); + } + } Ok(()) } - pub async fn delete(&self, remote_path: &str) -> Result<(), Error> { - self.client + pub async fn delete(&self, remote_path: &str) -> Result<(), AwsError> { + let response = self + .client .delete_object() .bucket(&self.bucket_name) .key(remote_path) .send() - .await?; + .await; + + if response.is_err() { + return Err(AwsError::RemoteError(response.err().unwrap().into())); + } Ok(()) } } impl AwsBucket { - pub async fn new(config: AwsConfig, bucket_name: &str) -> Result { + pub async fn new(config: AwsConfig, bucket_name: &str) -> Result { let region = Region::new(config.region); let mut builder = aws_config::defaults(aws_config::BehaviorVersion::latest()).region(region); @@ -133,12 +285,9 @@ impl remote::Remote for AwsBucket { } async fn upload_file(&self, path: &Path, remote_path: &Path) -> Result<(), remote::Error> { - let mut content: Vec = vec![]; - let mut file = File::open(path).await?; - file.read_to_end(&mut content).await?; - - let remote_path = remote_path.to_str().unwrap(); - self.bucket.put_object(remote_path, content).await?; + self.bucket + .put_object(remote_path.to_str().unwrap(), path) + .await?; Ok(()) } @@ -147,10 +296,10 @@ impl remote::Remote for AwsBucket { path: &Path, remote_path: &Path, ) -> Result<(), remote::Error> { - let compressed_bytes = self.compress_file(path).await?; + let compressed_file = self.compress_file(path).await?; let remote_path = self.remote_compressed_file_path(remote_path); self.bucket - .put_object(remote_path.to_str().unwrap(), compressed_bytes) + .put_object(remote_path.to_str().unwrap(), compressed_file.path()) .await?; Ok(()) } diff --git a/src/remotes/git.rs b/src/remotes/git.rs index 7d5dc67..43e1cf3 100644 --- a/src/remotes/git.rs +++ b/src/remotes/git.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,8 +17,6 @@ use crate::remotes::remote; use crate::remotes::ssh; use tokio::fs; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; use std::io; @@ -238,19 +236,17 @@ impl remote::Remote for Git { remote_path: &Path, ) -> Result<(), remote::Error> { // Read and compress - let compressed_bytes = self.compress_file(path).await?; + let compressed_file = self.compress_file(path).await?; let remote_path = self.remote_compressed_file_path(remote_path); - let mut buffer = File::create(&remote_path).await?; - buffer.write_all(&compressed_bytes).await?; - defer! { #[allow(unused_must_use)] { fs::remove_file(&remote_path); } } - self.upload_file(&remote_path, &remote_path).await?; + self.upload_file(compressed_file.path(), &remote_path) + .await?; Ok(()) } diff --git a/src/remotes/localhost.rs b/src/remotes/localhost.rs index 975a64f..0da21fd 100644 --- a/src/remotes/localhost.rs +++ b/src/remotes/localhost.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -147,9 +147,8 @@ impl remote::Remote for Localhost { remote_path: &Path, ) -> Result<(), remote::Error> { use tokio::fs; - use tokio::io::AsyncWriteExt; - let compressed_bytes = self.compress_file(path).await?; + let compressed_file = self.compress_file(path).await?; let remote_path = if remote_path.is_absolute() { remote_path.strip_prefix("/").unwrap() } else { @@ -163,8 +162,7 @@ impl remote::Remote for Localhost { self.remote_compressed_file_path(&PathBuf::from(remote_path.file_name().unwrap())), ); - let mut buffer = fs::File::create(remote_path).await?; - buffer.write_all(&compressed_bytes).await?; + fs::copy(&compressed_file, remote_path).await?; Ok(()) } diff --git a/src/remotes/mod.rs b/src/remotes/mod.rs index b65c23c..3a33ea1 100644 --- a/src/remotes/mod.rs +++ b/src/remotes/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index c55aa69..76440f1 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -288,27 +288,42 @@ impl remote::Remote for Ssh { remote_path: &Path, ) -> Result<(), remote::Error> { // Read and compress - let compressed_bytes = self.compress_file(path).await?; + let compressed_file = self.compress_file(path).await?; let remote_path = self.remote_compressed_file_path(remote_path); // cat file | ssh -Pxxx user@host "cat > file" - let mut ssh = Command::new(&self.ssh_cmd) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .args( - self.ssh_args - .iter() - .chain(once(&format!("cat > {} ", remote_path.display()))), - ) + + let mut cat = Command::new("cat") + .arg(format!("{}", compressed_file.path().display())) + .stdout(Stdio::piped()) .spawn()?; - ssh.stdin.as_mut().unwrap().write_all(&compressed_bytes)?; - let status = ssh.wait()?; - if !status.success() { - return Err(remote::Error::LocalError(io::Error::other( - "Failure while executing ssh command", - ))); + + if let Some(cat_output) = cat.stdout.take() { + let mut ssh = Command::new(&self.ssh_cmd) + .stdin(cat_output) + .stdout(Stdio::null()) + .args( + self.ssh_args + .iter() + .chain(once(&format!("cat > {} ", remote_path.display()))), + ) + .spawn()?; + + cat.wait()?; + + let status = ssh.wait()?; + if !status.success() { + return Err(remote::Error::LocalError(io::Error::other( + "Failure while executing ssh command", + ))); + } + Ok(()) + } else { + Err(remote::Error::LocalError(io::Error::other(format!( + "Unable to cat {}", + compressed_file.path().display() + )))) } - Ok(()) } async fn upload_folder( diff --git a/src/services/docker.rs b/src/services/docker.rs index e03e4ac..87b0d82 100644 --- a/src/services/docker.rs +++ b/src/services/docker.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/services/folders.rs b/src/services/folders.rs index 83f0be6..fdf0262 100644 --- a/src/services/folders.rs +++ b/src/services/folders.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/services/mod.rs b/src/services/mod.rs index bcc2c8c..906b7df 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/services/postgresql.rs b/src/services/postgresql.rs index 5ce714c..59bca11 100644 --- a/src/services/postgresql.rs +++ b/src/services/postgresql.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/services/service.rs b/src/services/service.rs index 45b1ae2..0a67978 100644 --- a/src/services/service.rs +++ b/src/services/service.rs @@ -1,4 +1,4 @@ -// Copyright 2022 Paolo Galeone +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 33e47cc..0672c87 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -1,5 +1,4 @@ -// src/services/zfs.rs -// Copyright 2024 Minimal ZFS Service +// Copyright 2022-2026 Paolo Galeone // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From af622452f6517de0751f7b99fef7e0792872934d Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sat, 8 Aug 2026 11:28:35 +0200 Subject: [PATCH 18/51] fmt --- src/remotes/aws.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 13206b1..66edcbd 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -192,10 +192,9 @@ impl Bucket { ); } - let completed_multipart_upload = - CompletedMultipartUpload::builder() - .set_parts(Some(upload_parts)) - .build(); + let completed_multipart_upload = CompletedMultipartUpload::builder() + .set_parts(Some(upload_parts)) + .build(); let complete_multipart_upload_res = self .client From a6a16bf0c968097306a7f10574bdcd5e34ae2e87 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sat, 8 Aug 2026 14:56:14 +0200 Subject: [PATCH 19/51] upload logging --- src/remotes/aws.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 66edcbd..c2a93c2 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -28,6 +28,7 @@ use tokio::io::AsyncReadExt; use async_trait::async_trait; +use log::info; use std::io; #[derive(Clone)] @@ -92,6 +93,7 @@ impl Bucket { let file_size = path.file_size().await.unwrap_or_default(); let remote_path = remote_path.trim_start_matches('/'); + info!("Uploading file {} to {}", path.display(), remote_path); if file_size <= CHUNK_SIZE { // Just read the file and upload to bytes. @@ -151,6 +153,8 @@ impl Bucket { let mut upload_parts: Vec = Vec::new(); for chunk_index in 0..chunk_count { + info!("Uploading chunk {} of {}", chunk_index, chunk_count); + let this_chunk = if chunk_count - 1 == chunk_index { size_of_last_chunk } else { From edc0daa71293bc24ea921b3c491ae1e220eb5df5 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sat, 8 Aug 2026 16:03:44 +0200 Subject: [PATCH 20/51] better logging --- src/remotes/aws.rs | 58 ++++++++++++++++++++++++++++++++++++++-- src/remotes/git.rs | 29 +++++++++++++++++--- src/remotes/localhost.rs | 21 ++++++++++++++- src/remotes/remote.rs | 26 +++++++++++++++--- src/remotes/ssh.rs | 43 ++++++++++++++++++++++++++++- 5 files changed, 165 insertions(+), 12 deletions(-) diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index c2a93c2..0699aab 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -93,7 +93,12 @@ impl Bucket { let file_size = path.file_size().await.unwrap_or_default(); let remote_path = remote_path.trim_start_matches('/'); - info!("Uploading file {} to {}", path.display(), remote_path); + info!( + "Uploading file {} ({:.2} MB) to {}", + path.display(), + file_size as f64 / 1_048_576.0, + remote_path + ); if file_size <= CHUNK_SIZE { // Just read the file and upload to bytes. @@ -129,6 +134,11 @@ impl Bucket { )); } let multipart_upload_res = multipart_upload_res.unwrap(); + info!( + "Multipart upload initiated for {} ({:.2} MB)", + remote_path, + file_size as f64 / 1_048_576.0 + ); let upload_id = multipart_upload_res .upload_id() @@ -142,6 +152,11 @@ impl Bucket { size_of_last_chunk = CHUNK_SIZE; chunk_count -= 1; } + info!( + "Will upload {} chunks ({:.2} MB each)", + chunk_count, + CHUNK_SIZE as f64 / 1_048_576.0 + ); if chunk_count > MAX_CHUNKS { return Err(AwsError::GenericError(format!( @@ -153,7 +168,18 @@ impl Bucket { let mut upload_parts: Vec = Vec::new(); for chunk_index in 0..chunk_count { - info!("Uploading chunk {} of {}", chunk_index, chunk_count); + let this_chunk = if chunk_count - 1 == chunk_index { + size_of_last_chunk + } else { + CHUNK_SIZE + }; + info!( + "Uploading chunk {} of {} ({:.2} MB) to {}", + chunk_index + 1, + chunk_count, + this_chunk as f64 / 1_048_576.0, + remote_path + ); let this_chunk = if chunk_count - 1 == chunk_index { size_of_last_chunk @@ -195,6 +221,7 @@ impl Bucket { .build(), ); } + info!("Uploaded all {} chunks to {}", chunk_count, remote_path); let completed_multipart_upload = CompletedMultipartUpload::builder() .set_parts(Some(upload_parts)) @@ -214,11 +241,19 @@ impl Bucket { complete_multipart_upload_res.err().unwrap().into(), )); } + info!( + "Multipart upload completed successfully for {}", + remote_path + ); } Ok(()) } pub async fn delete(&self, remote_path: &str) -> Result<(), AwsError> { + info!( + "Deleting object {} from bucket {}", + remote_path, self.bucket_name + ); let response = self .client .delete_object() @@ -230,6 +265,10 @@ impl Bucket { if response.is_err() { return Err(AwsError::RemoteError(response.err().unwrap().into())); } + info!( + "Successfully deleted {} from bucket {}", + remote_path, self.bucket_name + ); Ok(()) } @@ -341,6 +380,11 @@ impl remote::Remote for AwsBucket { } futures::future::join_all(futures).await; + info!( + "Successfully uploaded {} file(s) to {}", + tot, + remote_path.display() + ); Ok(()) } @@ -355,8 +399,18 @@ impl remote::Remote for AwsBucket { let remote_path = self.remote_archive_path(remote_path); let compressed_folder = self.compress_folder(path).await?; + info!( + "Uploading compressed folder archive {} to {}", + compressed_folder.path().display(), + remote_path.display() + ); self.upload_file(compressed_folder.path(), &remote_path) .await?; + info!( + "Successfully uploaded compressed folder {} to {}", + path.display(), + remote_path.display() + ); Ok(()) } } diff --git a/src/remotes/git.rs b/src/remotes/git.rs index 43e1cf3..447bd16 100644 --- a/src/remotes/git.rs +++ b/src/remotes/git.rs @@ -18,11 +18,10 @@ use crate::remotes::ssh; use tokio::fs; +use std::fmt; use std::io; - use std::path::{Path, PathBuf}; - -use std::fmt; +use std::process::Command; use std::string::String; use which::which; @@ -31,7 +30,7 @@ use async_trait::async_trait; use scopeguard::defer; -use std::process::Command; +use log::info; #[derive(Debug)] pub enum Error { @@ -173,6 +172,7 @@ impl remote::Remote for Git { // cp file /[] let dest = repo.join(remote_path.strip_prefix("/").unwrap()); + info!("Copying {} to {}", path.display(), dest.display()); if !dest.exists() { fs::create_dir_all(&dest).await.unwrap(); } @@ -227,6 +227,11 @@ impl remote::Remote for Git { dest.display() )))); } + info!( + "Successfully pushed {} to {}", + path.display(), + dest.display() + ); Ok(()) } @@ -259,10 +264,13 @@ impl remote::Remote for Git { // cp file /[] let dest = repo.join(remote_path.strip_prefix("/").unwrap()); + info!("Copying {} file(s) to {}", paths.len(), dest.display()); if !dest.exists() { fs::create_dir_all(&dest).await.unwrap(); } let git_folder = std::path::Component::Normal(".git".as_ref()); + let mut files_copied = 0; + let mut dirs_created = 0; for path in paths.iter() { // Skip .git and content of this folder if path.components().any(|x| x == git_folder) { @@ -270,10 +278,18 @@ impl remote::Remote for Git { } if path.is_dir() { fs::create_dir_all(dest.join(path.file_name().unwrap())).await?; + dirs_created += 1; } else { fs::copy(path, dest.join(path.file_name().unwrap())).await?; + files_copied += 1; } } + info!( + "Copied {} files and {} directories to {}", + files_copied, + dirs_created, + dest.display() + ); // cd let cwd = std::env::current_dir()?; @@ -324,6 +340,11 @@ impl remote::Remote for Git { dest.display() )))); } + info!( + "Successfully pushed {} file(s) to {}", + files_copied, + dest.display() + ); Ok(()) } diff --git a/src/remotes/localhost.rs b/src/remotes/localhost.rs index 0da21fd..5a32b39 100644 --- a/src/remotes/localhost.rs +++ b/src/remotes/localhost.rs @@ -15,12 +15,13 @@ use crate::config::LocalhostConfig; use crate::remotes::remote; +use std::fmt; use std::io; use std::path::{Path, PathBuf}; use async_trait::async_trait; -use std::fmt; +use log::info; #[derive(Debug, PartialEq)] pub enum Error { @@ -134,10 +135,16 @@ impl remote::Remote for Localhost { }; let dest = self.path.join(remote_path.parent().unwrap()); + info!("Copying file {} to {}", path.display(), dest.display()); if !dest.exists() { fs::create_dir_all(&dest).await?; } fs::copy(path, dest.join(remote_path.file_name().unwrap())).await?; + info!( + "Successfully uploaded {} to {}", + path.display(), + dest.display() + ); Ok(()) } @@ -195,6 +202,12 @@ impl remote::Remote for Localhost { PathBuf::from(remote_path) }; + info!( + "Uploading {} file(s) to {}", + paths.len(), + remote_path.display() + ); + for path in paths.iter() { if path.is_file() { let dest = self @@ -204,10 +217,16 @@ impl remote::Remote for Localhost { if !parent.exists() { fs::create_dir_all(parent).await?; } + info!("Uploading {} to {}", path.display(), dest.display()); fs::copy(path, dest).await?; } } + info!( + "Successfully uploaded {} file(s) to {}", + paths.len(), + remote_path.display() + ); Ok(()) } diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index f9a56b8..c5a8bad 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -80,7 +80,12 @@ pub trait Remote: DynClone + Send + Sync { where Self: Sized, { - info!("Compressing folder {}", path.display()); + let folder_size = fs::metadata(path).await?.len(); + info!( + "Compressing folder {} ({:.2} MB) to archive...", + path.display(), + folder_size as f64 / 1_048_576.0 + ); let archive_path = NamedTempFile::new_in(std::env::current_dir().unwrap())?; let file = fs::File::create(&archive_path).await?; @@ -89,7 +94,9 @@ pub trait Remote: DynClone + Send + Sync { let mut builder = tokio_tar::Builder::new(encoder); builder .append_dir_all(path.file_name().unwrap(), path) - .await?; + .await + .unwrap(); + info!("Added items to archive"); let mut encoder = builder.into_inner().await?; encoder.flush().await?; @@ -102,7 +109,12 @@ pub trait Remote: DynClone + Send + Sync { where Self: Sized, { - info!("Compressing file {}...", path.display()); + let file_size = fs::metadata(path).await?.len(); + info!( + "Compressing file {} ({:.2} MB) to archive...", + path.display(), + file_size as f64 / 1_048_576.0 + ); let mut input_file = match fs::File::open(path).await { Ok(file) => file, Err(error) => return Err(Error::LocalError(error)), @@ -113,7 +125,13 @@ pub trait Remote: DynClone + Send + Sync { let mut encoder = GzipEncoder::new(file); // Stream the file contents directly into the encoder without loading it all into RAM - tokio::io::copy(&mut input_file, &mut encoder).await?; + let bytes_copied = tokio::io::copy(&mut input_file, &mut encoder).await?; + info!( + "Compressed {} bytes to {} bytes ({:.2} MB)", + file_size, + bytes_copied, + bytes_copied as f64 / 1_048_576.0 + ); encoder.flush().await?; encoder.shutdown().await?; diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index 76440f1..732733c 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -25,7 +25,7 @@ use std::path::{Path, PathBuf}; use std::fmt; use std::string::String; -use log::warn; +use log::{info, warn}; use tokio::fs; use tokio::fs::File; @@ -239,8 +239,15 @@ impl remote::Remote for Ssh { // Read file let mut content: Vec = vec![]; let mut file = File::open(path).await?; + let file_size = content.len(); file.read_to_end(&mut content).await?; let remote_path = remote_path.to_str().unwrap(); + info!( + "Uploading {} bytes from {} to {}", + file_size, + path.display(), + remote_path + ); // cat file | ssh -Pxxx user@host "cat > file" let mut ssh = Command::new(&self.ssh_cmd) @@ -279,6 +286,12 @@ impl remote::Remote for Ssh { ); return Err(remote::Error::LocalError(io::Error::other(message))); } + info!( + "Successfully uploaded {} bytes from {} to {}", + file_size, + path.display(), + remote_path + ); Ok(()) } @@ -290,6 +303,11 @@ impl remote::Remote for Ssh { // Read and compress let compressed_file = self.compress_file(path).await?; let remote_path = self.remote_compressed_file_path(remote_path); + info!( + "Uploading compressed file {} to {}", + compressed_file.path().display(), + remote_path.display() + ); // cat file | ssh -Pxxx user@host "cat > file" @@ -317,6 +335,11 @@ impl remote::Remote for Ssh { "Failure while executing ssh command", ))); } + info!( + "Successfully uploaded compressed file {} to {}", + compressed_file.path().display(), + remote_path.display() + ); Ok(()) } else { Err(remote::Error::LocalError(io::Error::other(format!( @@ -354,6 +377,13 @@ impl remote::Remote for Ssh { // delete is used to remove from remote and keep it in sync with local let args = vec!["-az", "-e", &ssh_port_opt, src, &dest, "--delete"]; + info!( + "Synchronizing {} file(s) from {} to {}", + paths.len(), + src, + dest + ); + let status = Command::new(&self.rsync_cmd) .stderr(Stdio::null()) .stdout(Stdio::null()) @@ -366,6 +396,12 @@ impl remote::Remote for Ssh { ))); } + info!( + "Successfully synchronized {} file(s) from {} to {}", + paths.len(), + src, + dest + ); Ok(()) } @@ -380,6 +416,11 @@ impl remote::Remote for Ssh { let remote_path = self.remote_archive_path(remote_path); let compressed_folder = self.compress_folder(path).await?; + info!( + "Uploading compressed folder archive {} to {}", + compressed_folder.path().display(), + remote_path.display() + ); self.upload_file(compressed_folder.path(), &remote_path) .await From 3783318a1377fbdd07d3bfe3ca598df04a5c7dec Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 9 Aug 2026 16:29:38 +0200 Subject: [PATCH 21/51] has_enough_space when creating archives --- Cargo.toml | 2 ++ src/disks.rs | 63 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/remotes/remote.rs | 34 +++++++++++++++++++++++ src/services/zfs.rs | 3 ++- 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 src/disks.rs diff --git a/Cargo.toml b/Cargo.toml index 4458d82..64a3c03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,3 +43,5 @@ croner = "3.0.1" astral-tokio-tar = "0.6.4" get-file-size = "0.1.2" aws-smithy-runtime-api = "1.14.0" +sysinfo = "0.38.4" +itertools = "0.15.0" diff --git a/src/disks.rs b/src/disks.rs new file mode 100644 index 0000000..9afb1c4 --- /dev/null +++ b/src/disks.rs @@ -0,0 +1,63 @@ +// Copyright 2022-2026 Paolo Galeone +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::path::Path; + +use sysinfo::Disks; +use tokio::fs; + +#[derive(Debug)] +pub enum Error { + LocalError(std::io::Error), +} + +pub async fn has_enough_space(destination_file: &Path, required_space: u64) -> Result { + let destination_path = + destination_file + .parent() + .ok_or(Error::LocalError(std::io::Error::other( + "parent no".to_string(), + )))?; + if !destination_path.exists() { + Err(Error::LocalError(std::io::Error::other( + "parent no exist".to_string(), + ))) + } else { + // Resolve symlinks + let destination_path = fs::canonicalize(destination_path).await.unwrap(); + let disks = Disks::new_with_refreshed_list(); + let potential_mountpoints = disks + .list() + .iter() + .filter(|disk| destination_path.starts_with(disk.mount_point())); + // The longer path with this prefix, is the correect disk + let mut path_space = HashMap::new(); + for mp in potential_mountpoints { + path_space.insert(mp.mount_point(), mp.available_space()); + } + // Get the longest mount point first + let mount_point = path_space + .iter() + .max_by_key(|(k, _)| k.to_str().unwrap_or("").len()) + .map(|(k, _)| k.to_owned()) + .ok_or(Error::LocalError(std::io::Error::other( + "no mounts".to_string(), + )))?; + let available_space = path_space + .get(&mount_point) + .ok_or(Error::LocalError(std::io::Error::other("??".to_string())))?; + Ok(required_space < *available_space) + } +} diff --git a/src/lib.rs b/src/lib.rs index 5cdaf4f..b83227b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,5 +14,6 @@ pub mod backup; pub mod config; +pub mod disks; pub mod remotes; pub mod services; diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index c5a8bad..181a2fc 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -31,11 +31,13 @@ use tokio::io::AsyncWriteExt; use log::info; +use super::super::disks; use super::aws::AwsError; #[derive(Debug)] pub enum Error { LocalError(std::io::Error), + StorageError(disks::Error), RemoteError(AwsError), CompressionError, NotADirectory, @@ -47,6 +49,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: disks::Error) -> Self { + Error::StorageError(error) + } +} + impl From for Error { fn from(error: AwsError) -> Self { Error::RemoteError(error) @@ -61,6 +69,7 @@ impl fmt::Display for Error { Error::CompressionError => write!(f, "Unable to compress the file/folder"), Error::NotADirectory => write!(f, "The specified file is not a directory"), Error::RemoteError(error) => write!(f, "Remote error: {:?}", error), + Error::StorageError(error) => write!(f, "Storagee error: {:?}", error), } } } @@ -88,6 +97,18 @@ pub trait Remote: DynClone + Send + Sync { ); let archive_path = NamedTempFile::new_in(std::env::current_dir().unwrap())?; + match disks::has_enough_space(archive_path.path(), folder_size).await { + Ok(enough_space) => { + if !enough_space { + return Err(Error::LocalError(std::io::Error::other(format!( + "Not enough space to create {}", + archive_path.path().display() + )))); + } + } + Err(error) => return Err(error.into()), + } + let file = fs::File::create(&archive_path).await?; let encoder = GzipEncoder::new(file); @@ -121,6 +142,19 @@ pub trait Remote: DynClone + Send + Sync { }; let archive_path = NamedTempFile::new_in(std::env::current_dir().unwrap())?; + + match disks::has_enough_space(archive_path.path(), file_size).await { + Ok(enough_space) => { + if !enough_space { + return Err(Error::LocalError(std::io::Error::other(format!( + "Not enough space to create {}", + archive_path.path().display() + )))); + } + } + Err(error) => return Err(error.into()), + } + let file = fs::File::create(&archive_path).await?; let mut encoder = GzipEncoder::new(file); diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 0672c87..8a131e4 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -165,7 +165,8 @@ impl Service for Zfs { } let dest_file = File::create(&dest).await?; - + // TODO: send dry-run, get the "total estimated size is XXX$unit" + // convert to bytes, and use has_enough_space to check. let send_args = vec![ String::from("send"), String::from("-R"), From 87a26e2b677acea3206228a6b2d895f361018572 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 9 Aug 2026 16:46:13 +0200 Subject: [PATCH 22/51] folder size --- src/disks.rs | 25 +++++++++++++++++++++++++ src/remotes/remote.rs | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/disks.rs b/src/disks.rs index 9afb1c4..0005384 100644 --- a/src/disks.rs +++ b/src/disks.rs @@ -18,11 +18,36 @@ use std::path::Path; use sysinfo::Disks; use tokio::fs; +use crate::disks::Error::LocalError; + #[derive(Debug)] pub enum Error { LocalError(std::io::Error), } +impl From for Error { + fn from(error: std::io::Error) -> Self { + LocalError(error) + } +} + +pub async fn calculate_folder_size(path: &Path) -> Result { + let mut total_size = 0u64; + + let mut entries = fs::read_dir(path).await?; + while let Some(entry_result) = entries.next_entry().await? { + let metadata = fs::metadata(entry_result.path()).await?; + + if metadata.is_file() { + total_size += metadata.len(); + } else if metadata.is_dir() { + total_size += Box::pin(calculate_folder_size(&entry_result.path())).await?; + } + } + + Ok(total_size) +} + pub async fn has_enough_space(destination_file: &Path, required_space: u64) -> Result { let destination_path = destination_file diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index 181a2fc..3dd37b4 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -89,7 +89,7 @@ pub trait Remote: DynClone + Send + Sync { where Self: Sized, { - let folder_size = fs::metadata(path).await?.len(); + let folder_size = disks::calculate_folder_size(path).await?; info!( "Compressing folder {} ({:.2} MB) to archive...", path.display(), From 9f142a84fe811d08ad52f7e157a17d8dcf13b147 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 9 Aug 2026 16:52:33 +0200 Subject: [PATCH 23/51] tests --- src/disks.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/disks.rs b/src/disks.rs index 0005384..29a157a 100644 --- a/src/disks.rs +++ b/src/disks.rs @@ -86,3 +86,88 @@ pub async fn has_enough_space(destination_file: &Path, required_space: u64) -> R Ok(required_space < *available_space) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::path::PathBuf; + use tempfile::tempdir; + + #[tokio::test] + async fn test_calculate_folder_size_empty_dir() { + let dir = tempdir().unwrap(); + let size = calculate_folder_size(dir.path()).await.unwrap(); + assert_eq!(size, 0); + } + + #[tokio::test] + async fn test_calculate_folder_size_single_file() { + let dir = tempdir().unwrap(); + let test_file = dir.path().join("test.txt"); + fs::write(&test_file, "Hello, World!").unwrap(); + let size = calculate_folder_size(dir.path()).await.unwrap(); + assert_eq!(size, 13); // "Hello, World!".len() + } + + #[tokio::test] + async fn test_calculate_folder_size_nested() { + let dir = tempdir().unwrap(); + let subdir = dir.path().join("subdir"); + fs::create_dir(&subdir).unwrap(); + + let file1 = dir.path().join("file1.txt"); + let c1 = "Content 1"; + fs::write(&file1, c1).unwrap(); + + let file2 = subdir.join("file2.txt"); + let c2 = "Content 2 more content"; + fs::write(&file2, c2).unwrap(); + + let size = calculate_folder_size(dir.path()).await.unwrap(); + assert_eq!(size, (c1.len() + c2.len()) as u64); // "Content 1".len() + "Content 2 more content".len() + } + + #[tokio::test] + async fn test_calculate_folder_size_subdirectory() { + let dir = tempdir().unwrap(); + let subdir = dir.path().join("subdir"); + fs::create_dir(&subdir).unwrap(); + + let file = subdir.join("nested.txt"); + let c1 = "Nested content"; + fs::write(&file, c1).unwrap(); + + let size = calculate_folder_size(dir.path()).await.unwrap(); + assert_eq!(size, c1.len() as u64); // "Nested content".len() + } + + #[tokio::test] + async fn test_has_enough_space_sufficient_space() { + let dir = tempdir().unwrap(); + let test_file = dir.path().join("test.txt"); + fs::write(&test_file, "test").unwrap(); + + let result = has_enough_space(&test_file, 100).await.unwrap(); + assert!(result, "Should have enough space"); + } + + #[tokio::test] + async fn test_has_enough_space_insufficient_space() { + let dir = tempdir().unwrap(); + let test_file = dir.path().join("test.txt"); + fs::write(&test_file, "test").unwrap(); + + // Use a very large number to test the boundary condition + let result = has_enough_space(&test_file, u64::MAX / 2).await.unwrap(); + assert!(!result, "Should not have enough space"); + } + + #[tokio::test] + async fn test_has_enough_space_parent_not_exists() { + let test_file = PathBuf::from("/nonexistent/path/test.txt"); + let result = has_enough_space(&test_file, 100).await; + assert!(result.is_err()); + } +} + From b3eded6a3596d4e9aa6731b7fdcd056796fe93a6 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 9 Aug 2026 16:53:49 +0200 Subject: [PATCH 24/51] fmt --- src/disks.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/disks.rs b/src/disks.rs index 29a157a..55f5db9 100644 --- a/src/disks.rs +++ b/src/disks.rs @@ -170,4 +170,3 @@ mod tests { assert!(result.is_err()); } } - From f00b1cdff8dad06ccef627f8c3fb713618a852f0 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 08:34:13 +0200 Subject: [PATCH 25/51] ssh remote: fix file size, stream uploads, fix cat/ssh pipe and quote remote path --- src/remotes/ssh.rs | 87 ++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index 732733c..114f922 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -67,6 +67,11 @@ impl fmt::Display for Error { } } +/// Quote a value for use inside a remote shell command (POSIX single-quoting). +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + #[derive(Clone)] pub struct Ssh { remote_name: String, @@ -236,11 +241,7 @@ impl remote::Remote for Ssh { } async fn upload_file(&self, path: &Path, remote_path: &Path) -> Result<(), remote::Error> { - // Read file - let mut content: Vec = vec![]; - let mut file = File::open(path).await?; - let file_size = content.len(); - file.read_to_end(&mut content).await?; + let file_size = fs::metadata(path).await?.len(); let remote_path = remote_path.to_str().unwrap(); info!( "Uploading {} bytes from {} to {}", @@ -250,11 +251,12 @@ impl remote::Remote for Ssh { ); // cat file | ssh -Pxxx user@host "cat > file" + let mut file = File::open(path).await?; let mut ssh = Command::new(&self.ssh_cmd) .args( self.ssh_args .iter() - .chain(once(&format!("cat > {}", remote_path))), + .chain(once(&format!("cat > {}", shell_quote(remote_path)))), ) .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -263,9 +265,16 @@ impl remote::Remote for Ssh { { let stdin = ssh.stdin.as_mut().unwrap(); - // This is the "cat file" on localhost piped into ssh - // when stdin is dropped - stdin.write_all(&content)?; + // This is the "cat file" on localhost piped into ssh: + // stream the file in chunks instead of reading it into memory. + let mut buf = vec![0u8; 8192]; + loop { + let n = file.read(&mut buf).await?; + if n == 0 { + break; + } + stdin.write_all(&buf[..n])?; + } } // Close stdin for being 100% sure that the process read all the file @@ -316,37 +325,41 @@ impl remote::Remote for Ssh { .stdout(Stdio::piped()) .spawn()?; - if let Some(cat_output) = cat.stdout.take() { - let mut ssh = Command::new(&self.ssh_cmd) - .stdin(cat_output) - .stdout(Stdio::null()) - .args( - self.ssh_args - .iter() - .chain(once(&format!("cat > {} ", remote_path.display()))), - ) - .spawn()?; - - cat.wait()?; - - let status = ssh.wait()?; - if !status.success() { - return Err(remote::Error::LocalError(io::Error::other( - "Failure while executing ssh command", - ))); + let cat_output = match cat.stdout.take() { + Some(out) => out, + None => { + return Err(remote::Error::LocalError(io::Error::other(format!( + "Unable to cat {}", + compressed_file.path().display() + )))) } - info!( - "Successfully uploaded compressed file {} to {}", - compressed_file.path().display(), - remote_path.display() - ); - Ok(()) - } else { - Err(remote::Error::LocalError(io::Error::other(format!( - "Unable to cat {}", - compressed_file.path().display() + }; + + let mut ssh = Command::new(&self.ssh_cmd) + .stdin(cat_output) + .stdout(Stdio::null()) + .args(self.ssh_args.iter().chain(once(&format!( + "cat > {}", + shell_quote(&remote_path.display().to_string()) )))) + .spawn()?; + + // Wait on ssh first: if ssh dies early, cat receives SIGPIPE and + // its failure must not mask the real ssh error. + let status = ssh.wait()?; + let _ = cat.wait(); + + if !status.success() { + return Err(remote::Error::LocalError(io::Error::other( + "Failure while executing ssh command", + ))); } + info!( + "Successfully uploaded compressed file {} to {}", + compressed_file.path().display(), + remote_path.display() + ); + Ok(()) } async fn upload_folder( From 9282db247d7a8a6acad9ed13ef9b281dba9ef42a Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 08:34:27 +0200 Subject: [PATCH 26/51] gitignore: ignore .DS_Store and swap files --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 785d6f3..29a8258 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ config.toml # IDE stuff .vscode/ + +# macOS +.DS_Store + +# Editor swap files +*.swp From 60f0ef733fc975af3545b8b129118b7549e99ff2 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 08:40:08 +0200 Subject: [PATCH 27/51] update dependencies to latest compatible versions --- Cargo.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 64a3c03..fca62d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,12 +14,12 @@ categories = ["command-line-utilities"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-trait = "0.1.91" +async-trait = "0.1.92" chrono = "0.4.45" dyn-clone = "1.0.20" -futures = "0.3.33" +futures = "0.3.34" glob = "0.3.4" -log = "0.4.33" +log = "0.4.34" regex = "1.13.1" scopeguard = "1.2.0" serde = { version = "1.0.229", features = ["derive"] } @@ -27,21 +27,21 @@ shellexpand = "3.1.2" stderrlog = "0.6.0" structopt = "0.3.26" tempfile = "3.27.0" -tokio = { version = "1.53.1", features = ["full"] } -toml = "1.1.4" +tokio = { version = "1.53.2", features = ["full"] } +toml = "1.1.6" url = { version = "2.5.8", features = ["serde"] } -which = "8.0.5" -aws-config = "1.10.1" +which = "8.0.6" +aws-config = "1.12.0" aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } -aws-sdk-s3 = "1.141.0" -async-compression = { version = "0.4.43", default-features = false, features = ["tokio", "gzip"] } +aws-sdk-s3 = "1.152.0" +async-compression = { version = "0.4.50", default-features = false, features = ["tokio", "gzip"] } tokio-cron-scheduler = "0.15.1" cron = "0.17.0" -uuid = { version = "1.24.0", features = ["v4"] } -aws-types = "1.5.0" +uuid = { version = "1.27.0", features = ["v4"] } +aws-types = "1.6.0" croner = "3.0.1" astral-tokio-tar = "0.6.4" get-file-size = "0.1.2" -aws-smithy-runtime-api = "1.14.0" +aws-smithy-runtime-api = "1.19.0" sysinfo = "0.38.4" itertools = "0.15.0" From 2f1b4ee9fe31656744039f2b749c7a9e7f9589cc Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 08:41:30 +0200 Subject: [PATCH 28/51] update croner, astral-tokio-tar and sysinfo to major versions --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fca62d9..a21ac99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,9 +39,9 @@ tokio-cron-scheduler = "0.15.1" cron = "0.17.0" uuid = { version = "1.27.0", features = ["v4"] } aws-types = "1.6.0" -croner = "3.0.1" -astral-tokio-tar = "0.6.4" +croner = "4.0.1" +astral-tokio-tar = "0.7.0" get-file-size = "0.1.2" aws-smithy-runtime-api = "1.19.0" -sysinfo = "0.38.4" +sysinfo = "0.39.6" itertools = "0.15.0" From 1751742b7cb18f4e0e9ef0ee721efb42099b4ac4 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 08:59:58 +0200 Subject: [PATCH 29/51] fix review issues: zfs snapshot cleanup and permission check, propagate append_dir_all error, abort failed s3 multipart uploads, skip symlinks in folder size calculation, remove unused deps, fix checks.yml newline --- .github/workflows/checks.yml | 2 +- Cargo.toml | 2 - src/disks.rs | 9 ++- src/remotes/aws.rs | 25 +++++++- src/remotes/remote.rs | 2 +- src/services/zfs.rs | 118 +++++++++++++++++++++++++++++++---- 6 files changed, 138 insertions(+), 20 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index dea6a9c..a04e2f4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -53,4 +53,4 @@ jobs: uses: clechasseur/rs-cargo@v5.0.6 with: command: clippy - args: -- -D warnings \ No newline at end of file + args: -- -D warnings diff --git a/Cargo.toml b/Cargo.toml index a21ac99..cf60049 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,4 @@ aws-types = "1.6.0" croner = "4.0.1" astral-tokio-tar = "0.7.0" get-file-size = "0.1.2" -aws-smithy-runtime-api = "1.19.0" sysinfo = "0.39.6" -itertools = "0.15.0" diff --git a/src/disks.rs b/src/disks.rs index 55f5db9..fd00fbb 100644 --- a/src/disks.rs +++ b/src/disks.rs @@ -35,13 +35,16 @@ pub async fn calculate_folder_size(path: &Path) -> Result { let mut total_size = 0u64; let mut entries = fs::read_dir(path).await?; - while let Some(entry_result) = entries.next_entry().await? { - let metadata = fs::metadata(entry_result.path()).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + // Use symlink_metadata (which does not follow symlinks) to avoid + // symlink cycles and double-counting of files reachable through links. + let metadata = fs::symlink_metadata(&path).await?; if metadata.is_file() { total_size += metadata.len(); } else if metadata.is_dir() { - total_size += Box::pin(calculate_folder_size(&entry_result.path())).await?; + total_size += Box::pin(calculate_folder_size(&path)).await?; } } diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 0699aab..9cfacd1 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -28,7 +28,7 @@ use tokio::io::AsyncReadExt; use async_trait::async_trait; -use log::info; +use log::{info, warn}; use std::io; #[derive(Clone)] @@ -159,6 +159,7 @@ impl Bucket { ); if chunk_count > MAX_CHUNKS { + self.abort_multipart_upload(remote_path, upload_id).await; return Err(AwsError::GenericError(format!( "Too many chunks: {} > {}", chunk_count, MAX_CHUNKS @@ -208,6 +209,7 @@ impl Bucket { .await; if upload_part_res.is_err() { + self.abort_multipart_upload(remote_path, upload_id).await; return Err(AwsError::RemoteError( upload_part_res.err().unwrap().into_service_error().into(), )); @@ -237,6 +239,7 @@ impl Bucket { .send() .await; if complete_multipart_upload_res.is_err() { + self.abort_multipart_upload(remote_path, upload_id).await; return Err(AwsError::RemoteError( complete_multipart_upload_res.err().unwrap().into(), )); @@ -249,6 +252,26 @@ impl Bucket { Ok(()) } + /// Best-effort abort of a multipart upload, so failed uploads don't + /// leave incomplete parts in the bucket. + async fn abort_multipart_upload(&self, remote_path: &str, upload_id: &str) { + match self + .client + .abort_multipart_upload() + .bucket(&self.bucket_name) + .key(remote_path) + .upload_id(upload_id) + .send() + .await + { + Ok(_) => info!("Aborted multipart upload for {}", remote_path), + Err(err) => warn!( + "Failed to abort multipart upload for {}: {}", + remote_path, err + ), + } + } + pub async fn delete(&self, remote_path: &str) -> Result<(), AwsError> { info!( "Deleting object {} from bucket {}", diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index 3dd37b4..46938c5 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -116,7 +116,7 @@ pub trait Remote: DynClone + Send + Sync { builder .append_dir_all(path.file_name().unwrap(), path) .await - .unwrap(); + .map_err(Error::LocalError)?; info!("Added items to archive"); let mut encoder = builder.into_inner().await?; diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 8a131e4..442b318 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; use std::string::String; use std::vec::Vec; -use log::info; +use log::{info, warn}; use crate::config::ZfsConfig; use crate::services::service::{Dump, Service}; @@ -80,21 +80,41 @@ impl Zfs { let output = output.unwrap(); let status = output.status; if !status.success() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER hold,send,snapshot {}`", config.dataset))); + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER destroy,hold,send,snapshot {}`", config.dataset))); } let stdout = String::from_utf8(output.stdout).unwrap(); if stdout.is_empty() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER hold,send,snapshot {}", config.dataset))); + return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER destroy,hold,send,snapshot {}", config.dataset))); } - // Check if the string user $USER hold,send,snapshot is in the stdout - let needle = format!("user {} hold,send,snapshot", std::env::var("USER").unwrap()); - if !stdout.contains(&needle) { - return Err(Error::ZfsError(format!( - "\"{}\" not found in output of `zfs allow {}`", - needle, config.dataset - ))); + // Check that the current user is in the allow list for snapshot, send and + // destroy (destroy is needed to clean up old snapshots created by bacup). + let user = std::env::var("USER").map_err(|_| { + Error::ZfsError( + "USER environment variable is not set, cannot verify zfs permissions".to_string(), + ) + })?; + let required = ["destroy", "hold", "send", "snapshot"]; + let prefix = format!("user {} ", user); + let allowed = stdout + .lines() + .find(|line| line.trim_start().starts_with(&prefix)) + .map(|line| { + line.trim_start() + .trim_start_matches(&prefix) + .split(',') + .map(str::trim) + .collect::>() + }); + match allowed { + Some(permissions) if required.iter().all(|perm| permissions.contains(perm)) => {} + _ => { + return Err(Error::ZfsError(format!( + "user \"{}\" with permissions {:?} not found in output of `zfs allow {}`. Please run `zfs allow {} destroy,hold,send,snapshot {}`", + user, required, config.dataset, user, config.dataset + ))); + } } // If here, the current user is in the allow list for zfs send and snapshot @@ -142,7 +162,7 @@ impl Service for Zfs { // Step 1, execute the checkpoint (atomic, immediate action) info!("Executing: {} {:?}", self.cmd.display(), args); let status = Command::new(&self.cmd) - .args(args) + .args(&args) .stdout(Stdio::null()) .status() .await; @@ -191,7 +211,11 @@ impl Service for Zfs { } let status = status?; match status.success() { - true => Ok(Dump { path: Some(dest) }), + true => { + info!("ZFS checkpoint completed successfully"); + destroy_old_snapshots(&args).await; + Ok(Dump { path: Some(dest) }) + } false => Err(Error::RuntimeError(io::Error::other(format!( "{} {:?} failed with exit code {}", self.cmd.display(), @@ -202,3 +226,73 @@ impl Service for Zfs { } } } + +/// Destroy all old snapshots matching the checkpoint name used by bacup, +/// except the one just created, so they don't accumulate indefinitely. +/// +/// `args` is the zfs send arg list; its last element is the checkpoint +/// name, e.g. `dataset@name-20260101-120000`. +async fn destroy_old_snapshots(args: &[String]) { + let Some(checkpoint) = args.last() else { + return; + }; + let Some(at_pos) = checkpoint.rfind('@') else { + return; + }; + let dataset = &checkpoint[..at_pos]; + let snapshot_prefix = &checkpoint[at_pos + 1..]; + + let output = tokio::process::Command::new("zfs") + .arg("list") + .arg("-H") + .arg("-o") + .arg("name") + .arg("-t") + .arg("snapshot") + .arg(format!("{}@{}-*", dataset, snapshot_prefix)) + .output() + .await; + + let Ok(output) = output else { + warn!("Failed to run `zfs list` to find old snapshots"); + return; + }; + if !output.status.success() { + warn!( + "zfs list failed with exit code: {}", + output.status.code().unwrap_or(-1) + ); + return; + } + let stdout = String::from_utf8_lossy(&output.stdout); + let mut destroyed = 0; + for name in stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + if name == checkpoint { + continue; // keep the snapshot we just created + } + match tokio::process::Command::new("zfs") + .arg("destroy") + .arg(name) + .output() + .await + { + Ok(o) if o.status.success() => { + info!("Destroyed old ZFS snapshot {}", name); + destroyed += 1; + } + Ok(o) => warn!( + "Failed to destroy ZFS snapshot {}: {}", + name, + String::from_utf8_lossy(&o.stderr) + ), + Err(e) => warn!("Failed to run `zfs destroy {}`: {}", name, e), + } + } + if destroyed > 0 { + info!("Destroyed {} old ZFS snapshot(s)", destroyed); + } +} From 0f536a372f38e828d7442c25984ef2ee09eae236 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 09:07:14 +0200 Subject: [PATCH 30/51] fix clippy and build warnings - remove empty stray src/main.rs that caused duplicate bin target - aws.rs: box aws_sdk_s3::Error in AwsError::RemoteError (clippy result_large_err) - backup.rs: drop needless late init in upload loop (clippy needless_late_init) - folders.rs: assert !is_empty() instead of len() > 0 (clippy len_zero) --- src/backup.rs | 11 +++++------ src/remotes/aws.rs | 28 +++++++++++++++++----------- src/services/folders.rs | 4 ++-- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 572e0cc..7fe7929 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -403,8 +403,7 @@ impl Backup { remote_prefix.join(file.strip_prefix(local_prefix).unwrap()) }; - let result: Result<(), remote::Error>; - if file.is_dir() { + let result = if file.is_dir() { // compress for sure, the uncompressed scenarios has been treated // outside this loop info!( @@ -413,7 +412,7 @@ impl Backup { file.display(), remote_path.display() ); - result = remote.upload_folder_compressed(&file, &remote_path).await; + remote.upload_folder_compressed(&file, &remote_path).await } else if compress { info!( "[{}] Compressing file {} and uploading to {}", @@ -421,7 +420,7 @@ impl Backup { file.display(), remote_path.display() ); - result = remote.upload_file_compressed(&file, &remote_path).await; + remote.upload_file_compressed(&file, &remote_path).await } else { info!( "[{}] Uploading file {} to {}", @@ -429,8 +428,8 @@ impl Backup { file.display(), remote_path.display() ); - result = remote.upload_file(&file, &remote_path).await; - } + remote.upload_file(&file, &remote_path).await + }; // Handle keep_last if let Some(to_keep) = keep_last { diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 9cfacd1..45ff4ec 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -45,7 +45,7 @@ struct Bucket { #[derive(Debug)] pub enum AwsError { - RemoteError(aws_sdk_s3::Error), + RemoteError(Box), LocalError(io::Error), GenericError(String), } @@ -58,7 +58,7 @@ impl From for AwsError { impl From for AwsError { fn from(err: aws_sdk_s3::Error) -> Self { - AwsError::RemoteError(err) + AwsError::RemoteError(Box::new(err)) } } @@ -72,7 +72,9 @@ impl Bucket { .send() .await; if response.is_err() { - return Err(AwsError::RemoteError(response.err().unwrap().into())); + return Err(AwsError::RemoteError(Box::new( + response.err().unwrap().into(), + ))); } let response = response.unwrap(); let mut ret: Vec = vec![]; @@ -116,7 +118,9 @@ impl Bucket { .await; if response.is_err() { - return Err(AwsError::RemoteError(response.err().unwrap().into())); + return Err(AwsError::RemoteError(Box::new( + response.err().unwrap().into(), + ))); } } else { // Multipart upload @@ -129,9 +133,9 @@ impl Bucket { .send() .await; if multipart_upload_res.is_err() { - return Err(AwsError::RemoteError( + return Err(AwsError::RemoteError(Box::new( multipart_upload_res.err().unwrap().into(), - )); + ))); } let multipart_upload_res = multipart_upload_res.unwrap(); info!( @@ -210,9 +214,9 @@ impl Bucket { if upload_part_res.is_err() { self.abort_multipart_upload(remote_path, upload_id).await; - return Err(AwsError::RemoteError( + return Err(AwsError::RemoteError(Box::new( upload_part_res.err().unwrap().into_service_error().into(), - )); + ))); } let upload_part_res = upload_part_res.unwrap(); @@ -240,9 +244,9 @@ impl Bucket { .await; if complete_multipart_upload_res.is_err() { self.abort_multipart_upload(remote_path, upload_id).await; - return Err(AwsError::RemoteError( + return Err(AwsError::RemoteError(Box::new( complete_multipart_upload_res.err().unwrap().into(), - )); + ))); } info!( "Multipart upload completed successfully for {}", @@ -286,7 +290,9 @@ impl Bucket { .await; if response.is_err() { - return Err(AwsError::RemoteError(response.err().unwrap().into())); + return Err(AwsError::RemoteError(Box::new( + response.err().unwrap().into(), + ))); } info!( "Successfully deleted {} from bucket {}", diff --git a/src/services/folders.rs b/src/services/folders.rs index fdf0262..5f1b7f7 100644 --- a/src/services/folders.rs +++ b/src/services/folders.rs @@ -137,7 +137,7 @@ mod tests { assert!(folder.dump().await.is_ok()); let files = folder.list().await; - assert!(files.len() > 0); + assert!(!files.is_empty()); let git_info = cwd.join(".git").join("info"); assert!(files.contains(&git_info)); @@ -173,7 +173,7 @@ mod tests { assert!(folder.dump().await.is_ok()); let files = folder.list().await; - assert!(files.len() > 0); + assert!(!files.is_empty()); let lib_path = cwd.join("src").join("lib.rs"); assert!(files.contains(&lib_path)); From 40c799bc46e75ed5eb11feb100e17d5b1fa72fa2 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 09:10:57 +0200 Subject: [PATCH 31/51] readme: document zfs permissions and snapshot cleanup --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2189c59..eafbc49 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,9 @@ When configuring the backups, the field **when** accepts configuration strings i command = "pg_dumpall -c -U postgres" # dump to stdout always [zfs] + # the user needs zfs permissions on the datasets: + # zfs allow $USER destroy,hold,send,snapshot + # old snapshots are destroyed after each successful backup [zfs.root] snapshot_name = "root-fs" dataset = "zroot" From f440cc624b2f27197a432a2cdb4a6feac885064f Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 10:08:55 +0200 Subject: [PATCH 32/51] Integrate ZFS full + incremental snapshot design Keep the working-tree design (new full_when config option, new-style full/inc snapshot naming, chain-aware cleanup) and fix the remaining integration issues: - ZfsConfig in src/config.rs gains optional full_when; Zfs parses it as a croner cron expression - restore the public Service/DynClone integration (Zfs made pub, new(config, name), Box trait impls) - drop thiserror/serde_with/crate::backup imports that did not exist as dependencies; hand-rolled Error enum with ZfsError(io) and String variants - fix broken helpers: latest_of_kind Option call, DateTime::from_naive_utc_and_offset, current_dir in list_files - send() writes zfs send's stdout (not stdin) to the dump file, as the original code did - plan_* tests use real cron expressions (croner does not parse the human 'monthly 1 01:00' form) 38 tests pass, clippy and rustfmt clean. --- src/bin/bacup.rs | 2 +- src/config.rs | 7 + src/services/zfs.rs | 961 +++++++++++++++++++++++++++++++++----------- 3 files changed, 735 insertions(+), 235 deletions(-) diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index b7bd107..5133a23 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -181,7 +181,7 @@ async fn main() -> Result<(), i32> { let key = format!("zfs.{}", service_name); services.insert( key, - Box::new(Zfs::new(instance_config, &service_name).await.unwrap()), + Box::new(Zfs::new(&instance_config, &service_name).await.unwrap()), ); } } diff --git a/src/config.rs b/src/config.rs index bd9ba9b..bbb4940 100644 --- a/src/config.rs +++ b/src/config.rs @@ -70,6 +70,13 @@ pub struct DockerConfig { pub struct ZfsConfig { pub dataset: String, pub snapshot_name: String, + /// Optional cron expression (e.g. `"0 1 1 * *"`) controlling how often a + /// full backup is taken. When set, only the runs where it is due take a + /// full backup and the runs in between are incrementals against the + /// latest existing snapshot. When absent, every run is a full backup + /// (previous behavior). + #[serde(default)] + pub full_when: Option, } #[derive(Serialize, Deserialize)] diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 442b318..a5bfc32 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -1,298 +1,791 @@ -// Copyright 2022-2026 Paolo Galeone -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - use std::fmt; use std::path::PathBuf; -use std::string::String; -use std::vec::Vec; +use std::process::Stdio; +use std::str::FromStr; -use log::{info, warn}; +use chrono::{DateTime, NaiveDateTime, Utc}; +use croner::Cron; +use log::{debug, info, warn}; +use tokio::fs; +use tokio::process::Command; use crate::config::ZfsConfig; use crate::services::service::{Dump, Service}; -use which::which; - -use async_trait::async_trait; -use tokio::{fs::metadata, fs::File, io}; +const TS_FORMAT: &str = "%Y%m%d-%H%M%S"; +const FULL_MARK: &str = "full"; +const INC_MARK: &str = "inc"; +const SNAPSHOT_EXT: &str = ".snapshot"; -use std::process::Stdio; -use tokio::process::Command; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SnapshotKind { + /// A full snapshot, taken when the `full_when` schedule was due. + Full, + /// An incremental snapshot taken against a previous snapshot. + Incremental, + /// A snapshot taken by a bacup version without full/incremental naming. + /// Legacy snapshots count as valid bases for incrementals and are + /// cleaned up like any other of our snapshots. + Legacy, +} -#[derive(Clone)] -pub struct Zfs { +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Snapshot { + /// The full name as reported by `zfs list`, e.g. `tank@snap-full-20260101-010000`. + /// Child datasets carry the same snapshot name (e.g. `tank/data@snap-full-...`). pub name: String, - pub cmd: PathBuf, - pub args: Vec, + pub kind: SnapshotKind, + /// Time encoded in the snapshot name. + pub time: DateTime, +} + +/// Parse a `zfs list` snapshot name (e.g. `tank/data@snap-inc-20260101-010000`) +/// into a [Snapshot] if it belongs to the snapshot chain rooted at +/// `snapshot_base`, i.e. it is named `-full-`, `-inc-` +/// (new style) or `-` (legacy style). Returns `None` for names that +/// do not belong to this chain. +pub(crate) fn parse_snapshot(name: &str, snapshot_base: &str) -> Option { + let after_at = name.split('@').nth(1)?; + let rest = after_at.strip_prefix(snapshot_base)?.strip_prefix('-')?; + let (kind, ts) = if let Some(ts) = rest.strip_prefix(&format!("{FULL_MARK}-")) { + (SnapshotKind::Full, ts) + } else if let Some(ts) = rest.strip_prefix(&format!("{INC_MARK}-")) { + (SnapshotKind::Incremental, ts) + } else { + (SnapshotKind::Legacy, rest) + }; + let naive = NaiveDateTime::parse_from_str(ts, TS_FORMAT).ok()?; + let time = DateTime::from_naive_utc_and_offset(naive, Utc); + Some(Snapshot { + name: name.to_string(), + kind, + time, + }) +} + +/// Format the timestamp part used in snapshot names and dump file names. +pub(crate) fn format_ts(now: DateTime) -> String { + now.format(TS_FORMAT).to_string() +} + +/// Local dump file name for a new snapshot, e.g. `mysnap-inc-20260101-010000.snapshot`. +/// Remote names track local names, so the name must be unique per snapshot +/// for the incremental chain to survive on the remote side. +pub(crate) fn dump_file_name(name: &str, kind: SnapshotKind, ts: &str) -> String { + let mark = match kind { + SnapshotKind::Full => FULL_MARK, + SnapshotKind::Incremental => INC_MARK, + // Legacy snapshots are never created; their dump files use the + // legacy file name `.snapshot` directly. + SnapshotKind::Legacy => return format!("{name}{SNAPSHOT_EXT}"), + }; + format!("{name}-{mark}-{ts}{SNAPSHOT_EXT}") +} + +/// Arguments for the `zfs send` command to dump the given snapshot. +/// A full snapshot is sent with `send -R -v`, an incremental one with +/// `send -R -v -i ` where `` is the name of the snapshot it is +/// based on. +pub(crate) fn send_args(kind: SnapshotKind, base: Option<&str>, new_name: &str) -> Vec { + match (kind, base) { + (SnapshotKind::Incremental, Some(base)) => { + vec![ + "send".into(), + "-R".into(), + "-v".into(), + "-i".into(), + base.into(), + new_name.into(), + ] + } + // A full snapshot, or (defensively) an incremental without a base. + _ => vec!["send".into(), "-R".into(), "-v".into(), new_name.into()], + } +} + +/// Decide whether the next run at `now` should take a full or an +/// incremental snapshot. +/// +/// A full is due when `full_when` is absent (always full, previous behavior), +/// when no full snapshot exists yet, or when the next occurrence of +/// `full_when` after the last full is not in the future. Otherwise the run +/// is an incremental against the latest existing snapshot of any kind. If no +/// base snapshot exists (e.g. first run) an incremental is impossible, so a +/// full is taken. +pub(crate) fn select_plan( + now: DateTime, + full_when: Option<&Cron>, + snapshots: &[Snapshot], +) -> Vec { + let is_full_due = match ( + full_when, + latest_of_kind(snapshots, Some(SnapshotKind::Full)), + ) { + (None, _) => true, + (Some(_), None) => true, + (Some(cron), Some(last_full)) => match cron.find_next_occurrence(&last_full.time, false) { + Ok(next) => next <= now, + // On parse issues fall back to a full: safe and self-healing. + Err(_) => true, + }, + }; + + if is_full_due { + return vec!["full".to_string()]; + } + + match latest_of_kind(snapshots, None).map(|s| s.name.clone()) { + Some(base) => vec!["inc".to_string(), base], + // No base snapshot available, so an incremental is impossible. + None => vec!["full".to_string()], + } +} + +/// Returns the latest snapshot in `snapshots` of the given kind, or the +/// latest snapshot of any kind when `kind` is `None`. +fn latest_of_kind(snapshots: &[Snapshot], kind: Option) -> Option<&Snapshot> { + snapshots + .iter() + .filter(|s| kind.map(|k| s.kind == k).unwrap_or(true)) + .max_by(|a, b| a.time.cmp(&b.time)) +} + +/// The names of the snapshots to destroy after a new full snapshot has been +/// created and sent: every snapshot of our chain except the new full +/// (which, being the newest, is the only one the incremental chain needs). +pub(crate) fn cleanup_list(new_full_name: &str, snapshots: &[Snapshot]) -> Vec { + snapshots + .iter() + .map(|s| s.name.clone()) + .filter(|n| n != new_full_name) + .collect() +} + +/// Filter out the new-style local dump files (matching `-{full,inc}-.snapshot`) +/// older than the snapshot with name `newer_name`, i.e. the ones to remove +/// after a new full backup. Entries that do not belong to our chain are dropped. +pub(crate) fn old_dump_files(entries: &[String], name: &str, newer_name: &str) -> Vec { + let Some(newer) = parse_snapshot_name(newer_name, name) else { + return vec![]; + }; + entries + .iter() + .filter_map(|e| parse_snapshot_name(e, name)) + .filter(|s| s.time < newer.time) + .map(|s| s.name.clone()) + .collect() } #[derive(Debug)] pub enum Error { - CommandNotFound(which::Error), - RuntimeError(io::Error), - ZfsError(String), + ZfsError(std::io::Error), + String(String), } -impl std::error::Error for Error {} impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Error::CommandNotFound(error) => write!(f, "Command not found: {}", error), - Error::RuntimeError(error) => write!(f, "Runtime error: {}", error), - Error::ZfsError(error) => write!(f, "ZFS error: {}", error), + Error::ZfsError(e) => write!(f, "zfs error: {e}"), + Error::String(s) => write!(f, "{s}"), } } } -impl Zfs { - pub async fn new(config: ZfsConfig, name: &str) -> Result { - let cmd = match which("zfs") { - Err(error) => return Err(Error::CommandNotFound(error)), - Ok(cmd) => cmd, - }; +impl std::error::Error for Error {} - // Verify zfs is working and the current user is in the allow list for executing - // snapshot and send - let args = vec![String::from("allow"), config.dataset.clone()]; - info!("Executing {} {:?}", cmd.display(), args); - let output = Command::new(&cmd) - .args(args) - // capture output to variable to check if the user is there - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await; +impl From for Error { + fn from(e: std::io::Error) -> Self { + Error::ZfsError(e) + } +} - if let Err(err) = output { - return Err(Error::RuntimeError(err)); - } - let output = output.unwrap(); - let status = output.status; - if !status.success() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER destroy,hold,send,snapshot {}`", config.dataset))); - } +impl From for Error { + fn from(e: String) -> Self { + Error::String(e) + } +} - let stdout = String::from_utf8(output.stdout).unwrap(); - if stdout.is_empty() { - return Err(Error::ZfsError(format!("Failed to verify zfs user permissions. Please run `zfs allow $USER destroy,hold,send,snapshot {}", config.dataset))); - } +/// Zfs service, based on the zfs command line tool. +/// +/// With `full_when` set, runs where the expression is due take a full backup +/// and the runs in between take incrementals against the latest existing +/// snapshot of the chain; without it, every run is a full backup. +#[derive(Clone)] +pub struct Zfs { + name: String, + cmd: String, + dataset: String, + snapshot_base: String, + full_when: Option, +} - // Check that the current user is in the allow list for snapshot, send and - // destroy (destroy is needed to clean up old snapshots created by bacup). +impl Zfs { + pub async fn new(config: &ZfsConfig, name: &str) -> Result { + let dataset = &config.dataset; + let snapshot_base = &config.snapshot_name; + + let zfs = + which::which("zfs").map_err(|_| Error::String("zfs not found in PATH".to_string()))?; + let cmd = zfs.to_str().unwrap().to_string(); + + let full_when = match &config.full_when { + Some(expr) => { + let cron = Cron::from_str(expr).map_err(|e| { + Error::String(format!("invalid full_when expression {expr}: {e}")) + })?; + Some(cron) + } + None => None, + }; + + // Check that the current user is allowed to manage snapshots of the dataset. let user = std::env::var("USER").map_err(|_| { - Error::ZfsError( - "USER environment variable is not set, cannot verify zfs permissions".to_string(), + Error::String( + "USER environment variable is not set, cannot check zfs permissions".to_string(), ) })?; - let required = ["destroy", "hold", "send", "snapshot"]; - let prefix = format!("user {} ", user); - let allowed = stdout - .lines() - .find(|line| line.trim_start().starts_with(&prefix)) - .map(|line| { - line.trim_start() - .trim_start_matches(&prefix) - .split(',') - .map(str::trim) - .collect::>() - }); - match allowed { - Some(permissions) if required.iter().all(|perm| permissions.contains(perm)) => {} - _ => { - return Err(Error::ZfsError(format!( - "user \"{}\" with permissions {:?} not found in output of `zfs allow {}`. Please run `zfs allow {} destroy,hold,send,snapshot {}`", - user, required, config.dataset, user, config.dataset - ))); - } + let allow_output = Command::new(&cmd) + .arg("allow") + .arg(dataset) + .output() + .await?; + if !allow_output.status.success() { + return Err(Error::String(format!( + "failed to check zfs permissions on {dataset}: {}", + String::from_utf8_lossy(&allow_output.stderr) + ))); + } + let allow_text = String::from_utf8_lossy(&allow_output.stdout).to_string(); + let required = ["destroy", "list", "send", "snapshot"]; + let ok = allow_text.lines().any(|line| { + let line = line.trim(); + let is_user_line = line.starts_with(&format!("user {user}")) + || line.starts_with(&format!("user '{user}'")) + || line.starts_with(&format!("user \"{}\"", user)); + is_user_line && required.iter().all(|p| line.contains(p)) + }); + if !ok { + let required_list = required.join(","); + return Err(Error::String(format!( + "user \"{user}\" is not allowed to manage zfs snapshots on {dataset}. \ + Run `zfs allow {user} {required_list} {dataset}` (permissions are inherited \ + by child datasets) and make sure `list` is granted so bacup can enumerate snapshots." + ))); } - // If here, the current user is in the allow list for zfs send and snapshot - - let args: Vec = vec![ - String::from("snapshot"), - String::from("-r"), - format!( - "{}@{}", - config.dataset.clone().trim(), - config.snapshot_name.trim() - ), - ]; - + debug!("new zfs service on {dataset} (snapshot base {snapshot_base})"); Ok(Zfs { - name: String::from(name), - args, + name: name.to_string(), cmd, + dataset: dataset.clone(), + snapshot_base: snapshot_base.clone(), + full_when, }) } -} -#[async_trait] -impl Service for Zfs { - async fn list(&self) -> Vec { - let dest = std::env::current_dir() - .unwrap() - .join(PathBuf::from(format!("{}.snapshot", self.name))); + /// List all snapshots of our chain on the dataset tree, including those + /// on child datasets (created with `-r`). + async fn list_our_snapshots(&self) -> Vec { + let output = match Command::new(&self.cmd) + .arg("list") + .arg("-r") + .arg("-H") + .arg("-o") + .arg("name") + .arg("-t") + .arg("snapshot") + .arg(&self.dataset) + .output() + .await + { + Ok(output) if output.status.success() => output, + _ => return vec![], + }; + String::from_utf8_lossy(&output.stdout) + .lines() + .map(|line| line.trim().to_string()) + .filter_map(|line| parse_snapshot(&line, &self.snapshot_base)) + .collect() + } - if metadata(&dest).await.is_ok() { - return vec![dest]; + async fn create_snapshot(&self, name: &str) -> Result<(), Error> { + let status = Command::new(&self.cmd) + .arg("snapshot") + .arg("-r") + .arg(name) + .status() + .await?; + if !status.success() { + return Err(Error::String(format!( + "zfs snapshot {name} failed (exit {status})" + ))); } - return vec![]; + Ok(()) } - async fn dump(&self) -> Result> { - let mut args = self.args.clone(); - let checkpoint = args.last_mut().unwrap(); - - // Current date in ISO format - checkpoint.push_str(&format!("-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"))); - - let checkpoint_name = checkpoint.clone(); - - // Step 1, execute the checkpoint (atomic, immediate action) - info!("Executing: {} {:?}", self.cmd.display(), args); + async fn send(&self, args: &[String], dest: &str) -> Result<(), Error> { + let dest_file = fs::File::create(dest).await?; let status = Command::new(&self.cmd) - .args(&args) - .stdout(Stdio::null()) + .args(args) + .stdout(Stdio::from(dest_file.try_into_std().unwrap())) .status() - .await; - if status.is_err() { - return Err(Error::RuntimeError(status.err().unwrap()).into()); + .await?; + + if !status.success() { + return Err(Error::String(format!("zfs send failed (exit {status})"))); } + Ok(()) + } - // Step 2, send the checkpoint to a local file, named: name.snapshot - let dest = std::env::current_dir() - .unwrap() - .join(PathBuf::from(format!("{}.snapshot", self.name))); - - let parent = dest.parent().unwrap(); - if !parent.exists() { - return Err(Error::RuntimeError(io::Error::other(format!( - "Folder {} does not exist.", - parent.display() - ))) - .into()); + async fn destroy(&self, name: &str) -> bool { + match Command::new(&self.cmd) + .arg("destroy") + .arg(name) + .status() + .await + { + Ok(status) => status.success(), + Err(e) => { + warn!("failed to run zfs destroy {name}: {e}"); + false + } } + } +} - let dest_file = File::create(&dest).await?; - // TODO: send dry-run, get the "total estimated size is XXX$unit" - // convert to bytes, and use has_enough_space to check. - let send_args = vec![ - String::from("send"), - String::from("-R"), - String::from("-v"), - checkpoint_name.to_string(), - ]; +#[async_trait::async_trait] +impl Service for Zfs { + async fn dump(&self) -> Result> { + self.do_dump() + .await + .map_err(|e| Box::new(e) as Box) + } + + async fn list(&self) -> Vec { + self.list_files().await + } +} + +impl Zfs { + async fn do_dump(&self) -> Result { + let now = Utc::now(); + let ts = format_ts(now); + + let snapshots = self.list_our_snapshots().await; + let plan = select_plan(now, self.full_when.as_ref(), &snapshots); + let is_full = plan[0] == "full"; + let kind: SnapshotKind = if is_full { + SnapshotKind::Full + } else { + SnapshotKind::Incremental + }; + let base = plan.get(1).cloned(); + let new_name = format!( + "{}@{}-{}-{}", + self.dataset, + self.snapshot_base, + if is_full { FULL_MARK } else { INC_MARK }, + ts + ); + let dest = dump_file_name(&self.name, kind, &ts); info!( - "Executing: {} {:?} > {}", - self.cmd.display(), - send_args, - dest.display() + "taking {} zfs snapshot {} of {}{}", + if is_full { "full" } else { "incremental" }, + new_name, + self.dataset, + base.as_ref() + .map(|b| format!(" based on {b}")) + .unwrap_or_default() ); + self.create_snapshot(&new_name).await?; - let status = Command::new(&self.cmd) - .args(&send_args) - .stdout(Stdio::from(dest_file.try_into_std().unwrap())) - .status() - .await; - if let Err(error) = status { - return Err(Error::RuntimeError(error).into()); + if is_full { + // A full chain replaces all previous ones: dump it and clean up. + let send = send_args(kind, base.as_deref(), &new_name); + self.send(&send, &dest).await?; + info!("ZFS full checkpoint completed successfully"); + self.cleanup(&snapshots, &new_name).await; + } else { + // Incremental: fall back to a full if the base is gone + // (e.g. a child dataset was destroyed and recreated). + let base = base.expect("plan guarantees a base for incrementals"); + let send = send_args(kind, Some(&base), &new_name); + if self.send(&send, &dest).await.is_err() { + warn!( + "incremental send from base {base} failed; \ + destroying the incremental snapshot and retrying as a full backup" + ); + if self.destroy(&new_name).await { + info!("incremental snapshot {new_name} destroyed"); + } else { + warn!("failed to destroy incremental snapshot {new_name}"); + } + let full_name = format!("{}@{}-{FULL_MARK}-{ts}", self.dataset, self.snapshot_base); + self.create_snapshot(&full_name).await?; + let send = send_args(SnapshotKind::Full, None, &full_name); + self.send(&send, &dest).await?; + info!("ZFS full checkpoint completed successfully (incremental fallback)"); + self.cleanup(&snapshots, &full_name).await; + } else { + info!("ZFS incremental checkpoint completed successfully (base {base})"); + } } - let status = status?; - match status.success() { - true => { - info!("ZFS checkpoint completed successfully"); - destroy_old_snapshots(&args).await; - Ok(Dump { path: Some(dest) }) + + Ok(Dump { + path: Some(dest.into()), + }) + } + + /// List the dump files this service should upload: the newest new-style + /// dump file (or the legacy `.snapshot` if none exists yet). + async fn list_files(&self) -> Vec { + let current_dir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let mut entries = vec![]; + let mut newest: Option = None; + + let mut read_dir = match fs::read_dir(¤t_dir).await { + Ok(rd) => rd, + Err(e) => { + warn!("could not read dump directory {:?}: {e}", current_dir); + return entries; + } + }; + loop { + let Some(entry) = (match read_dir.next_entry().await { + Ok(Some(e)) => Some(e), + Ok(None) => None, + Err(e) => { + warn!("could not read dump directory entry: {e}"); + None + } + }) else { + break; + }; + let file_name = entry.file_name().to_string_lossy().to_string(); + // New-style dump files: -{full,inc}-.snapshot + let Some(snap) = parse_snapshot_name(&file_name, &self.name) else { + continue; + }; + if newest.as_ref().map(|n| snap.time > n.time).unwrap_or(true) { + newest = Some(snap); + } + } + + if let Some(snap) = newest { + entries.push(current_dir.join(snap.name)); + } else if let Ok(legacy) = fs::metadata(format!("{}{SNAPSHOT_EXT}", self.name)).await { + if legacy.is_file() { + entries.push(current_dir.join(format!("{}{SNAPSHOT_EXT}", self.name))); } - false => Err(Error::RuntimeError(io::Error::other(format!( - "{} {:?} failed with exit code {}", - self.cmd.display(), - self.args, - status.code().unwrap() - ))) - .into()), } + + entries } } -/// Destroy all old snapshots matching the checkpoint name used by bacup, -/// except the one just created, so they don't accumulate indefinitely. -/// -/// `args` is the zfs send arg list; its last element is the checkpoint -/// name, e.g. `dataset@name-20260101-120000`. -async fn destroy_old_snapshots(args: &[String]) { - let Some(checkpoint) = args.last() else { - return; +/// Parse a local dump file name into a [Snapshot] if it belongs to the chain +/// rooted at `name` (new-style `-{full,inc}-.snapshot`). +/// The returned [Snapshot::name] is the file name itself. +fn parse_snapshot_name(file_name: &str, name: &str) -> Option { + let stem = file_name.strip_suffix(SNAPSHOT_EXT)?; + let after_name = stem.strip_prefix(name)?.strip_prefix('-')?; + // Local dump files are only ever new-style. + let (kind, ts) = if let Some(ts) = after_name.strip_prefix(&format!("{FULL_MARK}-")) { + (SnapshotKind::Full, ts) + } else { + let ts = after_name.strip_prefix(&format!("{INC_MARK}-"))?; + (SnapshotKind::Incremental, ts) }; - let Some(at_pos) = checkpoint.rfind('@') else { - return; - }; - let dataset = &checkpoint[..at_pos]; - let snapshot_prefix = &checkpoint[at_pos + 1..]; - - let output = tokio::process::Command::new("zfs") - .arg("list") - .arg("-H") - .arg("-o") - .arg("name") - .arg("-t") - .arg("snapshot") - .arg(format!("{}@{}-*", dataset, snapshot_prefix)) - .output() - .await; - - let Ok(output) = output else { - warn!("Failed to run `zfs list` to find old snapshots"); - return; - }; - if !output.status.success() { - warn!( - "zfs list failed with exit code: {}", - output.status.code().unwrap_or(-1) - ); - return; - } - let stdout = String::from_utf8_lossy(&output.stdout); - let mut destroyed = 0; - for name in stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - { - if name == checkpoint { - continue; // keep the snapshot we just created - } - match tokio::process::Command::new("zfs") - .arg("destroy") - .arg(name) - .output() - .await - { - Ok(o) if o.status.success() => { - info!("Destroyed old ZFS snapshot {}", name); + let naive = NaiveDateTime::parse_from_str(ts, TS_FORMAT).ok()?; + let time = DateTime::from_naive_utc_and_offset(naive, Utc); + Some(Snapshot { + name: file_name.to_string(), + kind, + time, + }) +} + +impl Zfs { + /// After a successful full: destroy all older snapshots of our chain on + /// the dataset tree and remove old local dump files. Best effort: failures + /// are logged but do not fail the backup. + async fn cleanup(&self, snapshots: &[Snapshot], new_full_name: &str) { + let to_destroy = cleanup_list(new_full_name, snapshots); + let mut destroyed = 0; + for name in &to_destroy { + if self.destroy(name).await { + info!("destroyed old zfs snapshot {name}"); destroyed += 1; + } else { + warn!("failed to destroy old zfs snapshot {name}"); } - Ok(o) => warn!( - "Failed to destroy ZFS snapshot {}: {}", - name, - String::from_utf8_lossy(&o.stderr) - ), - Err(e) => warn!("Failed to run `zfs destroy {}`: {}", name, e), } + if !to_destroy.is_empty() { + info!( + "destroyed {destroyed}/{} old zfs snapshot(s)", + to_destroy.len() + ); + } + + let files: Vec = match fs::read_dir(".").await { + Ok(mut read_dir) => { + let mut files = vec![]; + while let Ok(Some(entry)) = read_dir.next_entry().await { + files.push(entry.file_name().to_string_lossy().to_string()); + } + files + } + Err(e) => { + warn!("failed to list local dump files for cleanup: {e}"); + vec![] + } + }; + for file in old_dump_files(&files, &self.name, new_full_name) { + match fs::remove_file(&file).await { + Ok(()) => info!("removed old local dump file {file}"), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!("failed to remove old local dump file {file}: {e}"), + } + } + } +} + +impl fmt::Display for Zfs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "Zfs {{ name: {:?}, cmd: {:?}, dataset: {:?}, full_when: {:?} }}", + self.name, + self.cmd, + self.dataset, + self.full_when.as_ref().map(|c| c.to_string()) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn t(ymd: &str) -> DateTime { + Utc.with_ymd_and_hms( + ymd[0..4].parse().unwrap(), + ymd[4..6].parse().unwrap(), + ymd[6..8].parse().unwrap(), + 1, + 0, + 0, + ) + .unwrap() + } + + fn snap(name: &str, kind: SnapshotKind, ymd: &str) -> Snapshot { + Snapshot { + name: name.to_string(), + kind, + time: t(ymd), + } + } + + #[test] + fn parse_snapshot_new_style() { + let s = parse_snapshot("tank@snap-full-20260101-010000", "snap").unwrap(); + assert_eq!(s.kind, SnapshotKind::Full); + assert_eq!(s.time, t("20260101")); + + let s = parse_snapshot("tank/data@snap-inc-20260102-020000", "snap").unwrap(); + assert_eq!(s.kind, SnapshotKind::Incremental); + assert_eq!(s.name, "tank/data@snap-inc-20260102-020000"); + } + + #[test] + fn parse_snapshot_legacy() { + let s = parse_snapshot("tank@snap-20251231-235959", "snap").unwrap(); + assert_eq!(s.kind, SnapshotKind::Legacy); + } + + #[test] + fn parse_snapshot_rejects_foreign() { + // Different base name. + assert!(parse_snapshot("tank@other-20260101-010000", "snap").is_none()); + // Prefix collision: "snapx" is not "snap". + assert!(parse_snapshot("tank@snapx-20260101-010000", "snap").is_none()); + // Malformed timestamp. + assert!(parse_snapshot("tank@snap-full-20260101-01", "snap").is_none()); + // Missing @. + assert!(parse_snapshot("tank", "snap").is_none()); + // Not our mark and not a valid timestamp: "snap-full" without ts. + assert!(parse_snapshot("tank@snap-full", "snap").is_none()); + } + + #[test] + fn dump_file_names() { + assert_eq!( + dump_file_name("snap", SnapshotKind::Full, "20260101-010000"), + "snap-full-20260101-010000.snapshot" + ); + assert_eq!( + dump_file_name("snap", SnapshotKind::Incremental, "20260102-020000"), + "snap-inc-20260102-020000.snapshot" + ); + } + + #[test] + fn send_args_full_and_inc() { + assert_eq!( + send_args(SnapshotKind::Full, None, "tank@snap-full-1"), + vec!["send", "-R", "-v", "tank@snap-full-1"] + ); + assert_eq!( + send_args( + SnapshotKind::Incremental, + Some("tank@snap-full-1"), + "tank@snap-inc-2" + ), + vec![ + "send", + "-R", + "-v", + "-i", + "tank@snap-full-1", + "tank@snap-inc-2" + ] + ); + } + + #[test] + fn plan_without_full_when_is_always_full() { + let snaps = vec![snap( + "tank@snap-inc-20260101-010000", + SnapshotKind::Incremental, + "20260101", + )]; + assert_eq!(select_plan(t("20260201"), None, &snaps), vec!["full"]); + } + + #[test] + fn plan_first_run_is_full() { + let cron = Cron::from_str("0 1 1 * *").unwrap(); + assert_eq!(select_plan(t("20260115"), Some(&cron), &[]), vec!["full"]); + } + + #[test] + fn plan_incremental_before_full_is_due() { + let cron = Cron::from_str("0 1 1 * *").unwrap(); + // Last full on Jan 1; it is now Jan 15: next full (Feb 1) not due. + let snaps = vec![ + snap( + "tank@snap-full-20260101-010000", + SnapshotKind::Full, + "20260101", + ), + snap( + "tank@snap-inc-20260114-010000", + SnapshotKind::Incremental, + "20260114", + ), + ]; + assert_eq!( + select_plan(t("20260115"), Some(&cron), &snaps), + vec![ + "inc".to_string(), + "tank@snap-inc-20260114-010000".to_string() + ] + ); + } + + #[test] + fn plan_full_when_due() { + let cron = Cron::from_str("0 1 1 * *").unwrap(); + // Last full on Jan 1; it is now Feb 1: full is due. + let snaps = vec![ + snap( + "tank@snap-full-20260101-010000", + SnapshotKind::Full, + "20260101", + ), + snap( + "tank@snap-inc-20260131-010000", + SnapshotKind::Incremental, + "20260131", + ), + ]; + assert_eq!( + select_plan(t("20260201"), Some(&cron), &snaps), + vec!["full"] + ); + } + + #[test] + fn plan_falls_back_to_full_without_base() { + let cron = Cron::from_str("0 1 1 * *").unwrap(); + // Non-due time but no snapshots at all: incremental impossible. + assert_eq!(select_plan(t("20260115"), Some(&cron), &[]), vec!["full"]); } - if destroyed > 0 { - info!("Destroyed {} old ZFS snapshot(s)", destroyed); + + #[test] + fn cleanup_keeps_only_new_full() { + let snaps = vec![ + snap( + "tank@snap-full-20260101-010000", + SnapshotKind::Full, + "20260101", + ), + snap( + "tank@snap-inc-20260102-010000", + SnapshotKind::Incremental, + "20260102", + ), + snap( + "tank/data@snap-inc-20260102-010000", + SnapshotKind::Incremental, + "20260102", + ), + snap( + "tank@snap-20251231-235959", + SnapshotKind::Legacy, + "20251231", + ), + ]; + let cleaned = cleanup_list("tank@snap-full-20260201-010000", &snaps); + assert_eq!( + cleaned, + vec![ + "tank@snap-full-20260101-010000", + "tank@snap-inc-20260102-010000", + "tank/data@snap-inc-20260102-010000", + "tank@snap-20251231-235959", + ] + ); + } + + #[test] + fn parse_snapshot_name_file() { + let s = parse_snapshot_name("snap-inc-20260102-020000.snapshot", "snap").unwrap(); + assert_eq!(s.kind, SnapshotKind::Incremental); + assert_eq!(s.name, "snap-inc-20260102-020000.snapshot"); + assert!(parse_snapshot_name("snap-20260102-020000.snapshot", "snap").is_none()); + assert!(parse_snapshot_name("other-full-20260102-020000.snapshot", "snap").is_none()); + } + + #[test] + fn old_dump_files_are_strictly_older() { + let entries = vec![ + "snap-full-20260101-010000.snapshot".to_string(), + "snap-inc-20260101-120000.snapshot".to_string(), + "snap-full-20260102-010000.snapshot".to_string(), // the new full itself + "other-full-20260101-010000.snapshot".to_string(), // not ours + "readme.txt".to_string(), + ]; + let old = old_dump_files(&entries, "snap", "snap-full-20260102-010000.snapshot"); + assert_eq!( + old, + vec![ + "snap-full-20260101-010000.snapshot".to_string(), + "snap-inc-20260101-120000.snapshot".to_string(), + ] + ); } } From 131be8f7803266b3716ee4a87de6d1887ee2810b Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 14:58:48 +0200 Subject: [PATCH 33/51] ZFS full_when accepts the 'when' format; enforce keep_last restorability; zfs-side compression - Extract the human when-format parser from backup.rs into the shared src/when.rs module; zfs full_when now accepts the same format as the when field (e.g. 'monthly 1 01:00') or a raw cron expression. - Fix the weekly day-of-week off-by-one (chrono Weekday numbering starts at Sunday): 'weekly monday' now maps to cron dow 1. - Startup gate: for every zfs backup with full_when, compute the required keep_last (max runs between two fulls + 1) and refuse to start if the configured keep_last would prune the full that incrementals still need to restore from. - zfs send now uses -c -L so blocks compressed on disk stay compressed in the dump (OpenZFS >= 2.1.1 sender); log a hint at startup when the dataset is compressed and the backup has compression enabled. - Document the ZFS backup/restore/keep_last semantics in the README. --- README.md | 40 ++++++- src/backup.rs | 255 +++++++------------------------------------ src/bin/bacup.rs | 115 ++++++++++++++++++- src/config.rs | 11 +- src/lib.rs | 1 + src/services/zfs.rs | 198 +++++++++++++++++++++++++++++++-- src/when.rs | 261 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 646 insertions(+), 235 deletions(-) create mode 100644 src/when.rs diff --git a/README.md b/README.md index eafbc49..92836de 100644 --- a/README.md +++ b/README.md @@ -81,15 +81,20 @@ When configuring the backups, the field **when** accepts configuration strings i command = "pg_dumpall -c -U postgres" # dump to stdout always [zfs] - # the user needs zfs permissions on the datasets: - # zfs allow $USER destroy,hold,send,snapshot - # old snapshots are destroyed after each successful backup + # the user needs zfs permissions on the datasets (bacup verifies them at startup): + # zfs allow $USER destroy,list,send,snapshot + # (permissions are inherited by child datasets) [zfs.root] snapshot_name = "root-fs" dataset = "zroot" [zfs.storage] snapshot_name = "storage-fs" dataset = "storage" + # optional: how often a full backup is taken. Accepts the same format + # as the `when` field (e.g. "monthly 1 01:00") or a raw cron expression. + # The runs in between take incremental backups against the latest + # snapshot. When omitted every run is a full backup. + #full_when = "monthly 1 01:00" # mapping services to remote [backup] @@ -156,6 +161,35 @@ When `compression = true`, the file/folder are compressed using Gzip and the fil YYYY-MM-DD-hh:mm-filename.gz # or .tar.gz if filename is an archive ``` +## ZFS backups + +The `zfs` service backs up a dataset tree with `zfs snapshot` + `zfs send`. Every service produces one dump file per run, which is then uploaded to the remote like any other backup: the schedule, `remote_path` and `keep_last` are the usual `[backup.]` fields (`what = "zfs."`). + +### Full and incremental backups + +- On every run bacup creates a snapshot `dataset@snapshot_name--` on the dataset and all of its children, then `zfs send -R` writes it to the working directory as `--.snapshot` (`` is `full` or `inc`). +- Without `full_when` every run is a full backup. +- With `full_when` set, a full backup is taken when the schedule is due (the first run is always a full) and the runs in between are **incrementals** against the latest existing snapshot of the chain, so they only contain what changed since the last run and stay small. +- If an incremental can't be sent (e.g. a child dataset was destroyed and recreated), bacup destroys the incremental snapshot and retries the run as a full backup, so a run never fails silently. +- After each **full** backup the previous chain is replaced: all older snapshots of the service (including those on child datasets) and the older local dump files are destroyed. On the remote, dump files are pruned by `keep_last` as usual. +- The dump is produced with `zfs send -c -L`: blocks that are compressed on disk stay compressed in the dump file (the `-c` flag needs OpenZFS >= 2.1.1 on the sender). If the dataset uses compression (check with `zfs get -o value compression `), set `compress = false` on the backup — gzip'ing an already-compressed stream at upload time is wasted CPU with no size gain. Keep `compress = true` only for uncompressed datasets. bacup logs a hint at startup when the dataset is compressed. + +### Restoring + +Dump files are standard `zfs receive` streams. To restore, take the newest full and then apply every incremental after it, in timestamp order: + +``` +zfs receive -F targetds < -full-20260901-010000.snapshot +zfs receive -F targetds < -inc-20260902-010000.snapshot +zfs receive -F targetds < -inc-20260903-010000.snapshot +``` + +Dumps of compressed datasets carry compressed blocks (sent with `-c`), so the receiving pool must have the matching compression features enabled (`lz4_compress`/`zstd_compress`) — i.e. the same or a newer ZFS than the original pool. + +A restore therefore needs the newest full and every incremental taken after it. If you plan to restore from the remote, size `keep_last` so it can never prune a full that incrementals on top of it still need. + +bacup enforces this at startup: for every zfs backup with a `full_when`, it computes the longest stretch of incremental runs between two fulls from the two schedules and refuses to start if `keep_last` is set below what keeps a full and all its incrementals. Without `keep_last` nothing is pruned, so nothing can break the chain. + ## Installation & service setup ``` diff --git a/src/backup.rs b/src/backup.rs index 7fe7929..56a4356 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -15,9 +15,9 @@ use crate::config::BackupConfig; use crate::remotes::remote; use crate::services::service::Service; +use crate::when; use cron::Schedule; -use regex::Regex; use std::fmt; use std::io; use std::path::{Path, PathBuf}; @@ -26,7 +26,6 @@ use std::sync::Arc; use tokio_cron_scheduler::JobSchedulerError; use tokio_cron_scheduler::{Job, JobScheduler}; -use chrono::Weekday; use log::{error, info}; use uuid::Uuid; @@ -63,15 +62,30 @@ pub struct Backup { } impl Backup { - fn get_hours_and_minutes(when: &str) -> Option<(i8, i8)> { - let re = Regex::new(r"(\d{2}):(\d{2})").unwrap(); - let cap = re.captures(when)?; + pub async fn new( + name: &str, + remote: Box, + service: Box, + config: &BackupConfig, + ) -> Result { + let parsable = when::parse_when(&config.when).ok(); + let to_parse: &str = parsable.as_deref().unwrap_or(&config.when); - let ret: (i8, i8) = (cap[1].parse().unwrap(), cap[2].parse().unwrap()); - if (0..24).contains(&ret.0) && (0..60).contains(&ret.1) { - return Some(ret); - } - None + let schedule = cron::Schedule::from_str(to_parse); + if schedule.is_err() { + return Err(Error::InvalidCronConfiguration(schedule.err().unwrap())); + }; + + Ok(Backup { + name: String::from(name), + what: service, + r#where: remote, + remote_path: PathBuf::from(config.remote_path.clone()), + when: config.when.clone(), + compress: config.compress, + schedule: schedule.unwrap(), + keep_last: config.keep_last, + }) } fn log_result( @@ -97,198 +111,13 @@ impl Backup { "[{}] Error during upload{} of {}: {}. Error: {}", name, if compress { " or compression" } else { "" }, - if file.is_dir() { "folder" } else { "file" }, file.display(), + remote_name, result.err().unwrap() ); } } - fn parse_daily(input: &str) -> Result { - // Daily 12:30 - let daily = "daily"; - if input.contains(daily) { - let input = input.replace(daily, ""); - - let hm = Self::get_hours_and_minutes(&input); - if hm.is_none() { - return Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find hours:minutes", - ))); - } - let hm = hm.unwrap(); - let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); - let input = input.trim(); - if !input.is_empty() { - return Err(Error::InvalidWhenConfiguration(format!( - "Expected to consume all the when string, unable to parse \ - remaining part: {}", - input - ))); - } - - // sec min hour day of month month day of week - return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, "*", "*", "*")); - } - Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find daily identifier", - ))) - } - - fn parse_weekly(input: &str) -> Result { - // Monday 15:40 or Weekly Monday 15:40 - let weekdays = [ - (Weekday::Mon, "Monday"), - (Weekday::Tue, "Tuesday"), - (Weekday::Wed, "Wednesday"), - (Weekday::Thu, "Thursday"), - (Weekday::Fri, "Friday"), - (Weekday::Sat, "Saturday"), - (Weekday::Sun, "Sunday"), - ]; - - let weekdays = weekdays.iter().map(|d| { - ( - d.0.to_string().to_lowercase(), - String::from(d.1).to_lowercase(), - ) - }); - for day in weekdays { - let short = input.contains(&day.0); - let long = input.contains(&day.1); - if short || long { - let input = input.replace(if long { &day.1 } else { &day.0 }, ""); - let hm = Backup::get_hours_and_minutes(&input); - if hm.is_none() { - return Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find hours:minutes", - ))); - } - let hm = hm.unwrap(); - let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); - let input = input.trim(); - if !["", "weekly"].contains(&input) { - return Err(Error::InvalidWhenConfiguration(format!( - "Expected to consume all the when string, unable to parse \ - remaining part: {}", - input - ))); - } - let day = Weekday::from_str(&day.0).unwrap().number_from_monday(); - - // sec min hour day of month month day of week - return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, "*", "*", day)); - } - } - Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find any weekday identifier", - ))) - } - - fn parse_monthly(input: &str) -> Result { - // Monthly 1 12:40 - let monthly = "monthly"; - if input.contains(monthly) { - let input = input.replace(monthly, ""); - let hm = Backup::get_hours_and_minutes(&input); - if hm.is_none() { - return Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find hours:minutes", - ))); - } - let hm = hm.unwrap(); - let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); - let input = input.trim(); - // Input should now contain only the "day of the month" - - let day: i8 = match input.parse() { - Ok(day) => day, - Err(error) => { - return Err(Error::InvalidWhenConfiguration(format!( - "Unable to correctly parse the string for the day of the month. \ - Given input: {}. Error: {}", - input, error - ))) - } - }; - - let valid_days = 1..32; - if !valid_days.contains(&day) { - return Err(Error::InvalidWhenConfiguration(String::from( - "Invalid day of the month specified, out of range [1,31]", - ))); - } - - // sec min hour day of month month day of week - return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, day, "*", "*")); - } - Err(Error::InvalidWhenConfiguration(String::from( - "Unable to find monthly identifier", - ))) - } - - fn parse_when(when: &str) -> Result { - // sec min hour day of month month day of week - // * * * * * * - let input = when.to_lowercase(); - let daily = Backup::parse_daily(&input); - if daily.is_ok() { - return daily; - } - - let monthly = Backup::parse_monthly(&input); - if monthly.is_ok() { - return monthly; - } - - let weekly = Backup::parse_weekly(&input); - if weekly.is_ok() { - return weekly; - } - - Err(Error::InvalidWhenConfiguration(format!( - "Unable to parse for:\n\ - Daily: {}\n - Weekly: {}\n - Monthly: {}", - daily.unwrap_err(), - weekly.unwrap_err(), - monthly.unwrap_err() - ))) - } - pub async fn new( - name: &str, - remote: Box, - service: Box, - config: &BackupConfig, - ) -> Result { - let when_to_schedule = Backup::parse_when(&config.when); - let to_parse: &str; - let parsable: String; - if let Ok(value) = when_to_schedule { - parsable = value; - to_parse = &parsable; - } else { - to_parse = &config.when; - }; - - let schedule = cron::Schedule::from_str(to_parse); - if schedule.is_err() { - return Err(Error::InvalidCronConfiguration(schedule.err().unwrap())); - }; - - Ok(Backup { - name: String::from(name), - what: service, - r#where: remote, - remote_path: PathBuf::from(config.remote_path.clone()), - when: config.when.clone(), - compress: config.compress, - schedule: schedule.unwrap(), - keep_last: config.keep_last, - }) - } - pub async fn schedule( self: Arc, scheduler: &mut JobScheduler, @@ -490,7 +319,7 @@ mod tests { use croner::Cron; fn validate_cron_expression(when: &str) { - let result = Backup::parse_when(when); + let result = when::parse_when(when); assert!( result.is_ok(), "Failed to parse when string '{}': {}", @@ -516,11 +345,11 @@ mod tests { validate_cron_expression("DAILY 11:11"); // Invalid cases - assert!(Backup::parse_when("dayly 00:00").is_err()); - assert!(Backup::parse_when("daily 55:00").is_err()); - assert!(Backup::parse_when("daily 00:61").is_err()); - assert!(Backup::parse_when("daily 00:60").is_err()); - assert!(Backup::parse_when("daily 24:01").is_err()); + assert!(when::parse_when("dayly 00:00").is_err()); + assert!(when::parse_when("daily 55:00").is_err()); + assert!(when::parse_when("daily 00:61").is_err()); + assert!(when::parse_when("daily 00:60").is_err()); + assert!(when::parse_when("daily 24:01").is_err()); } #[test] @@ -544,13 +373,13 @@ mod tests { validate_cron_expression(" sunday 12:30"); // Invalid cases - assert!(Backup::parse_when("watly monzay 00:00").is_err()); - assert!(Backup::parse_when("monzay 00:00").is_err()); - assert!(Backup::parse_when("Moonday 00:00").is_err()); - assert!(Backup::parse_when("Sundays 1:00").is_err()); - assert!(Backup::parse_when("Today 00:00").is_err()); - assert!(Backup::parse_when("Tomorrow 00:00").is_err()); - assert!(Backup::parse_when("Toyota -1:00").is_err()); + assert!(when::parse_when("watly monzay 00:00").is_err()); + assert!(when::parse_when("monzay 00:00").is_err()); + assert!(when::parse_when("Moonday 00:00").is_err()); + assert!(when::parse_when("Sundays 1:00").is_err()); + assert!(when::parse_when("Today 00:00").is_err()); + assert!(when::parse_when("Tomorrow 00:00").is_err()); + assert!(when::parse_when("Toyota -1:00").is_err()); } #[test] @@ -560,9 +389,9 @@ mod tests { validate_cron_expression("Monthly 31 02:30"); // Invalid cases - assert!(Backup::parse_when("Monthly 00:00").is_err()); - assert!(Backup::parse_when("Monthtly -1 00:00").is_err()); - assert!(Backup::parse_when("Monthtly 0 00:00").is_err()); - assert!(Backup::parse_when("Monthtly 32 00:00").is_err()); + assert!(when::parse_when("Monthly 00:00").is_err()); + assert!(when::parse_when("Monthtly -1 00:00").is_err()); + assert!(when::parse_when("Monthtly 0 00:00").is_err()); + assert!(when::parse_when("Monthtly 32 00:00").is_err()); } } diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index 5133a23..0060e86 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use std::string::String; use bacup::backup::Backup; -use bacup::config::Config; +use bacup::config::{BackupConfig, Config, ZfsConfig}; use bacup::remotes::aws::AwsBucket; use bacup::remotes::git::Git; @@ -34,6 +34,7 @@ use bacup::services::folders::Folder; use bacup::services::postgresql::PostgreSql; use bacup::services::service::Service; +use bacup::services::zfs::required_keep_last; use bacup::services::zfs::Zfs; use log::*; use structopt::StructOpt; @@ -51,6 +52,37 @@ struct Opt { verbose: usize, } +/// Returns a message if a zfs backup's keep_last is too small to keep a +/// restorable snapshot chain. +fn check_zfs_keep_last( + backup_name: &str, + backup: &BackupConfig, + zfs: &Option>, +) -> Result<(), String> { + let Some(service_name) = backup.what.strip_prefix("zfs.") else { + return Ok(()); + }; + let Some(configs) = zfs else { + return Ok(()); + }; + let Some(zfs_config) = configs.get(service_name) else { + return Ok(()); + }; + let Some(full_when) = &zfs_config.full_when else { + return Ok(()); + }; + let required = required_keep_last(&backup.when, full_when) + .map_err(|error| format!("Backup {backup_name}: {error}"))?; + if let Some(keep_last) = backup.keep_last { + if keep_last < required { + return Err(format!( + "Backup {backup_name}: keep_last {keep_last} is too small for zfs service {service_name} (full_when '{full_when}'): the remote must keep at least {required} latest snapshots to stay restorable. Raise keep_last or remove it." + )); + } + } + Ok(()) +} + #[tokio::main] async fn main() -> Result<(), i32> { let opt = Opt::from_args(); @@ -136,6 +168,16 @@ async fn main() -> Result<(), i32> { None => warn!("No Git remotes configured."), } + // ZFS full + incremental backups are only restorable if the remote keeps + // a complete chain (a full and its incrementals), so refuse to start + // when keep_last would prune away older fulls. + for (backup_name, backup_config) in &config.backup { + if let Err(message) = check_zfs_keep_last(backup_name, backup_config, &config.zfs) { + error!("{message}"); + return Err(-1); + } + } + let mut services: HashMap> = HashMap::new(); match config.folders { Some(folders) => { @@ -185,7 +227,7 @@ async fn main() -> Result<(), i32> { ); } } - None => warn!("No Docker to backup."), + None => warn!("No Zfs to backup."), } let mut backup: HashMap> = HashMap::new(); @@ -260,3 +302,72 @@ async fn main() -> Result<(), i32> { tokio::time::sleep(Duration::from_millis(50)).await; } } + +#[cfg(test)] +mod tests { + use super::*; + + fn backup(what: &str, when: &str, keep_last: Option) -> BackupConfig { + BackupConfig { + what: what.to_string(), + r#where: "localhost.test".to_string(), + when: when.to_string(), + remote_path: "/backups".to_string(), + compress: false, + keep_last, + } + } + + fn zfs(full_when: Option<&str>) -> Option> { + let mut configs = HashMap::new(); + configs.insert( + "pool".to_string(), + ZfsConfig { + dataset: "tank".to_string(), + snapshot_name: "snap".to_string(), + full_when: full_when.map(String::from), + }, + ); + Some(configs) + } + + #[test] + fn keep_last_too_small_refuses_to_start() { + let b = backup("zfs.pool", "daily 01:00", Some(7)); + let err = check_zfs_keep_last("db", &b, &zfs(Some("monthly 1 01:00"))).unwrap_err(); + assert!(err.contains("keep_last 7 is too small"), "{}", err); + assert!(err.contains("at least 31"), "{}", err); + } + + #[test] + fn keep_last_sufficient_starts() { + for keep_last in [31u32, 32] { + let b = backup("zfs.pool", "daily 01:00", Some(keep_last)); + assert!(check_zfs_keep_last("db", &b, &zfs(Some("monthly 1 01:00"))).is_ok()); + } + } + + #[test] + fn no_keep_last_never_refuses() { + let b = backup("zfs.pool", "daily 01:00", None); + assert!(check_zfs_keep_last("db", &b, &zfs(Some("monthly 1 01:00"))).is_ok()); + } + + #[test] + fn no_full_when_never_refuses() { + let b = backup("zfs.pool", "daily 01:00", Some(1)); + assert!(check_zfs_keep_last("db", &b, &zfs(None)).is_ok()); + } + + #[test] + fn non_zfs_service_is_ignored() { + let b = backup("folders.home", "daily 01:00", Some(1)); + assert!(check_zfs_keep_last("home", &b, &zfs(Some("monthly 1 01:00"))).is_ok()); + } + + #[test] + fn invalid_full_when_refuses() { + let b = backup("zfs.pool", "daily 01:00", Some(100)); + assert!(check_zfs_keep_last("db", &b, &zfs(Some("nonsense"))).is_err()); + } +} diff --git a/src/config.rs b/src/config.rs index bbb4940..3f9122f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -70,11 +70,12 @@ pub struct DockerConfig { pub struct ZfsConfig { pub dataset: String, pub snapshot_name: String, - /// Optional cron expression (e.g. `"0 1 1 * *"`) controlling how often a - /// full backup is taken. When set, only the runs where it is due take a - /// full backup and the runs in between are incrementals against the - /// latest existing snapshot. When absent, every run is a full backup - /// (previous behavior). + /// Optional expression controlling how often a full backup is taken. It + /// accepts the same format as the `when` field (e.g. `"monthly 1 01:00"`) + /// or a raw cron expression (e.g. `"0 1 1 * *"`). When set, only the runs + /// where it is due take a full backup and the runs in between are + /// incrementals against the latest existing snapshot. When absent, every + /// run is a full backup (previous behavior). #[serde(default)] pub full_when: Option, } diff --git a/src/lib.rs b/src/lib.rs index b83227b..4c84047 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,3 +17,4 @@ pub mod config; pub mod disks; pub mod remotes; pub mod services; +pub mod when; diff --git a/src/services/zfs.rs b/src/services/zfs.rs index a5bfc32..354f162 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::process::Stdio; use std::str::FromStr; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use croner::Cron; use log::{debug, info, warn}; use tokio::fs; @@ -11,6 +11,7 @@ use tokio::process::Command; use crate::config::ZfsConfig; use crate::services::service::{Dump, Service}; +use crate::when; const TS_FORMAT: &str = "%Y%m%d-%H%M%S"; const FULL_MARK: &str = "full"; @@ -83,9 +84,17 @@ pub(crate) fn dump_file_name(name: &str, kind: SnapshotKind, ts: &str) -> String } /// Arguments for the `zfs send` command to dump the given snapshot. -/// A full snapshot is sent with `send -R -v`, an incremental one with -/// `send -R -v -i ` where `` is the name of the snapshot it is -/// based on. +/// A full snapshot is sent with `send -R -v -c -L`, an incremental one with +/// `send -R -v -c -L -i ` where `` is the name of the snapshot it +/// is based on. +/// +/// `-c` keeps blocks compressed in the stream when they are compressed on +/// disk (OpenZFS >= 2.1.1), and `-L` keeps the on-disk block sizes so that +/// `-c` is not defeated by the `large_blocks` feature (without `-L`, data is +/// decompressed before sending to be split into smaller blocks). For datasets +/// without compression the two flags are a no-op. Dump files of compressed +/// datasets are therefore already compressed and do not need the gzip layer +/// at upload time (`compress = false`). pub(crate) fn send_args(kind: SnapshotKind, base: Option<&str>, new_name: &str) -> Vec { match (kind, base) { (SnapshotKind::Incremental, Some(base)) => { @@ -93,13 +102,22 @@ pub(crate) fn send_args(kind: SnapshotKind, base: Option<&str>, new_name: &str) "send".into(), "-R".into(), "-v".into(), + "-c".into(), + "-L".into(), "-i".into(), base.into(), new_name.into(), ] } // A full snapshot, or (defensively) an incremental without a base. - _ => vec!["send".into(), "-R".into(), "-v".into(), new_name.into()], + _ => vec![ + "send".into(), + "-R".into(), + "-v".into(), + "-c".into(), + "-L".into(), + new_name.into(), + ], } } @@ -219,6 +237,97 @@ pub struct Zfs { full_when: Option, } +/// Parses a schedule expression, accepting the human friendly formats of +/// the `when` field (e.g. "monthly 1 01:00") and falling back to a raw +/// cron expression. +pub fn parse_schedule(expr: &str) -> Result { + let parsable = when::parse_when(expr).unwrap_or_else(|_| expr.to_string()); + Cron::from_str(&parsable).map_err(|e| { + Error::String(format!( + "invalid schedule expression {expr}: not a valid when format nor a cron expression ({e})" + )) + }) +} + +/// Returns the minimum `keep_last` a zfs backup needs so the remote always +/// retains a restorable chain: the latest full snapshot plus every +/// incremental taken since it. +/// +/// An incremental stream (`zfs send -i base`) cannot be restored without its +/// base snapshot, so `keep_last` must survive the longest stretch of +/// incremental runs between two consecutive fulls. `backup_when` is the +/// backup's `when` field, `full_when` the zfs service's `full_when`; both +/// accept the human when formats or raw cron expressions. +/// +/// Returns an error if the expressions cannot be parsed or if `full_when` +/// fires less than twice in the four year simulation window, in which case +/// the required `keep_last` cannot be bounded. +pub fn required_keep_last(backup_when: &str, full_when: &str) -> Result { + let backup = parse_schedule(backup_when)?; + let full = parse_schedule(full_when)?; + + // A fixed four year window (spanning a leap year) is enough to observe + // every gap pattern the two periodic schedules can produce. The fixed + // start keeps the result deterministic. + let from = NaiveDate::from_ymd_opt(2026, 1, 1) + .and_then(|d| d.and_hms_opt(0, 0, 0)) + .expect("fixed simulation window start is a valid date"); + let until = from + chrono::Duration::days(4 * 365 + 1); + + let backup_runs = occurrences(&backup, from, until)?; + let full_runs = occurrences(&full, from, until)?; + if full_runs.len() < 2 { + return Err(Error::String(format!( + "full_when '{full_when}' fires less than twice in four years, \ + the required keep_last cannot be bounded" + ))); + } + + // Max number of backup runs strictly between two consecutive fulls. + let mut max_inc = 0usize; + for pair in full_runs.windows(2) { + let (lo, hi) = (pair[0], pair[1]); + // Strictly between: a backup at lo or hi is that full's own run. + let before = backup_runs.partition_point(|t| *t <= lo); + let up_to = backup_runs.partition_point(|t| *t < hi); + max_inc = max_inc.max(up_to - before); + } + + // One for the full snapshot itself, one per incremental in the longest + // gap. + Ok(max_inc as u32 + 1) +} + +/// All occurrences of `cron` in `[from, until]`. +fn occurrences( + cron: &Cron, + from: NaiveDateTime, + until: NaiveDateTime, +) -> Result, Error> { + let mut out = Vec::new(); + if cron + .is_time_matching(&from) + .map_err(|e| Error::String(e.to_string()))? + { + out.push(from); + } + let mut cursor = from; + while out.len() < 100_000 { + let next = cron.find_next_occurrence(&cursor, false).map_err(|e| { + Error::String(format!( + "schedule '{}' stopped producing occurrences: {e}", + cron.as_str() + )) + })?; + if next > until { + break; + } + out.push(next); + cursor = next; + } + Ok(out) +} + impl Zfs { pub async fn new(config: &ZfsConfig, name: &str) -> Result { let dataset = &config.dataset; @@ -229,12 +338,7 @@ impl Zfs { let cmd = zfs.to_str().unwrap().to_string(); let full_when = match &config.full_when { - Some(expr) => { - let cron = Cron::from_str(expr).map_err(|e| { - Error::String(format!("invalid full_when expression {expr}: {e}")) - })?; - Some(cron) - } + Some(expr) => Some(parse_schedule(expr)?), None => None, }; @@ -273,6 +377,29 @@ impl Zfs { ))); } + // The dump is produced with `zfs send -c -L` (see `send_args`): blocks + // compressed on disk stay compressed in the stream. Gzip'ing such a + // dump at upload time is wasted CPU with no size gain, so flag it when + // the dataset is compressed. + let compress_output = Command::new(&cmd) + .args(["get", "-p", "-o", "value", "compression", dataset]) + .output() + .await?; + if !compress_output.status.success() { + return Err(Error::String(format!( + "failed to query compression on {dataset}: {}", + String::from_utf8_lossy(&compress_output.stderr) + ))); + } + let compression = String::from_utf8_lossy(&compress_output.stdout) + .trim() + .to_string(); + if compression != "off" { + info!( + "dataset {dataset} is compressed ({compression}): the zfs dump is already compressed, consider compress = false for its backups" + ); + } + debug!("new zfs service on {dataset} (snapshot base {snapshot_base})"); Ok(Zfs { name: name.to_string(), @@ -636,7 +763,7 @@ mod tests { fn send_args_full_and_inc() { assert_eq!( send_args(SnapshotKind::Full, None, "tank@snap-full-1"), - vec!["send", "-R", "-v", "tank@snap-full-1"] + vec!["send", "-R", "-v", "-c", "-L", "tank@snap-full-1"] ); assert_eq!( send_args( @@ -648,6 +775,8 @@ mod tests { "send", "-R", "-v", + "-c", + "-L", "-i", "tank@snap-full-1", "tank@snap-inc-2" @@ -788,4 +917,49 @@ mod tests { ] ); } + + #[test] + fn parse_schedule_accepts_when_and_cron_formats() { + assert!(parse_schedule("monthly 1 01:00").is_ok()); + assert!(parse_schedule("0 1 1 * *").is_ok()); + assert!(parse_schedule("nonsense").is_err()); + } + + #[test] + fn required_keep_last_monthly_full_daily_backup() { + // 31 day month: 30 incrementals between the fulls, plus the full. + assert_eq!( + required_keep_last("daily 01:00", "monthly 1 01:00").unwrap(), + 31 + ); + // Same schedules expressed as raw cron. + assert_eq!(required_keep_last("0 1 * * *", "0 1 1 * *").unwrap(), 31); + } + + #[test] + fn required_keep_last_weekly_full_daily_backup() { + assert_eq!( + required_keep_last("daily 01:00", "weekly monday 01:00").unwrap(), + 7 + ); + } + + #[test] + fn required_keep_last_daily_full() { + assert_eq!(required_keep_last("daily 01:00", "daily 01:00").unwrap(), 1); + } + + #[test] + fn required_keep_last_invalid_expressions() { + assert!(required_keep_last("nonsense", "monthly 1 01:00").is_err()); + assert!(required_keep_last("daily 01:00", "nonsense").is_err()); + } + + #[test] + fn required_keep_last_unbounded_full_schedule() { + // Never fires (February 31). + assert!(required_keep_last("daily 01:00", "0 0 31 2 *").is_err()); + // Fires once in the window (February 29, leap year only). + assert!(required_keep_last("daily 01:00", "0 0 29 2 *").is_err()); + } } diff --git a/src/when.rs b/src/when.rs new file mode 100644 index 0000000..667905f --- /dev/null +++ b/src/when.rs @@ -0,0 +1,261 @@ +// Copyright 2022-2026 Paolo Galeone +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::str::FromStr; + +use chrono::Weekday; +use regex::Regex; + +fn get_hours_and_minutes(when: &str) -> Option<(i8, i8)> { + let re = Regex::new(r"(\d{2}):(\d{2})").unwrap(); + let caps = re.captures(when)?; + let hours: i8 = caps.get(1)?.as_str().parse().ok()?; + let minutes: i8 = caps.get(2)?.as_str().parse().ok()?; + if hours < 24 && minutes < 60 { + Some((hours, minutes)) + } else { + None + } +} + +pub fn parse_daily(input: &str) -> Result { + // Daily 12:40 + let daily = "daily"; + if input.contains(daily) { + let input = input.replace(daily, "").trim().to_string(); + let hm = get_hours_and_minutes(&input); + if hm.is_none() { + return Err(String::from("Unable to find hours:minutes")); + } + let hm = hm.unwrap(); + let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); + let input = input.trim(); + if !input.is_empty() { + return Err(format!( + "Expected to consume all the when string, unable to parse remaining part: {}", + input + )); + } + // sec min hour day of month month day of week + return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, "*", "*", "*")); + } + Err(String::from("Invalid format for when: daily HH:MM")) +} + +pub fn parse_weekly(input: &str) -> Result { + // Monday 15:40 or Weekly Monday 15:40 + let weekdays = [ + (Weekday::Mon, "Monday"), + (Weekday::Tue, "Tuesday"), + (Weekday::Wed, "Wednesday"), + (Weekday::Thu, "Thursday"), + (Weekday::Fri, "Friday"), + (Weekday::Sat, "Saturday"), + (Weekday::Sun, "Sunday"), + ]; + + let weekdays = weekdays.iter().map(|d| { + ( + d.0.to_string().to_lowercase(), + String::from(d.1).to_lowercase(), + ) + }); + for day in weekdays { + let short = input.contains(&day.0); + let long = input.contains(&day.1); + if short || long { + let input = input.replace(if long { &day.1 } else { &day.0 }, ""); + let hm = get_hours_and_minutes(&input); + if hm.is_none() { + return Err(String::from("Unable to find hours:minutes")); + } + let hm = hm.unwrap(); + let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); + let input = input.trim(); + if !["", "weekly"].contains(&input) { + return Err(format!( + "Expected to consume all the when string, unable to parse \ + remaining part: {}", + input + )); + } + // Chrono's `number_from_sunday` is ISO-8601 (Mon=2 .. Sat=7, Sun=1); + // cron day-of-week is Sun=0, Mon=1, ..., Sat=6, so subtract one. + let day = Weekday::from_str(&day.0).unwrap().number_from_sunday() - 1; + + // sec min hour day of month month day of week + return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, "*", "*", day)); + } + } + Err(String::from("Unable to find any weekday identifier")) +} + +pub fn parse_monthly(input: &str) -> Result { + // Monthly 1 12:40 + let monthly = "monthly"; + if input.contains(monthly) { + let input = input.replace(monthly, ""); + let hm = get_hours_and_minutes(&input); + if hm.is_none() { + return Err(String::from("Unable to find hours:minutes")); + } + let hm = hm.unwrap(); + let input = input.replace(&format!("{:02}:{:02}", hm.0, hm.1), ""); + let input = input.trim(); + // Input should now contain only the "day of the month" + + let day: i8 = match input.parse() { + Ok(day) => day, + Err(error) => { + return Err(format!( + "Unable to correctly parse the string for the day of the month. \ + Given input: {}. Error: {}", + input, error + )) + } + }; + + let valid_days = 1..32; + if !valid_days.contains(&day) { + return Err(String::from( + "Invalid day of the month specified, out of range [1,31]", + )); + } + + // sec min hour day of month month day of week + return Ok(format!("{} {} {} {} {} {}", 0, hm.1, hm.0, day, "*", "*")); + } + Err(String::from("Unable to find monthly identifier")) +} + +/// Parse a human friendly `when` expression into a cron expression +/// (sec min hour dom month dow), accepted by both the `cron` and `croner` +/// crates. +/// +/// Accepted formats: +/// - `daily HH:MM` (e.g. `daily 03:00`) +/// - `monthly D HH:MM` (e.g. `monthly 1 01:00`) +/// - `weekly HH:MM` or ` HH:MM` (e.g. `weekly monday 12:00`, `sun 12:00`) +pub fn parse_when(when: &str) -> Result { + // sec min hour day of month month day of week + // * * * * * * + let input = when.to_lowercase(); + let daily = parse_daily(&input); + if daily.is_ok() { + return daily; + } + + let monthly = parse_monthly(&input); + if monthly.is_ok() { + return monthly; + } + + let weekly = parse_weekly(&input); + if weekly.is_ok() { + return weekly; + } + + Err(format!( + "Unable to parse for:\n\ + Daily: {}\n + Weekly: {}\n + Monthly: {}", + daily.unwrap_err(), + weekly.unwrap_err(), + monthly.unwrap_err() + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use croner::Cron; + + fn validate_cron_expression(when: &str) { + let cron_str = + parse_when(when).unwrap_or_else(|e| panic!("Unable to parse when: {}: {}", when, e)); + Cron::from_str(&cron_str).unwrap_or_else(|e| { + panic!( + "Invalid cron expression generated: {} -> {}: {}", + when, cron_str, e + ) + }); + } + + #[test] + fn test_parse_when_daily() { + validate_cron_expression("daily 00:00"); + validate_cron_expression("Daily 00:00"); + validate_cron_expression("daily 12:30"); + validate_cron_expression("DAILY 12:30"); + + assert!(parse_when("dayly 00:00").is_err()); + assert!(parse_when("daily 55:00").is_err()); + assert!(parse_when("daily 00:61").is_err()); + assert!(parse_when("daily 00:60").is_err()); + assert!(parse_when("daily 24:01").is_err()); + } + + #[test] + fn test_parse_when_weekly() { + for day in ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] { + let when = format!("{day} 12:30"); + validate_cron_expression(&when); + } + + validate_cron_expression(" SUN 12:30"); + validate_cron_expression(" sunday 12:30"); + + assert!(parse_when("Sundays 1:00").is_err()); + assert!(parse_when("Today 00:00").is_err()); + assert!(parse_when("Tomorrow 00:00").is_err()); + assert!(parse_when("Toyota -1:00").is_err()); + } + + #[test] + fn test_parse_when_weekly_day_mapping() { + // cron day-of-week: 0=Sunday, 1=Monday, ..., 6=Saturday + for (day, dow) in [ + ("mon", "1"), + ("tue", "2"), + ("wed", "3"), + ("thu", "4"), + ("fri", "5"), + ("sat", "6"), + ("sun", "0"), + ] { + let cron = parse_when(&format!("{day} 12:30")).unwrap(); + let fields: Vec<&str> = cron.split_whitespace().collect(); + assert_eq!(fields[5], dow, "wrong day-of-week for {day}: {cron}"); + } + } + + #[test] + fn test_parse_when_monthly() { + validate_cron_expression("Monthly 1 02:30"); + validate_cron_expression("Monthly 31 02:30"); + + assert!(parse_when("Monthly 0 01:00").is_err()); + assert!(parse_when("Monthly 32 01:00").is_err()); + assert!(parse_when("Monthly 01:00").is_err()); + } + + #[test] + fn test_croner_accepts_human_format() { + // This is what zfs full_when relies on: the human format must + // produce an expression croner can parse. + validate_cron_expression("monthly 1 01:00"); + validate_cron_expression("weekly monday 12:00"); + } +} From 1e8041367ea5ac3b780e704c22ed9033fbaa844f Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 15:13:26 +0200 Subject: [PATCH 34/51] Restore the license header in zfs.rs --- src/services/zfs.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 354f162..0ae87ec 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -1,3 +1,17 @@ +// Copyright 2022-2026 Paolo Galeone +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use std::fmt; use std::path::PathBuf; use std::process::Stdio; From 69b3160be9805255d59bfdaf13d98857f5d5eeeb Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 15:17:58 +0200 Subject: [PATCH 35/51] Document how to install a branch locally (cargo install --path .) --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 92836de..280215e 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,21 @@ sudo systemctl enable --now bacup@$USER.service **Note**: the working directory is important if you plan to back-up big files. The files are created in that directory before being uploaded, so set it to a location where you have the write right and enough space. +## Development + +To test an unreleased branch, build and install it from your local checkout — it replaces the crates.io binary in `~/.cargo/bin`: + +``` +git clone https://github.com/galeone/bacup +cd bacup +git checkout +cargo install --path . +``` + +Useful before wiring the new version into the systemd service, or to quickly verify a fix on a real backup. When a release you like is out, `cargo install bacup` puts the released binary back. + +Alternatively, `cargo build --release` leaves the binary at `target/release/bacup` without touching the installed one — handy for a one-off run (it still reads `$HOME/.bacup/config.toml`). + ## Remote configuration Configuring the remotes is straightforward. Every remote have a different way of getting the access code, here we try to share some useful reference. From 3fec8777881eb202bc575aeb43633b427818701e Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 16:06:16 +0200 Subject: [PATCH 36/51] Fix bug-hunt findings: pg_dump status, ssh quoting, aws pagination, git worktree/pull/push fixes, second-precision archive names, typos See BUGS.md for the full audit; items 1.1, 1.5, 1.7, 3.1, 3.2, 3.3, 3.5, 3.9 and 3.10 are fixed, the rest remain open. --- src/bin/bacup.rs | 5 --- src/remotes/aws.rs | 48 ++++++++++++++++--------- src/remotes/git.rs | 73 ++++++++++++++++++++++++++++++++------ src/remotes/localhost.rs | 4 +-- src/remotes/remote.rs | 4 +-- src/remotes/ssh.rs | 6 ++-- src/services/postgresql.rs | 11 ++++-- 7 files changed, 110 insertions(+), 41 deletions(-) diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index 0060e86..e9130ea 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -294,11 +294,6 @@ async fn main() -> Result<(), i32> { } use tokio::time::Duration; loop { - /*if let Err(e) = scheduler.tick() { - error!("Scheduler tick error: {:?}", e); - return Err(-1); - } - */ tokio::time::sleep(Duration::from_millis(50)).await; } } diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 45ff4ec..e9a4dbd 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -64,23 +64,39 @@ impl From for AwsError { impl Bucket { pub async fn list(&self, prefix: &str) -> Result, AwsError> { - let response = self - .client - .list_objects_v2() - .bucket(&self.bucket_name) - .prefix(prefix.trim_start_matches('/')) - .send() - .await; - if response.is_err() { - return Err(AwsError::RemoteError(Box::new( - response.err().unwrap().into(), - ))); - } - let response = response.unwrap(); + // list_objects_v2 returns at most 1000 keys per page; follow the + // continuation token until the prefix is fully enumerated, otherwise + // keep_last pruning would only see the first 1000 objects. + let prefix = prefix.trim_start_matches('/'); + let mut continuation_token: Option = None; let mut ret: Vec = vec![]; - for res in response.contents.iter() { - for object in res { - ret.push(object.key.as_ref().unwrap().to_owned()); + loop { + let mut lister = self + .client + .list_objects_v2() + .bucket(&self.bucket_name) + .prefix(prefix); + if let Some(token) = &continuation_token { + lister = lister.continuation_token(token); + } + let response = lister.send().await; + if response.is_err() { + return Err(AwsError::RemoteError(Box::new( + response.err().unwrap().into(), + ))); + } + let response = response.unwrap(); + for res in response.contents.iter() { + for object in res { + ret.push(object.key.as_ref().unwrap().to_owned()); + } + } + match ( + response.is_truncated().unwrap_or(false), + response.next_continuation_token, + ) { + (true, Some(token)) => continuation_token = Some(token), + _ => break, } } Ok(ret) diff --git a/src/remotes/git.rs b/src/remotes/git.rs index 447bd16..137d001 100644 --- a/src/remotes/git.rs +++ b/src/remotes/git.rs @@ -30,7 +30,7 @@ use async_trait::async_trait; use scopeguard::defer; -use log::info; +use log::{info, warn}; #[derive(Debug)] pub enum Error { @@ -118,7 +118,14 @@ impl Git { } fn clone_repository(&self) -> Result { - let dest = PathBuf::from(&self.config.repository.split('/').next_back().unwrap()); + let dest = PathBuf::from(format!( + "{}__{}", + self.config.host, + self.config + .repository + .trim_start_matches('/') + .replace('/', "-") + )); if dest.exists() { let git_repo = dest.join(".git"); if git_repo.exists() && git_repo.is_dir() { @@ -141,7 +148,14 @@ impl Git { )))); } - let dest = PathBuf::from(&self.config.repository.split('/').next_back().unwrap()); + let dest = PathBuf::from(format!( + "{}__{}", + self.config.host, + self.config + .repository + .trim_start_matches('/') + .replace('/', "-") + )); if !dest.exists() { return Err(Error::DoesNotExist(dest)); } @@ -192,10 +206,22 @@ impl remote::Remote for Git { .args(["switch", "-c", &self.config.branch]) .status()?; - // git pull origin branch (ignore failures) - Command::new(&self.git_cmd) + // git pull origin branch: a failure leaves the worktree with conflict + // markers or diverged state that must never be committed as a backup + let status = Command::new(&self.git_cmd) .args(["pull", "origin", &self.config.branch]) .status()?; + if !status.success() { + warn!( + "Pull failed in {}, discarding local state and retrying", + dest.display() + ); + let _ = Command::new(&self.git_cmd) + .args(["reset", "--hard"]) + .status(); + let _ = Command::new(&self.git_cmd).args(["clean", "-fd"]).status(); + fs::copy(path, dest.join(path.file_name().unwrap())).await?; + } // git add . -A let status = Command::new(&self.git_cmd) @@ -223,7 +249,7 @@ impl remote::Remote for Git { .status()?; if !status.success() { return Err(remote::Error::LocalError(io::Error::other(format!( - "Unable to execute git add . -A into {}", + "Unable to execute git push into {}", dest.display() )))); } @@ -247,7 +273,7 @@ impl remote::Remote for Git { defer! { #[allow(unused_must_use)] { - fs::remove_file(&remote_path); + fs::remove_file(compressed_file.path()); } } self.upload_file(compressed_file.path(), &remote_path) @@ -305,10 +331,37 @@ impl remote::Remote for Git { .args(["switch", "-c", &self.config.branch]) .status()?; - // git pull origin branch (ignore failures) - Command::new(&self.git_cmd) + // git pull origin branch: a failure leaves the worktree with conflict + // markers or diverged state that must never be committed as a backup + let status = Command::new(&self.git_cmd) .args(["pull", "origin", &self.config.branch]) .status()?; + if !status.success() { + warn!( + "Pull failed in {}, discarding local state and retrying", + dest.display() + ); + let _ = Command::new(&self.git_cmd) + .args(["reset", "--hard"]) + .status(); + let _ = Command::new(&self.git_cmd).args(["clean", "-fd"]).status(); + for path in paths { + let mut relative = path.clone(); + if relative + .components() + .any(|component| component.as_os_str() == ".git") + { + continue; + } + if relative.is_dir() { + if dest.join(relative.file_name().unwrap()).exists() == false { + fs::create_dir_all(dest.join(relative.file_name().unwrap())).await?; + } + } else { + fs::copy(&relative, dest.join(relative.file_name().unwrap())).await?; + } + } + } // git add . -A let status = Command::new(&self.git_cmd) @@ -336,7 +389,7 @@ impl remote::Remote for Git { .status()?; if !status.success() { return Err(remote::Error::LocalError(io::Error::other(format!( - "Unable to execute git add . -A into {}", + "Unable to execute git push into {}", dest.display() )))); } diff --git a/src/remotes/localhost.rs b/src/remotes/localhost.rs index 5a32b39..61f35ef 100644 --- a/src/remotes/localhost.rs +++ b/src/remotes/localhost.rs @@ -290,7 +290,7 @@ mod tests { let now: chrono::DateTime = chrono::Utc::now(); let dest = tmp_dir .path() - .join(format!("{}-Cargo.toml.gz", now.format("%Y-%m-%d-%H.%M"),)); + .join(format!("{}-Cargo.toml.gz", now.format("%Y-%m-%d-%H.%M.%S"),)); assert!(dest.exists()); } @@ -350,7 +350,7 @@ mod tests { let now: chrono::DateTime = chrono::Utc::now(); let dest = tmp_dir.path().join(format!( "{}-{}.tar.gz", - now.format("%Y-%m-%d-%H.%M"), + now.format("%Y-%m-%d-%H.%M.%S"), remote_filename )); diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index 46938c5..fc166c1 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -183,7 +183,7 @@ pub trait Remote: DynClone + Send + Sync { parent.join(format!( "{}-{}.tar.gz", - now.format("%Y-%m-%d-%H.%M"), + now.format("%Y-%m-%d-%H.%M.%S"), remote_path.file_name().unwrap().to_str().unwrap() )) } @@ -197,7 +197,7 @@ pub trait Remote: DynClone + Send + Sync { parent.join(format!( "{}-{}.gz", - now.format("%Y-%m-%d-%H.%M"), + now.format("%Y-%m-%d-%H.%M.%S"), remote_path.file_name().unwrap().to_str().unwrap() )) } diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index 114f922..d49f00c 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -133,7 +133,7 @@ impl Ssh { // // But anyway this is a success since the connection was succesfull. warn!( - "Connection to {}@{}:{} succeded, but received: {}", + "Connection to {}@{}:{} succeeded, but received: {}", config.username, config.host, config.port, stderr ); } else { @@ -192,7 +192,7 @@ impl remote::Remote for Ssh { .args( self.ssh_args .iter() - .chain(once(&format!("find {}/*", remote_path))), + .chain(once(&format!("find {}/*", shell_quote(remote_path)))), ) .stdin(Stdio::null()) .stdout(Stdio::piped()) @@ -221,7 +221,7 @@ impl remote::Remote for Ssh { .args( self.ssh_args .iter() - .chain(once(&format!("rm -r {}", remote_path))), + .chain(once(&format!("rm -r {}", shell_quote(remote_path)))), ) .stdin(Stdio::null()) .stdout(Stdio::null()) diff --git a/src/services/postgresql.rs b/src/services/postgresql.rs index 59bca11..f688ccd 100644 --- a/src/services/postgresql.rs +++ b/src/services/postgresql.rs @@ -112,7 +112,7 @@ impl PostgreSql { let stdout = std::str::from_utf8(&output.stdout).unwrap().trim(); if stdout == "0" { return Err(Error::RuntimeError(io::Error::other(format!( - "database {} does not exit or user {} not allowed to query the db", + "database {} does not exist or user {} not allowed to query the db", db_name, username )))); } @@ -120,7 +120,7 @@ impl PostgreSql { args.pop(); args.pop(); - // All the database dumps shuld be performend without aksing for password + // All the database dumps should be performed without asking for password args.push("--no-password"); let cmd = match which("pg_dump") { @@ -174,7 +174,12 @@ impl Service for PostgreSql { .status() .await { - Ok(_) => Ok(Dump { path: Some(dest) }), + Ok(status) if status.success() => Ok(Dump { path: Some(dest) }), + Ok(status) => Err(Error::RuntimeError(io::Error::other(format!( + "pg_dump exited with {} for database {}", + status, self.db_name + ))) + .into()), Err(error) => Err(Error::RuntimeError(error).into()), } } From 56157c63a1547d80b655244e058580a260cc93c4 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 16:12:15 +0200 Subject: [PATCH 37/51] Fix clippy lints in the git pull-failure re-copy block (unused mut, bool comparison) --- src/remotes/git.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/remotes/git.rs b/src/remotes/git.rs index 137d001..a334b44 100644 --- a/src/remotes/git.rs +++ b/src/remotes/git.rs @@ -346,7 +346,7 @@ impl remote::Remote for Git { .status(); let _ = Command::new(&self.git_cmd).args(["clean", "-fd"]).status(); for path in paths { - let mut relative = path.clone(); + let relative = path.clone(); if relative .components() .any(|component| component.as_os_str() == ".git") @@ -354,7 +354,7 @@ impl remote::Remote for Git { continue; } if relative.is_dir() { - if dest.join(relative.file_name().unwrap()).exists() == false { + if !dest.join(relative.file_name().unwrap()).exists() { fs::create_dir_all(dest.join(relative.file_name().unwrap())).await?; } } else { From 17c8d9014f4242544eda243ca205d58ad3abf01b Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 20:57:40 +0200 Subject: [PATCH 38/51] fiz zfs required permissions --- README.md | 2 +- src/services/zfs.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 280215e..a7da28a 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ When configuring the backups, the field **when** accepts configuration strings i [zfs] # the user needs zfs permissions on the datasets (bacup verifies them at startup): - # zfs allow $USER destroy,list,send,snapshot + # zfs allow $USER destroy,mount,send,snapshot # (permissions are inherited by child datasets) [zfs.root] snapshot_name = "root-fs" diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 0ae87ec..058228b 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -374,7 +374,7 @@ impl Zfs { ))); } let allow_text = String::from_utf8_lossy(&allow_output.stdout).to_string(); - let required = ["destroy", "list", "send", "snapshot"]; + let required = ["destroy", "mount", "send", "snapshot"]; let ok = allow_text.lines().any(|line| { let line = line.trim(); let is_user_line = line.starts_with(&format!("user {user}")) @@ -387,7 +387,7 @@ impl Zfs { return Err(Error::String(format!( "user \"{user}\" is not allowed to manage zfs snapshots on {dataset}. \ Run `zfs allow {user} {required_list} {dataset}` (permissions are inherited \ - by child datasets) and make sure `list` is granted so bacup can enumerate snapshots." + by child datasets)." ))); } From b98407040e22652da677f9113935079ad274a9af Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 21:02:41 +0200 Subject: [PATCH 39/51] Suppress the zfs get header when querying compression (-H) --- src/services/zfs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 058228b..11a644c 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -396,7 +396,7 @@ impl Zfs { // dump at upload time is wasted CPU with no size gain, so flag it when // the dataset is compressed. let compress_output = Command::new(&cmd) - .args(["get", "-p", "-o", "value", "compression", dataset]) + .args(["get", "-H", "-p", "-o", "value", "compression", dataset]) .output() .await?; if !compress_output.status.success() { From a63133187dd3d59d52c7340f0aa9737eee787f08 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Sun, 4 Oct 2026 21:10:45 +0200 Subject: [PATCH 40/51] Make the zfs compression nudge depend on the backup's compress flag --- src/bin/bacup.rs | 29 +++++++++++++++++++++++++---- src/services/zfs.rs | 14 +++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index e9130ea..b1167ee 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -217,14 +217,14 @@ async fn main() -> Result<(), i32> { None => warn!("No Docker to backup."), } + let mut zfs_compression: HashMap = HashMap::new(); match config.zfs { Some(zfs) => { for (service_name, instance_config) in zfs { let key = format!("zfs.{}", service_name); - services.insert( - key, - Box::new(Zfs::new(&instance_config, &service_name).await.unwrap()), - ); + let service = Zfs::new(&instance_config, &service_name).await.unwrap(); + zfs_compression.insert(service_name, service.compression().to_string()); + services.insert(key, Box::new(service)); } } None => warn!("No Zfs to backup."), @@ -265,6 +265,27 @@ async fn main() -> Result<(), i32> { .unwrap(), ), ); + // The dump is produced with `zfs send -c -L`: on a compressed dataset + // the stream is already compressed, so gzip at upload is wasted CPU. + if let Some(compression) = config + .what + .strip_prefix("zfs.") + .and_then(|service_name| zfs_compression.get(service_name)) + .filter(|compression| **compression != "off") + { + if config.compress { + warn!( + "Backup {}: dataset is compressed ({compression}) but compress = true: gzip wastes CPU without size gain, consider compress = false", + backup_name + ); + } else { + info!( + "Backup {}: dataset is compressed ({compression}): compress = false is the right choice", + backup_name + ); + } + } + info!("Backup {} -> {} configured", config.what, config.r#where); } diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 11a644c..281fded 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -249,6 +249,7 @@ pub struct Zfs { dataset: String, snapshot_base: String, full_when: Option, + compression: String, } /// Parses a schedule expression, accepting the human friendly formats of @@ -408,11 +409,6 @@ impl Zfs { let compression = String::from_utf8_lossy(&compress_output.stdout) .trim() .to_string(); - if compression != "off" { - info!( - "dataset {dataset} is compressed ({compression}): the zfs dump is already compressed, consider compress = false for its backups" - ); - } debug!("new zfs service on {dataset} (snapshot base {snapshot_base})"); Ok(Zfs { @@ -421,9 +417,17 @@ impl Zfs { dataset: dataset.clone(), snapshot_base: snapshot_base.clone(), full_when, + compression, }) } + /// Value of the dataset's `compression` property (`off`, `zstd`, ...). + /// With `zfs send -c -L`, a non-`off` value means the dump stream is + /// already compressed and gzip'ing it at upload is wasted CPU. + pub fn compression(&self) -> &str { + &self.compression + } + /// List all snapshots of our chain on the dataset tree, including those /// on child datasets (created with `-r`). async fn list_our_snapshots(&self) -> Vec { From 09e12ffb0c3a80fd39cb0ea97c5d2d749e179e74 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 08:22:20 +0200 Subject: [PATCH 41/51] Drop the legacy snapshot format Legacy (-) snapshots are no longer recognized: parse_snapshot returns None for them, so they are invisible to cleanup and list_files only matches -full-/-inc- dump files. Pre-existing legacy snapshots on the pool and orphan .gz files on remotes are simply left in place. Also fix a rollover race in the localhost upload tests: they computed the expected archive name from the clock *after* the upload, so a second boundary crossed during the upload made the assertion flake. Assert on the single matching archive instead. --- src/remotes/localhost.rs | 38 +++++++++++++++++++++------------ src/services/zfs.rs | 46 ++++++++++++---------------------------- 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/src/remotes/localhost.rs b/src/remotes/localhost.rs index 61f35ef..17597d8 100644 --- a/src/remotes/localhost.rs +++ b/src/remotes/localhost.rs @@ -287,12 +287,16 @@ mod tests { .await .unwrap(); - let now: chrono::DateTime = chrono::Utc::now(); - let dest = tmp_dir - .path() - .join(format!("{}-Cargo.toml.gz", now.format("%Y-%m-%d-%H.%M.%S"),)); - - assert!(dest.exists()); + // The archive name carries the upload time, so assert on the single + // matching file rather than on a name computed after the fact (the + // clock can roll over during the upload). + let matching: Vec<_> = std::fs::read_dir(tmp_dir.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with("-Cargo.toml.gz")) + .collect(); + + assert_eq!(matching.len(), 1); } #[tokio::test] @@ -347,13 +351,19 @@ mod tests { .await .unwrap(); - let now: chrono::DateTime = chrono::Utc::now(); - let dest = tmp_dir.path().join(format!( - "{}-{}.tar.gz", - now.format("%Y-%m-%d-%H.%M.%S"), - remote_filename - )); - - assert!(dest.exists()); + // The archive name carries the upload time, so assert on the single + // matching file rather than on a name computed after the fact (the + // clock can roll over during the upload). + let matching: Vec<_> = std::fs::read_dir(tmp_dir.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_string_lossy() + .ends_with(&format!("-{remote_filename}.tar.gz")) + }) + .collect(); + + assert_eq!(matching.len(), 1); } } diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 281fded..8d64eb9 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -38,10 +38,6 @@ pub(crate) enum SnapshotKind { Full, /// An incremental snapshot taken against a previous snapshot. Incremental, - /// A snapshot taken by a bacup version without full/incremental naming. - /// Legacy snapshots count as valid bases for incrementals and are - /// cleaned up like any other of our snapshots. - Legacy, } #[derive(Debug, Clone, PartialEq)] @@ -56,19 +52,18 @@ pub(crate) struct Snapshot { /// Parse a `zfs list` snapshot name (e.g. `tank/data@snap-inc-20260101-010000`) /// into a [Snapshot] if it belongs to the snapshot chain rooted at -/// `snapshot_base`, i.e. it is named `-full-`, `-inc-` -/// (new style) or `-` (legacy style). Returns `None` for names that -/// do not belong to this chain. +/// `snapshot_base`, i.e. it is named `-full-` or `-inc-`. +/// Returns `None` for names that do not belong to this chain. pub(crate) fn parse_snapshot(name: &str, snapshot_base: &str) -> Option { let after_at = name.split('@').nth(1)?; let rest = after_at.strip_prefix(snapshot_base)?.strip_prefix('-')?; - let (kind, ts) = if let Some(ts) = rest.strip_prefix(&format!("{FULL_MARK}-")) { - (SnapshotKind::Full, ts) - } else if let Some(ts) = rest.strip_prefix(&format!("{INC_MARK}-")) { - (SnapshotKind::Incremental, ts) - } else { - (SnapshotKind::Legacy, rest) - }; + let (kind, ts) = rest + .strip_prefix(&format!("{FULL_MARK}-")) + .map(|ts| (SnapshotKind::Full, ts)) + .or_else(|| { + rest.strip_prefix(&format!("{INC_MARK}-")) + .map(|ts| (SnapshotKind::Incremental, ts)) + })?; let naive = NaiveDateTime::parse_from_str(ts, TS_FORMAT).ok()?; let time = DateTime::from_naive_utc_and_offset(naive, Utc); Some(Snapshot { @@ -90,9 +85,6 @@ pub(crate) fn dump_file_name(name: &str, kind: SnapshotKind, ts: &str) -> String let mark = match kind { SnapshotKind::Full => FULL_MARK, SnapshotKind::Incremental => INC_MARK, - // Legacy snapshots are never created; their dump files use the - // legacy file name `.snapshot` directly. - SnapshotKind::Legacy => return format!("{name}{SNAPSHOT_EXT}"), }; format!("{name}-{mark}-{ts}{SNAPSHOT_EXT}") } @@ -582,8 +574,8 @@ impl Zfs { }) } - /// List the dump files this service should upload: the newest new-style - /// dump file (or the legacy `.snapshot` if none exists yet). + /// List the dump file to upload: the newest `-{full,inc}-.snapshot` + /// dump file, if any. async fn list_files(&self) -> Vec { let current_dir = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); let mut entries = vec![]; @@ -619,10 +611,6 @@ impl Zfs { if let Some(snap) = newest { entries.push(current_dir.join(snap.name)); - } else if let Ok(legacy) = fs::metadata(format!("{}{SNAPSHOT_EXT}", self.name)).await { - if legacy.is_file() { - entries.push(current_dir.join(format!("{}{SNAPSHOT_EXT}", self.name))); - } } entries @@ -746,9 +734,9 @@ mod tests { } #[test] - fn parse_snapshot_legacy() { - let s = parse_snapshot("tank@snap-20251231-235959", "snap").unwrap(); - assert_eq!(s.kind, SnapshotKind::Legacy); + fn parse_snapshot_rejects_unmarked() { + // Pre full/inc-era snapshot names are not part of our chain. + assert!(parse_snapshot("tank@snap-20251231-235959", "snap").is_none()); } #[test] @@ -890,11 +878,6 @@ mod tests { SnapshotKind::Incremental, "20260102", ), - snap( - "tank@snap-20251231-235959", - SnapshotKind::Legacy, - "20251231", - ), ]; let cleaned = cleanup_list("tank@snap-full-20260201-010000", &snaps); assert_eq!( @@ -903,7 +886,6 @@ mod tests { "tank@snap-full-20260101-010000", "tank@snap-inc-20260102-010000", "tank/data@snap-inc-20260102-010000", - "tank@snap-20251231-235959", ] ); } From ca34f22beca2959e19100381e65dd5eb9934e465 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 08:36:26 +0200 Subject: [PATCH 42/51] Track Cargo.lock bacup is an executable, so the lockfile should be committed (as the cargo-new template comment itself advises): fresh clones and gtr7-style checkouts now resolve the exact same dependency set instead of keeping stale untracked lockfiles around. --- .gitignore | 4 - Cargo.lock | 3756 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 3756 insertions(+), 4 deletions(-) create mode 100644 Cargo.lock diff --git a/.gitignore b/.gitignore index 29a8258..fe2aedf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,6 @@ # will have compiled files and executables /target/ -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - # These are backup files generated by rustfmt **/*.rs.bk diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..77ad495 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3756 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "astral-tokio-tar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2e989b33246fe9240d39accf4dd9a01e0b6c1f3ce9dd095e0a47fa02505523" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", + "zerocopy", +] + +[[package]] +name = "async-compression" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-config" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.5.0", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 1.5.0", + "http-body 1.1.0", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.152.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "004d5acbd6850ab58789a43822e0a2ef23927a46b012c17f8fa51253103db9f8" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac 0.13.0", + "http 1.5.0", + "http-body 1.1.0", + "lru", + "percent-encoding", + "regex-lite", + "sha2 0.11.0", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.114.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.5.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.5.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 1.5.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http 1.5.0", + "p256", + "percent-encoding", + "sha2 0.11.0", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1 0.11.0", + "sha2 0.11.0", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.61.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.19", + "http 0.2.12", + "http 1.5.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.11.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.10", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.45", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.6", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.5.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + +[[package]] +name = "aws-smithy-types" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.5.0", + "http-body 0.4.6", + "http-body 1.1.0", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "bacup" +version = "0.2.0" +dependencies = [ + "astral-tokio-tar", + "async-compression", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", + "chrono", + "cron", + "croner 4.0.1", + "dyn-clone", + "futures", + "get-file-size", + "glob", + "log", + "regex", + "scopeguard", + "serde", + "shellexpand", + "stderrlog", + "structopt", + "sysinfo", + "tempfile", + "tokio", + "tokio-cron-scheduler", + "toml", + "url", + "uuid", + "which", +] + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "case" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88b166b48e29667f5443df64df3c61dc07dc2b1a0b0d231800e07f09a33ecc1" + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim 0.8.0", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "compression-codecs" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cron" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dcd6f69605c2956916ce24e8af637b754964c9a83f4662d3a2361654cdba09" +dependencies = [ + "chrono", + "once_cell", + "phf 0.11.3", + "winnow 0.7.15", +] + +[[package]] +name = "croner" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa42bcd3d846ebf66e15bd528d1087f75d1c6c1c66ebff626178a106353c576" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + +[[package]] +name = "croner" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b5b023fc2978536bbb2a3b71237964fc4630bec1eb31077b4977127f4cdb9c0" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote 1.0.47", + "strsim 0.11.1", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive-error" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6238eafbc141db51c6d0d599dde74224b28437c7cf7c3a1755e3e3e5e00806" +dependencies = [ + "case", + "quote 0.3.15", + "syn 0.11.11", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "error-tree" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07cec0a4beddbd78d1b38d9018b08b00d19caa8fc3cdf1c45a424623b045974d" +dependencies = [ + "derive-error", + "export-magic", + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "export-magic" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed02add08ce10f7664f1567c72f195cec7e630a666a7a04a98a9633d402b02ba" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core", + "subtle", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "get-file-size" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a897b9c02b91b1d1d5c8d5a46ce2201ba9bbf7c5443006d238c224151dc57f4" +dependencies = [ + "async-trait", + "error-tree", + "export-magic", + "tokio", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.5.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http 1.5.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http 1.5.0", + "http-body 1.1.0", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.19", + "http 1.5.0", + "http-body 1.1.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http 1.5.0", + "hyper 1.11.1", + "hyper-util", + "rustls 0.23.45", + "rustls-native-certs", + "tokio", + "tokio-rustls 0.26.6", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.5.0", + "http-body 1.1.0", + "httparse", + "hyper 1.11.1", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.5", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi 0.5.3", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.2", + "objc2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-open-directory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb82bed227edf5201dfedf072bba4015a33d3d4a98519837295a90f0a23f676d" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote 1.0.47", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.15", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "dirs", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stderrlog" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c910772f992ab17d32d6760e167d2353f4130ed50e796752689556af07dc6b" +dependencies = [ + "chrono", + "is-terminal", + "log", + "termcolor", + "thread_local", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote 1.0.47", + "syn 1.0.109", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +dependencies = [ + "quote 0.3.15", + "synom", + "unicode-xid", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "unicode-ident", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "sysinfo" +version = "0.39.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2071df9448915b71c4fe6d25deaf1c22f12bd234f01540b77312bb8e41361e6" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "objc2-open-directory", + "windows", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-cron-scheduler" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f50e41f200fd8ed426489bd356910ede4f053e30cebfbd59ef0f856f0d7432a" +dependencies = [ + "chrono", + "chrono-tz", + "croner 3.0.1", + "num-derive", + "num-traits", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls 0.23.45", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "once_cell", + "regex-automata", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote 1.0.47", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote 1.0.47", + "syn 3.0.6", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" From c54d8d70d3285ce52f88efb002f0c5ed4c281ae1 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 14:40:33 +0200 Subject: [PATCH 43/51] remove dead code - Drop removes the files, nothing manual --- src/services/service.rs | 6 ++++- src/services/zfs.rs | 60 +++-------------------------------------- 2 files changed, 8 insertions(+), 58 deletions(-) diff --git a/src/services/service.rs b/src/services/service.rs index 0a67978..813b5bd 100644 --- a/src/services/service.rs +++ b/src/services/service.rs @@ -25,7 +25,11 @@ pub struct Dump { impl Drop for Dump { fn drop(&mut self) { if let Some(path) = &self.path { - // If we created a dump file, we should take care of removing it + // If we created a dump file, we should take care of removing it. + // Note: this only covers normal termination of the backup job; + // a crash (panic / kill) between the dump and this drop leaves + // the file behind — such stale files are not cleaned up by the + // service and must be removed manually if they appear. if path.exists() { #[allow(unused_must_use)] { diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 8d64eb9..171326a 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -185,21 +185,6 @@ pub(crate) fn cleanup_list(new_full_name: &str, snapshots: &[Snapshot]) -> Vec-{full,inc}-.snapshot`) -/// older than the snapshot with name `newer_name`, i.e. the ones to remove -/// after a new full backup. Entries that do not belong to our chain are dropped. -pub(crate) fn old_dump_files(entries: &[String], name: &str, newer_name: &str) -> Vec { - let Some(newer) = parse_snapshot_name(newer_name, name) else { - return vec![]; - }; - entries - .iter() - .filter_map(|e| parse_snapshot_name(e, name)) - .filter(|s| s.time < newer.time) - .map(|s| s.name.clone()) - .collect() -} - #[derive(Debug)] pub enum Error { ZfsError(std::io::Error), @@ -641,8 +626,9 @@ fn parse_snapshot_name(file_name: &str, name: &str) -> Option { impl Zfs { /// After a successful full: destroy all older snapshots of our chain on - /// the dataset tree and remove old local dump files. Best effort: failures - /// are logged but do not fail the backup. + /// the dataset tree. Best effort: failures are logged but do not fail the + /// backup. Local dump files are not cleaned up here: the [Dump] returned + /// by `do_dump` removes its own file when dropped by the backup job. async fn cleanup(&self, snapshots: &[Snapshot], new_full_name: &str) { let to_destroy = cleanup_list(new_full_name, snapshots); let mut destroyed = 0; @@ -660,27 +646,6 @@ impl Zfs { to_destroy.len() ); } - - let files: Vec = match fs::read_dir(".").await { - Ok(mut read_dir) => { - let mut files = vec![]; - while let Ok(Some(entry)) = read_dir.next_entry().await { - files.push(entry.file_name().to_string_lossy().to_string()); - } - files - } - Err(e) => { - warn!("failed to list local dump files for cleanup: {e}"); - vec![] - } - }; - for file in old_dump_files(&files, &self.name, new_full_name) { - match fs::remove_file(&file).await { - Ok(()) => info!("removed old local dump file {file}"), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => warn!("failed to remove old local dump file {file}: {e}"), - } - } } } @@ -899,25 +864,6 @@ mod tests { assert!(parse_snapshot_name("other-full-20260102-020000.snapshot", "snap").is_none()); } - #[test] - fn old_dump_files_are_strictly_older() { - let entries = vec![ - "snap-full-20260101-010000.snapshot".to_string(), - "snap-inc-20260101-120000.snapshot".to_string(), - "snap-full-20260102-010000.snapshot".to_string(), // the new full itself - "other-full-20260101-010000.snapshot".to_string(), // not ours - "readme.txt".to_string(), - ]; - let old = old_dump_files(&entries, "snap", "snap-full-20260102-010000.snapshot"); - assert_eq!( - old, - vec![ - "snap-full-20260101-010000.snapshot".to_string(), - "snap-inc-20260101-120000.snapshot".to_string(), - ] - ); - } - #[test] fn parse_schedule_accepts_when_and_cron_formats() { assert!(parse_schedule("monthly 1 01:00").is_ok()); From e838d188ffffa2673dbc16c5de8e54c4b710ea58 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 20:35:28 +0200 Subject: [PATCH 44/51] Fix bug-hunt findings: aws stream put_object, openssh encrypted-key detection, docker info connectivity, postgres password/quotes/exit status, whole-word weekday parsing, lossy process output, storage typo - 3.8: put_object small-file path streams via ByteStream::from_path instead of buffering the whole file in a Vec (up to 1 GiB RSS spike) - 3.4: detect encrypted openssh-key-v1 private keys (ciphername != none) - 3.6: docker connectivity check uses docker info instead of pulling hello-world - 3.7: optional PGPASSWORD, SQL-escape db_name, fail on psql exit status, pg_dump gets PGPASSWORD - 3.12: parse_weekly matches weekday words exactly instead of .contains() - 2.2: ssh probe output via from_utf8_lossy (strict text stays read_to_string) - 3.10: 'Storagee' -> 'Storage' - BUGS.md: full-repo bug report with status --- BUGS.md | 276 +++++++++++++++++++++++++++++++++++++ Cargo.lock | 19 ++- Cargo.toml | 1 + src/config.rs | 5 + src/remotes/aws.rs | 15 +- src/remotes/remote.rs | 2 +- src/remotes/ssh.rs | 120 +++++++++++++++- src/services/docker.rs | 7 +- src/services/postgresql.rs | 54 ++++++-- src/when.rs | 14 +- 10 files changed, 477 insertions(+), 36 deletions(-) create mode 100644 BUGS.md diff --git a/BUGS.md b/BUGS.md new file mode 100644 index 0000000..a5da005 --- /dev/null +++ b/BUGS.md @@ -0,0 +1,276 @@ +# bacup code review — bug report + +Full-repo review of all 18 files under `src/` (2026-10-04), re-audited +2026-10-05 against the post-fix tree (commits 3fec877, 17c8d90, a631331, +09e12ff, ca34f22). Findings ordered by severity: data-corruption / wrong-data +bugs first, then panic hazards, then functional oddities. Items marked +[FIXED] were verified against the current tree and the fix batch. + +**Second pass found a latent cleanup hole in #1.8 (the `cleanup()` local-file +deletion branch is dead code). Note: this is *not* an active disk leak — +normal runs delete the local dump via `Dump::drop`, verified on gtr7.** + +## 1. Data-corruption / wrong-data bugs + +### 1.1 [FIXED] `src/services/postgresql.rs` `dump()` — pg_dump exit status ignored + +The `match` on `.status().await` returns `Ok(Dump)` for *any* successful spawn, +discarding the `ExitStatus`. A non-zero `pg_dump` exit (permissions, lock wait, +OOM) is reported as a successful backup and the empty/truncated `.sql` gets +uploaded. **Fix:** check `status.code()` and return an error on non-zero. + +### 1.2 `src/remotes/git.rs` `upload_file` — file nested one level too deep + +`dest = repo.join(remote_path)` is the *file* path; the code does +`create_dir_all(dest)` then copies to `dest.join(file_name)`, so the first +upload creates `repo//mydb.sql/mydb.sql`. Every later run reuses the +directory, so the wrong nesting is stable. +**Fix:** `create_dir_all(dest.parent()?)` + `copy(path, dest)`. + +### 1.3 `src/remotes/git.rs` `upload_folder` — directory tree flattened + +Every file is copied to `dest.join(file_name)` regardless of its subdirectory, +so nested structures are lost and name collisions are silently overwritten. +**Fix:** preserve relative paths, like the localhost/aws remotes do. + +### 1.4 `src/remotes/ssh.rs` `upload_folder` — only one directory synced when the glob matches several + +`local_prefix = paths.min().parent()`. With a pattern like `/home/u/*/data` +(or any glob spanning sibling directories) rsync only sends the +lexicographically-first directory; the rest are silently skipped while +`--delete` keeps the remote "consistent" with that subset. +**Fix:** resolve the common ancestor of all matched paths (or require a single +root and error otherwise). + +### 1.5 [FIXED] `src/remotes/ssh.rs` `enumerate` / `delete` — unquoted remote paths + +`find {}/*` and `rm -r {}` interpolate `remote_path` raw, while `upload_file` +(correctly) uses the `shell_quote` helper. A path with spaces breaks `find` and +makes `rm -r` target the wrong thing — `delete` is the most dangerous one. +**Fix:** use the existing `shell_quote` helper in both commands. + +### 1.6 `src/remotes/ssh.rs` `upload_file` — no atomic upload; partial files on failure + +`cat > file` writes directly to the timestamped name. A network drop or +remote-disk-full mid-transfer leaves a truncated "backup" that later runs +treat as valid. **Fix:** upload to `name.tmp` and `mv` into place on success +(same for the compressed variant). + +### 1.7 [FIXED] `src/remotes/aws.rs` `enumerate` — `list_objects_v2` not paginated + +Only the first 1000 keys are ever seen, so `keep_last` pruning silently stops +working beyond 1000 objects per prefix. +**Fix:** loop on `continuation_token` / `is_truncated`. + +### 1.8 [FIXED] `src/services/zfs.rs` `cleanup()` — local-file deletion branch was dead code + +`do_dump` calls `self.cleanup(&snapshots, &new_name)` where `new_name` is the +ZFS **snapshot** name (`dataset@base-full-`, no `.snapshot` suffix), but +`cleanup` → `old_dump_files` → `parse_snapshot_name` does +`strip_suffix(".snapshot")?` on that argument, which fails, so +`old_dump_files` always returns `vec![]` and the `fs::remove_file` loop never +runs. The test `old_dump_files_are_strictly_older` passes a *file* name as +`newer_name`, masking the mismatch. + +**This is *not* an active disk leak.** The primary cleanup path is +`Dump::drop` (`src/services/service.rs`), which removes the dump file when +the `Dump` value goes out of scope at the end of the scheduled job in +`backup.rs`. Verified in production on gtr7: the current +`/storage/zroot-full-20261005-100000.snapshot` was already gone from the +working directory after the upload, and no older `*.snapshot` files +accumulated. `cleanup()`'s file deletion is only a redundant second layer. + +The real (narrow) leak window: if the process crashes (panic / SIGKILL / +OOM) between `zfs send` completing and `Drop` running, the raw stream is +left in the working directory and nothing will ever remove it — full dumps +are raw `zfs send` streams, so one stale file can be large. + +**Fix:** just delete the dead code. `Dump::drop` is the real cleanup path +and it works (verified in production), so the file-deletion layer in +`cleanup()` — the `fs::read_dir` + `old_dump_files` + `fs::remove_file` +loop — plus `old_dump_files` itself and its test +`old_dump_files_are_strictly_older` should be removed. Keep the rest of +`cleanup()` (`cleanup_list` + `destroy`), which is the live ZFS snapshot +cleanup. Optionally note in `Drop`'s doc that a crash between `send` and +drop can leave a stale file behind. + +**Applied:** removed the `fs::read_dir` / `old_dump_files` / `fs::remove_file` +block from `cleanup()`, deleted `old_dump_files` and its test, updated the +`cleanup()` doc to point at `Dump::drop`, and documented the crash-orphan +caveat in `Drop`'s doc in `service.rs`. Full test suite passes (48 + 6 ok). + +## 2. Panic hazards (crash instead of `Err`) + +### 2.1 `status.code().unwrap()` on signal-killed processes + +`docker.rs` (`new` + `dump`), `postgresql.rs` `new`, `ssh.rs` `new` (error +path). A SIGKILLed docker/pg_isready/ssh panics the whole scheduler instead of +returning an error. `status.code()` is `None` for signal-killed processes. + +### 2.2 [FIXED] `String::from_utf8(...).unwrap()` on command output + +`ssh.rs` `new`, `postgresql.rs` `new` (both stdout and stderr). Non-UTF8 +output (e.g. a broken locale message) panics startup. + +**Applied:** command output is best-effort diagnostic text (locale-dependent +remote banners/messages), so both `ssh.rs` and `postgresql.rs` now decode it +with `String::from_utf8_lossy` (the postgresql half landed as a drive-by in +#3.7, the ssh half directly). This is the only place lossy is *required* by +the strict-vs-lossy rule — the strict-text sites (`config.rs` TOML load, +private key load) already return `Err` on non-UTF8 via `fs::read_to_string`, +and the aws binary path keeps `Vec` with no text assumption. + +### 2.3 `.unwrap()` on collections that can be empty / non-UTF8 + +- `aws.rs` + `ssh.rs` `upload_folder`: `paths.iter().min_by(...).unwrap()` (empty paths) +- `remote.rs` `compress_folder`: `path.file_name().unwrap()` (root-level path) +- `git.rs`: `remote_path.strip_prefix("/").unwrap()` (empty remote_path) +- `remote.rs` / `postgresql.rs`: `std::env::current_dir().unwrap()` + +### 2.4 `src/bin/bacup.rs` + +Every service/remote constructor call ends in `.unwrap()` (including +`Backup::new(...).await.unwrap()`), so one misconfigured service kills the +entire process with a backtrace instead of the clean `error!` + +`return Err(-1)` style used by the keep_last gate. + +### 2.5 [FIXED] `src/backup.rs` scheduled job — `.unwrap()` on empty file list + +`local_files.iter().min_by(...).unwrap()` in the cron job panicked when +`service.list()` returned nothing (e.g. a folder glob that matches no files). +`Backup::new` uses `if let Some(...)`, so only the scheduled path was exposed. +Fixed with an early `error!("...no files, nothing to back up") + return` guard +right after `service.list()` in the job. + +## 3. Functional oddities (lower severity) + +### 3.1 [FIXED] `src/remotes/git.rs` `clone_repository` — clone dir is `/` + +Two git remotes whose repos share a basename silently share the first repo's +clone (the `dest.exists()` shortcut). Also, a failed `git pull` (diverging +shallow-clone branch) leaves conflict markers in the worktree, and +`git add . -A` commits them as a "backup" — pull failure output is ignored. + +### 3.2 [FIXED] `src/remotes/git.rs` `upload_file_compressed` — deferred cleanup targets the wrong path + +The deferred `fs::remove_file(&remote_path)` operates on the *remote* path +string (e.g. `/backups/x.sql-YYYY-MM-DD-HH.MM.gz`) as if it were a local path: +silently no-ops, `#[must_use]` result ignored. It should remove the local temp +file under the repo dir. + +### 3.3 [FIXED] `src/remotes/git.rs` push-failure error message + +Says "Unable to execute git add . -A" — copy-paste from the add step; the +failing command is `git push`. + +### 3.4 [FIXED] `src/remotes/ssh.rs` encrypted-key detection + +Matches only legacy PEM (`Proc-Type` + `ENCRYPTED`). Modern OpenSSH-format +encrypted keys are not flagged, so users get a confusing auth failure instead +of the "key is encrypted" hint. + +Fix: added `openssh_key_is_encrypted()` in src/remotes/ssh.rs — base64-decodes +the `-----BEGIN OPENSSH PRIVATE KEY-----` blob, validates the `openssh-key-v1` +magic, and checks the cipher name field against `"none"`; `Ssh::new` now rejects +passphrase-protected OpenSSH-format keys with the same `InvalidPrivateKey` +error as the legacy path. Added the `base64` crate (0.22.1) dependency. +4 new unit tests (unencrypted/encrypted OpenSSH keys, legacy PEM out of scope, +malformed blocks). + +### 3.5 [FIXED] `src/remotes/remote.rs` archive names have minute precision + +`%H.%M` means two folder/file backups finishing within the same minute produce +the same remote name; the second overwrites the first. (zfs dumps avoid this: +one file per run with second precision.) + +### 3.6 [FIXED] `src/services/docker.rs` `new` — connectivity check pulls an image + +Runs `docker run --rm hello-world` on every startup, downloading the +hello-world image each time and requiring network. + +Fix: `Docker::new` now runs `docker info` instead — validates the daemon is +running and reachable with no image pull and no network. + +### 3.7 [FIXED] `src/services/postgresql.rs` misc + +- db-existence check builds SQL by interpolation (`datname='{db_name}'`), so a + quote in the configured name breaks/mangles the query (low severity: config + is user-managed). +- The psql connection check treats *any* non-empty stderr as failure, so + benign locale warnings fail startup. +- No password field in config: `pg_dump` runs with `--no-password`, so + authentication relies entirely on `~/.pgpass`. + +Fix: +- single quotes in `db_name` are SQL-doubled (`'` → two single quotes) before + the name is interpolated into the `datname` query. +- The check now fails on `!status.success()` (exit status), not on any + non-empty stderr; stdout is expected to be exactly `"1"`. +- `PostgreSqlConfig` (and the `PostgreSql` service) gained + `password: Option` (serde `#[default]`, existing configs unaffected); + when set, both `psql` and `pg_dump` get it via the `PGPASSWORD` env var. +- psql stdout/stderr now read with `String::from_utf8_lossy` instead of + `str::from_utf8`. (6 existing test literals updated with `password: None`.) + +### 3.8 [FIXED] `src/remotes/aws.rs` `put_object` memory spike + +Buffers whole files ≤ 1024 MiB entirely into a `Vec` before upload — a 1 GB +dump spikes RSS by 1 GB unnecessarily. A streaming body would fix it. + +Fix: `put_object` no longer reads the file into a `Vec` — the small-file path +now uses `ByteStream::from_path(path)`, which streams the file from disk and +sets `Content-Length` from file metadata (aws-smithy-types 1.8.1 has no +`ByteStream::from(tokio::fs::File)`; the earlier note described a planned +approach, not what shipped). `CHUNK_SIZE` now only decides put-vs-multipart; +the unused `tokio::fs::File` and `AsyncReadExt` imports are gone. + +### 3.9 [FIXED] `src/bin/bacup.rs` main loop — tick errors invisible + +The `scheduler.tick()` error check is commented out and replaced with a bare +`sleep(50ms)` loop, so scheduler tick failures are never surfaced. + +### 3.10 [FIXED] Typos in user-visible messages / comments + +- `remote.rs`: "Storagee error" (should be "Storage") +- `ssh.rs`: "succeded" +- `postgresql.rs`: "does not exit" (→ "exist"); comments: "shuld be + performend", "aksing" + +### 3.11 [RESOLVED BY #3.5] `src/remotes/remote.rs` `remote_compressed_file_path` — space in timestamp + +Re-checked after the 3.5 fix: both `remote_archive_path` and +`remote_compressed_file_path` now build names with +`now.format("%Y-%m-%d-%H.%M.%S")` (no space, seconds precision) — the +`ts.replace(...)` code no longer exists. No action needed. + +### 3.12 [FIXED] `src/when.rs` `parse_weekly` — `.contains()` day matching is a substring scan + +`day.to_lowercase().contains(day_name)` matches substrings, not whole words +(no two day names are substrings of each other, and the HH:MM regex rejects +the leftovers, so no practical misfire — but `"tuesday 10:00"` containing +`"tue"`-style input is not a supported format anyway). Cosmetic; note only. + +Fix: `parse_weekly` now matches day names as whole words (whitespace-token +membership) and removes only the matched word when consuming the input, +instead of `.contains()`/`.replace()` substring scans. All existing when +tests still pass unchanged. + +## 4. What's solid + +- `zfs.rs` full+incremental chain logic, keep_last gate, delegated-permission + check, compression nudge, and the `when`/cron `full_when` resolution all + check out (the dead local-file cleanup in `cleanup()` was removed — see + #1.8; the real cleanup is `Dump::drop`). +- `when.rs` (shared when-format module, fixed dow mapping + regression test), + `folders.rs`, `localhost.rs`, `service.rs`, `config.rs`, `lib.rs` +- The keep_last startup gate, `what`/`where` validation, and the zfs + `required_keep_last` upper-bound simulation are correct (conservative by ≤ 1 + when `full_when` fires between backup runs — safe direction). + +## Suggested fix order (second pass, 2026-10-05) + +1. ~~#1.8 remove the dead local-file cleanup code in zfs `cleanup()`~~ [DONE] +2. #2.1–2.4 panic sweep (all are `unwrap()` on signal/utf8/empty inputs) +3. #1.2 / #1.3 git nesting + flattening +4. #1.4 ssh multi-dir glob, #1.6 ssh atomic upload +5. ~~#3.4, #3.6, #3.7, #3.8, #3.12~~ [DONE — applied and verified with `cargo build` + `cargo test` (58 passed, 2 ignored)] diff --git a/Cargo.lock b/Cargo.lock index 77ad495..878f52d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -593,6 +593,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "aws-types", + "base64 0.22.1", "chrono", "cron", "croner 4.0.1", @@ -623,6 +624,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64" version = "0.23.1" @@ -1155,7 +1162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1648,7 +1655,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", @@ -1824,7 +1831,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.3", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2425,7 +2432,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2935,7 +2942,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3408,7 +3415,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cf60049..71341e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ tokio = { version = "1.53.2", features = ["full"] } toml = "1.1.6" url = { version = "2.5.8", features = ["serde"] } which = "8.0.6" +base64 = "0.22.1" aws-config = "1.12.0" aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-s3 = "1.152.0" diff --git a/src/config.rs b/src/config.rs index 3f9122f..73b8f4a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -58,6 +58,11 @@ pub struct PostgreSqlConfig { pub db_name: String, pub host: Option, pub port: Option, + /// Optional password, passed to psql/pg_dump via the `PGPASSWORD` + /// environment variable (libpq). When absent, pg_dump runs with + /// `--no-password` and relies on peer/trust authentication. + #[serde(default)] + pub password: Option, } #[derive(Serialize, Deserialize)] diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index e9a4dbd..0bb5092 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -23,9 +23,6 @@ use crate::remotes::remote; use std::path::{Path, PathBuf}; -use tokio::fs::File; -use tokio::io::AsyncReadExt; - use async_trait::async_trait; use log::{info, warn}; @@ -119,17 +116,17 @@ impl Bucket { ); if file_size <= CHUNK_SIZE { - // Just read the file and upload to bytes. - // Suppose CHUNK_SIZE free memory available. - let mut content: Vec = vec![]; - let mut file = File::open(path).await?; - file.read_to_end(&mut content).await?; + // Stream the file directly instead of buffering it in memory; + // ByteStream::from_path sets Content-Length from file metadata. + let body = ByteStream::from_path(path) + .await + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; let response = self .client .put_object() .bucket(&self.bucket_name) .key(remote_path) - .body(ByteStream::from(content)) + .body(body) .send() .await; diff --git a/src/remotes/remote.rs b/src/remotes/remote.rs index fc166c1..46d9a70 100644 --- a/src/remotes/remote.rs +++ b/src/remotes/remote.rs @@ -69,7 +69,7 @@ impl fmt::Display for Error { Error::CompressionError => write!(f, "Unable to compress the file/folder"), Error::NotADirectory => write!(f, "The specified file is not a directory"), Error::RemoteError(error) => write!(f, "Remote error: {:?}", error), - Error::StorageError(error) => write!(f, "Storagee error: {:?}", error), + Error::StorageError(error) => write!(f, "Storage error: {:?}", error), } } } diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index d49f00c..9bd184c 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -36,6 +36,8 @@ use async_trait::async_trait; use std::process::{Command, Stdio}; use which::which; +use base64::Engine; + #[derive(Debug)] pub enum Error { InvalidPrivateKey(String), @@ -67,6 +69,50 @@ impl fmt::Display for Error { } } +/// Detect passphrase-encrypted private keys in the modern OpenSSH format +/// (`openssh-key-v1`), which unlike legacy PEM keys has no `Proc-Type`/ +/// `ENCRYPTED` header. The cipher name right after the key magic is `none` +/// for unencrypted keys and a cipher name (e.g. `aes256-ctr`) otherwise. +fn openssh_key_is_encrypted(key: &str) -> bool { + const BEGIN: &str = "-----BEGIN OPENSSH PRIVATE KEY-----"; + const END: &str = "-----END OPENSSH PRIVATE KEY-----"; + let Some(start) = key.find(BEGIN) else { + return false; + }; + let rest = &key[start + BEGIN.len()..]; + let Some(end) = rest.find(END) else { + return false; + }; + let b64: String = rest[..end] + .chars() + .filter(|c| c.is_ascii_alphanumeric() || *c == '+' || *c == '/') + .collect(); + let Ok(bytes) = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64.as_bytes()) + else { + return false; + }; + // openssh-key-v1 layout: uint32 len, magic, uint32 len, ciphername, ... + let read_u32 = |bytes: &[u8], pos: usize| -> Option { + bytes + .get(pos..pos + 4) + .map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + }; + let Some(magic_len) = read_u32(&bytes, 0).map(|v| v as usize) else { + return false; + }; + if bytes.len() < 4 + magic_len || &bytes[4..4 + magic_len] != b"openssh-key-v1" { + return false; + } + let Some(cipher_len) = read_u32(&bytes, 4 + magic_len).map(|v| v as usize) else { + return false; + }; + let pos = 4 + magic_len + 4; + if bytes.len() < pos + cipher_len { + return false; + } + bytes[pos..pos + cipher_len] != *b"none" +} + /// Quote a value for use inside a remote shell command (POSIX single-quoting). fn shell_quote(value: &str) -> String { format!("'{}'", value.replace('\'', "'\\''")) @@ -102,6 +148,15 @@ impl Ssh { private_key.display() ))); } + // Modern OpenSSH keys ("openssh-key-v1") have no Proc-Type header; + // detect encryption from the cipher name in the key blob. + if openssh_key_is_encrypted(&private_key_file) { + return Err(Error::InvalidPrivateKey(format!( + "Private key {} is encrypted with a passphrase. \ + A key without passphrase is required", + private_key.display() + ))); + } let port = format!("{}", config.port); let host = format!("{}@{}", config.username, config.host); @@ -119,8 +174,11 @@ impl Ssh { } let output = output.unwrap(); - let stdout = String::from_utf8(output.stdout).unwrap(); - let stderr = String::from_utf8(output.stderr).unwrap(); + // Process output is arbitrary bytes (locale-dependent remote + // banners/messages) — display it lossy rather than panicking on + // non-UTF8. + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); if stdout.is_empty() && stderr.contains("true") { // like on github.com -> can connect, can't execute anything on the shell @@ -439,3 +497,61 @@ impl remote::Remote for Ssh { .await } } + +#[cfg(test)] +mod tests { + use super::*; + + fn make_openssh_key(cipher: &[u8]) -> String { + // Minimal openssh-key-v1 blob: magic + cipher name. The parser only + // reads up to the cipher name, so the rest of the structure is not + // needed. + let magic = b"openssh-key-v1"; + let mut blob = vec![]; + blob.extend_from_slice(&(magic.len() as u32).to_be_bytes()); + blob.extend_from_slice(magic); + blob.extend_from_slice(&(cipher.len() as u32).to_be_bytes()); + blob.extend_from_slice(cipher); + let encoded = base64::engine::general_purpose::STANDARD_NO_PAD.encode(&blob); + format!( + "-----BEGIN OPENSSH PRIVATE KEY-----\n{encoded}\n-----END OPENSSH PRIVATE KEY-----\n" + ) + } + + #[test] + fn unencrypted_openssh_key_is_not_reported_as_encrypted() { + let key = make_openssh_key(b"none"); + assert!(!openssh_key_is_encrypted(&key)); + } + + #[test] + fn encrypted_openssh_key_is_reported_as_encrypted() { + for cipher in [b"aes256-ctr", b"aes128-cbc"] { + let key = make_openssh_key(cipher); + assert!(openssh_key_is_encrypted(&key)); + } + } + + #[test] + fn legacy_pem_keys_are_out_of_scope() { + // Legacy PEM (with or without the Proc-Type/ENCRYPTED header) has no + // OPENSSH block: the Proc-Type check in new() covers that format. + let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIE...\n-----END RSA PRIVATE KEY-----\n"; + assert!(!openssh_key_is_encrypted(pem)); + let encrypted_pem = "Proc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC\nMIIE..."; + assert!(!openssh_key_is_encrypted(encrypted_pem)); + } + + #[test] + fn malformed_openssh_blocks_are_not_reported_as_encrypted() { + let garbage = "-----BEGIN OPENSSH PRIVATE KEY-----\n!!!!not-base64!!!!\n-----END OPENSSH PRIVATE KEY-----\n"; + assert!(!openssh_key_is_encrypted(garbage)); + // Valid base64, but not an openssh-key-v1 blob. + let encoded = + base64::engine::general_purpose::STANDARD_NO_PAD.encode(b"not a key at all"); + let not_a_key = format!( + "-----BEGIN OPENSSH PRIVATE KEY-----\n{encoded}\n-----END OPENSSH PRIVATE KEY-----\n" + ); + assert!(!openssh_key_is_encrypted(¬_a_key)); + } +} diff --git a/src/services/docker.rs b/src/services/docker.rs index 87b0d82..3d46cf3 100644 --- a/src/services/docker.rs +++ b/src/services/docker.rs @@ -58,7 +58,10 @@ impl Docker { Ok(cmd) => cmd, }; - let args = vec!["run", "--rm", "hello-world"]; + // `docker info` only talks to the local daemon: unlike + // `docker run hello-world` it doesn't pull an image or need + // network access. + let args = vec!["info"]; let status = Command::new(&cmd) .args(&args) .stdout(Stdio::null()) @@ -70,7 +73,7 @@ impl Docker { let code = status.unwrap().code().unwrap(); if code != 0 { return Err(Error::RuntimeError(io::Error::other(format!( - "docker run hello-world failed, exit code {}", + "docker info failed (is the docker daemon running and accessible?), exit code {}", code )))); } diff --git a/src/services/postgresql.rs b/src/services/postgresql.rs index f688ccd..6691e3e 100644 --- a/src/services/postgresql.rs +++ b/src/services/postgresql.rs @@ -31,6 +31,7 @@ pub struct PostgreSql { pub cmd: PathBuf, pub args: Vec, pub dumped_to: PathBuf, + pub password: Option, } #[derive(Debug)] @@ -96,21 +97,37 @@ impl PostgreSql { }; args.push("-tAc"); - let query = format!(r#"SELECT 1 FROM pg_database WHERE datname='{}'"#, db_name); + // Escape single quotes so a database name like o'reilly can't break + // out of the string literal in the query. + let query = format!( + "SELECT 1 FROM pg_database WHERE datname='{}'", + db_name.replace('\'', "''") + ); args.push(&query); - let output = match Command::new(cmd).args(&args).output().await { + let mut psql = Command::new(cmd); + psql.args(&args); + if let Some(password) = &config.password { + psql.env("PGPASSWORD", password); + } + let output = match psql.output().await { Err(error) => return Err(Error::RuntimeError(error)), Ok(output) => output, }; - let stderr = std::str::from_utf8(&output.stderr).unwrap().trim(); - if !stderr.is_empty() { - return Err(Error::RuntimeError(io::Error::other(stderr))); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + // Fail on exit status, not on any stderr output: psql exits non-zero + // on connection/auth failures, while harmless warnings (e.g. locale) + // can appear on stderr with a successful run. + if !output.status.success() { + return Err(Error::RuntimeError(io::Error::other(format!( + "psql connection check failed: {}", + if stderr.is_empty() { "unknown error".to_string() } else { stderr } + )))); } - let stdout = std::str::from_utf8(&output.stdout).unwrap().trim(); - if stdout == "0" { + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if stdout != "1" { return Err(Error::RuntimeError(io::Error::other(format!( "database {} does not exist or user {} not allowed to query the db", db_name, username @@ -135,6 +152,7 @@ impl PostgreSql { args: args.iter().map(|s| s.to_string()).collect(), cmd, dumped_to: PathBuf::new(), + password: config.password.clone(), }) } } @@ -165,12 +183,16 @@ impl Service for PostgreSql { .into()); } - match Command::new(self.cmd.clone()) - .args( - self.args - .iter() - .chain(&["-f".to_string(), dest.to_str().unwrap().to_string()]), - ) + let mut command = Command::new(self.cmd.clone()); + command.args( + self.args + .iter() + .chain(&["-f".to_string(), dest.to_str().unwrap().to_string()]), + ); + if let Some(password) = &self.password { + command.env("PGPASSWORD", password); + } + match command .status() .await { @@ -203,6 +225,7 @@ mod tests { db_name: String::from(DB_NAME), host: Some(String::from(HOST)), port: Some(PORT), + password: None, }; assert!(PostgreSql::new(config, NAME).await.is_ok()); } @@ -214,6 +237,7 @@ mod tests { db_name: String::from(DB_NAME), host: Some(String::from(HOST)), port: Some(PORT), + password: None, }; assert!(PostgreSql::new(config, NAME).await.is_err()); } @@ -225,6 +249,7 @@ mod tests { db_name: String::from("wat"), host: Some(String::from(HOST)), port: Some(PORT), + password: None, }; assert!(PostgreSql::new(config, NAME).await.is_err()); } @@ -236,6 +261,7 @@ mod tests { db_name: String::from(DB_NAME), host: Some(String::from("wat")), port: Some(PORT), + password: None, }; assert!(PostgreSql::new(config, NAME).await.is_err()); } @@ -247,6 +273,7 @@ mod tests { db_name: String::from(DB_NAME), host: Some(String::from(HOST)), port: Some(69), + password: None, }; assert!(PostgreSql::new(config, NAME).await.is_err()); } @@ -259,6 +286,7 @@ mod tests { db_name: String::from(DB_NAME), host: Some(String::from(HOST)), port: Some(PORT), + password: None, }; let db = PostgreSql::new(config, NAME).await.unwrap(); diff --git a/src/when.rs b/src/when.rs index 667905f..045c7ba 100644 --- a/src/when.rs +++ b/src/when.rs @@ -71,11 +71,19 @@ pub fn parse_weekly(input: &str) -> Result { String::from(d.1).to_lowercase(), ) }); + // Match day names as whole words, not as substrings of other words. + let words: Vec<&str> = input.split_whitespace().collect(); for day in weekdays { - let short = input.contains(&day.0); - let long = input.contains(&day.1); + let short = words.contains(&day.0.as_str()); + let long = words.contains(&day.1.as_str()); if short || long { - let input = input.replace(if long { &day.1 } else { &day.0 }, ""); + let word = if long { &day.1 } else { &day.0 }; + let input = words + .iter() + .filter(|w| *w != word) + .cloned() + .collect::>() + .join(" "); let hm = get_hours_and_minutes(&input); if hm.is_none() { return Err(String::from("Unable to find hours:minutes")); From 0372a1f4f7ed9f164390da53df6381ff1a47037a Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 20:59:58 +0200 Subject: [PATCH 45/51] Fix panic sweep: #2.1 signal-killed status panics, #2.4 bin constructor unwraps, #2.3 aws upload_folder (partial) - docker.rs/postgresql.rs/ssh.rs: match the Command::status() Result and format ExitStatus via Display instead of status.code().unwrap() - bin/bacup.rs: every remote/service constructor .unwrap() becomes a match that logs the error and returns Err(-1); upcoming().next() becomes a let-else on empty schedules - aws.rs upload_folder: min_by().unwrap() -> let-else error, strip_prefix().unwrap() -> match error --- BUGS.md | 276 ------------------------------------- src/bin/bacup.rs | 152 +++++++++++++------- src/remotes/aws.rs | 18 ++- src/remotes/ssh.rs | 2 +- src/services/docker.rs | 18 +-- src/services/postgresql.rs | 14 +- 6 files changed, 134 insertions(+), 346 deletions(-) delete mode 100644 BUGS.md diff --git a/BUGS.md b/BUGS.md deleted file mode 100644 index a5da005..0000000 --- a/BUGS.md +++ /dev/null @@ -1,276 +0,0 @@ -# bacup code review — bug report - -Full-repo review of all 18 files under `src/` (2026-10-04), re-audited -2026-10-05 against the post-fix tree (commits 3fec877, 17c8d90, a631331, -09e12ff, ca34f22). Findings ordered by severity: data-corruption / wrong-data -bugs first, then panic hazards, then functional oddities. Items marked -[FIXED] were verified against the current tree and the fix batch. - -**Second pass found a latent cleanup hole in #1.8 (the `cleanup()` local-file -deletion branch is dead code). Note: this is *not* an active disk leak — -normal runs delete the local dump via `Dump::drop`, verified on gtr7.** - -## 1. Data-corruption / wrong-data bugs - -### 1.1 [FIXED] `src/services/postgresql.rs` `dump()` — pg_dump exit status ignored - -The `match` on `.status().await` returns `Ok(Dump)` for *any* successful spawn, -discarding the `ExitStatus`. A non-zero `pg_dump` exit (permissions, lock wait, -OOM) is reported as a successful backup and the empty/truncated `.sql` gets -uploaded. **Fix:** check `status.code()` and return an error on non-zero. - -### 1.2 `src/remotes/git.rs` `upload_file` — file nested one level too deep - -`dest = repo.join(remote_path)` is the *file* path; the code does -`create_dir_all(dest)` then copies to `dest.join(file_name)`, so the first -upload creates `repo//mydb.sql/mydb.sql`. Every later run reuses the -directory, so the wrong nesting is stable. -**Fix:** `create_dir_all(dest.parent()?)` + `copy(path, dest)`. - -### 1.3 `src/remotes/git.rs` `upload_folder` — directory tree flattened - -Every file is copied to `dest.join(file_name)` regardless of its subdirectory, -so nested structures are lost and name collisions are silently overwritten. -**Fix:** preserve relative paths, like the localhost/aws remotes do. - -### 1.4 `src/remotes/ssh.rs` `upload_folder` — only one directory synced when the glob matches several - -`local_prefix = paths.min().parent()`. With a pattern like `/home/u/*/data` -(or any glob spanning sibling directories) rsync only sends the -lexicographically-first directory; the rest are silently skipped while -`--delete` keeps the remote "consistent" with that subset. -**Fix:** resolve the common ancestor of all matched paths (or require a single -root and error otherwise). - -### 1.5 [FIXED] `src/remotes/ssh.rs` `enumerate` / `delete` — unquoted remote paths - -`find {}/*` and `rm -r {}` interpolate `remote_path` raw, while `upload_file` -(correctly) uses the `shell_quote` helper. A path with spaces breaks `find` and -makes `rm -r` target the wrong thing — `delete` is the most dangerous one. -**Fix:** use the existing `shell_quote` helper in both commands. - -### 1.6 `src/remotes/ssh.rs` `upload_file` — no atomic upload; partial files on failure - -`cat > file` writes directly to the timestamped name. A network drop or -remote-disk-full mid-transfer leaves a truncated "backup" that later runs -treat as valid. **Fix:** upload to `name.tmp` and `mv` into place on success -(same for the compressed variant). - -### 1.7 [FIXED] `src/remotes/aws.rs` `enumerate` — `list_objects_v2` not paginated - -Only the first 1000 keys are ever seen, so `keep_last` pruning silently stops -working beyond 1000 objects per prefix. -**Fix:** loop on `continuation_token` / `is_truncated`. - -### 1.8 [FIXED] `src/services/zfs.rs` `cleanup()` — local-file deletion branch was dead code - -`do_dump` calls `self.cleanup(&snapshots, &new_name)` where `new_name` is the -ZFS **snapshot** name (`dataset@base-full-`, no `.snapshot` suffix), but -`cleanup` → `old_dump_files` → `parse_snapshot_name` does -`strip_suffix(".snapshot")?` on that argument, which fails, so -`old_dump_files` always returns `vec![]` and the `fs::remove_file` loop never -runs. The test `old_dump_files_are_strictly_older` passes a *file* name as -`newer_name`, masking the mismatch. - -**This is *not* an active disk leak.** The primary cleanup path is -`Dump::drop` (`src/services/service.rs`), which removes the dump file when -the `Dump` value goes out of scope at the end of the scheduled job in -`backup.rs`. Verified in production on gtr7: the current -`/storage/zroot-full-20261005-100000.snapshot` was already gone from the -working directory after the upload, and no older `*.snapshot` files -accumulated. `cleanup()`'s file deletion is only a redundant second layer. - -The real (narrow) leak window: if the process crashes (panic / SIGKILL / -OOM) between `zfs send` completing and `Drop` running, the raw stream is -left in the working directory and nothing will ever remove it — full dumps -are raw `zfs send` streams, so one stale file can be large. - -**Fix:** just delete the dead code. `Dump::drop` is the real cleanup path -and it works (verified in production), so the file-deletion layer in -`cleanup()` — the `fs::read_dir` + `old_dump_files` + `fs::remove_file` -loop — plus `old_dump_files` itself and its test -`old_dump_files_are_strictly_older` should be removed. Keep the rest of -`cleanup()` (`cleanup_list` + `destroy`), which is the live ZFS snapshot -cleanup. Optionally note in `Drop`'s doc that a crash between `send` and -drop can leave a stale file behind. - -**Applied:** removed the `fs::read_dir` / `old_dump_files` / `fs::remove_file` -block from `cleanup()`, deleted `old_dump_files` and its test, updated the -`cleanup()` doc to point at `Dump::drop`, and documented the crash-orphan -caveat in `Drop`'s doc in `service.rs`. Full test suite passes (48 + 6 ok). - -## 2. Panic hazards (crash instead of `Err`) - -### 2.1 `status.code().unwrap()` on signal-killed processes - -`docker.rs` (`new` + `dump`), `postgresql.rs` `new`, `ssh.rs` `new` (error -path). A SIGKILLed docker/pg_isready/ssh panics the whole scheduler instead of -returning an error. `status.code()` is `None` for signal-killed processes. - -### 2.2 [FIXED] `String::from_utf8(...).unwrap()` on command output - -`ssh.rs` `new`, `postgresql.rs` `new` (both stdout and stderr). Non-UTF8 -output (e.g. a broken locale message) panics startup. - -**Applied:** command output is best-effort diagnostic text (locale-dependent -remote banners/messages), so both `ssh.rs` and `postgresql.rs` now decode it -with `String::from_utf8_lossy` (the postgresql half landed as a drive-by in -#3.7, the ssh half directly). This is the only place lossy is *required* by -the strict-vs-lossy rule — the strict-text sites (`config.rs` TOML load, -private key load) already return `Err` on non-UTF8 via `fs::read_to_string`, -and the aws binary path keeps `Vec` with no text assumption. - -### 2.3 `.unwrap()` on collections that can be empty / non-UTF8 - -- `aws.rs` + `ssh.rs` `upload_folder`: `paths.iter().min_by(...).unwrap()` (empty paths) -- `remote.rs` `compress_folder`: `path.file_name().unwrap()` (root-level path) -- `git.rs`: `remote_path.strip_prefix("/").unwrap()` (empty remote_path) -- `remote.rs` / `postgresql.rs`: `std::env::current_dir().unwrap()` - -### 2.4 `src/bin/bacup.rs` - -Every service/remote constructor call ends in `.unwrap()` (including -`Backup::new(...).await.unwrap()`), so one misconfigured service kills the -entire process with a backtrace instead of the clean `error!` + -`return Err(-1)` style used by the keep_last gate. - -### 2.5 [FIXED] `src/backup.rs` scheduled job — `.unwrap()` on empty file list - -`local_files.iter().min_by(...).unwrap()` in the cron job panicked when -`service.list()` returned nothing (e.g. a folder glob that matches no files). -`Backup::new` uses `if let Some(...)`, so only the scheduled path was exposed. -Fixed with an early `error!("...no files, nothing to back up") + return` guard -right after `service.list()` in the job. - -## 3. Functional oddities (lower severity) - -### 3.1 [FIXED] `src/remotes/git.rs` `clone_repository` — clone dir is `/` - -Two git remotes whose repos share a basename silently share the first repo's -clone (the `dest.exists()` shortcut). Also, a failed `git pull` (diverging -shallow-clone branch) leaves conflict markers in the worktree, and -`git add . -A` commits them as a "backup" — pull failure output is ignored. - -### 3.2 [FIXED] `src/remotes/git.rs` `upload_file_compressed` — deferred cleanup targets the wrong path - -The deferred `fs::remove_file(&remote_path)` operates on the *remote* path -string (e.g. `/backups/x.sql-YYYY-MM-DD-HH.MM.gz`) as if it were a local path: -silently no-ops, `#[must_use]` result ignored. It should remove the local temp -file under the repo dir. - -### 3.3 [FIXED] `src/remotes/git.rs` push-failure error message - -Says "Unable to execute git add . -A" — copy-paste from the add step; the -failing command is `git push`. - -### 3.4 [FIXED] `src/remotes/ssh.rs` encrypted-key detection - -Matches only legacy PEM (`Proc-Type` + `ENCRYPTED`). Modern OpenSSH-format -encrypted keys are not flagged, so users get a confusing auth failure instead -of the "key is encrypted" hint. - -Fix: added `openssh_key_is_encrypted()` in src/remotes/ssh.rs — base64-decodes -the `-----BEGIN OPENSSH PRIVATE KEY-----` blob, validates the `openssh-key-v1` -magic, and checks the cipher name field against `"none"`; `Ssh::new` now rejects -passphrase-protected OpenSSH-format keys with the same `InvalidPrivateKey` -error as the legacy path. Added the `base64` crate (0.22.1) dependency. -4 new unit tests (unencrypted/encrypted OpenSSH keys, legacy PEM out of scope, -malformed blocks). - -### 3.5 [FIXED] `src/remotes/remote.rs` archive names have minute precision - -`%H.%M` means two folder/file backups finishing within the same minute produce -the same remote name; the second overwrites the first. (zfs dumps avoid this: -one file per run with second precision.) - -### 3.6 [FIXED] `src/services/docker.rs` `new` — connectivity check pulls an image - -Runs `docker run --rm hello-world` on every startup, downloading the -hello-world image each time and requiring network. - -Fix: `Docker::new` now runs `docker info` instead — validates the daemon is -running and reachable with no image pull and no network. - -### 3.7 [FIXED] `src/services/postgresql.rs` misc - -- db-existence check builds SQL by interpolation (`datname='{db_name}'`), so a - quote in the configured name breaks/mangles the query (low severity: config - is user-managed). -- The psql connection check treats *any* non-empty stderr as failure, so - benign locale warnings fail startup. -- No password field in config: `pg_dump` runs with `--no-password`, so - authentication relies entirely on `~/.pgpass`. - -Fix: -- single quotes in `db_name` are SQL-doubled (`'` → two single quotes) before - the name is interpolated into the `datname` query. -- The check now fails on `!status.success()` (exit status), not on any - non-empty stderr; stdout is expected to be exactly `"1"`. -- `PostgreSqlConfig` (and the `PostgreSql` service) gained - `password: Option` (serde `#[default]`, existing configs unaffected); - when set, both `psql` and `pg_dump` get it via the `PGPASSWORD` env var. -- psql stdout/stderr now read with `String::from_utf8_lossy` instead of - `str::from_utf8`. (6 existing test literals updated with `password: None`.) - -### 3.8 [FIXED] `src/remotes/aws.rs` `put_object` memory spike - -Buffers whole files ≤ 1024 MiB entirely into a `Vec` before upload — a 1 GB -dump spikes RSS by 1 GB unnecessarily. A streaming body would fix it. - -Fix: `put_object` no longer reads the file into a `Vec` — the small-file path -now uses `ByteStream::from_path(path)`, which streams the file from disk and -sets `Content-Length` from file metadata (aws-smithy-types 1.8.1 has no -`ByteStream::from(tokio::fs::File)`; the earlier note described a planned -approach, not what shipped). `CHUNK_SIZE` now only decides put-vs-multipart; -the unused `tokio::fs::File` and `AsyncReadExt` imports are gone. - -### 3.9 [FIXED] `src/bin/bacup.rs` main loop — tick errors invisible - -The `scheduler.tick()` error check is commented out and replaced with a bare -`sleep(50ms)` loop, so scheduler tick failures are never surfaced. - -### 3.10 [FIXED] Typos in user-visible messages / comments - -- `remote.rs`: "Storagee error" (should be "Storage") -- `ssh.rs`: "succeded" -- `postgresql.rs`: "does not exit" (→ "exist"); comments: "shuld be - performend", "aksing" - -### 3.11 [RESOLVED BY #3.5] `src/remotes/remote.rs` `remote_compressed_file_path` — space in timestamp - -Re-checked after the 3.5 fix: both `remote_archive_path` and -`remote_compressed_file_path` now build names with -`now.format("%Y-%m-%d-%H.%M.%S")` (no space, seconds precision) — the -`ts.replace(...)` code no longer exists. No action needed. - -### 3.12 [FIXED] `src/when.rs` `parse_weekly` — `.contains()` day matching is a substring scan - -`day.to_lowercase().contains(day_name)` matches substrings, not whole words -(no two day names are substrings of each other, and the HH:MM regex rejects -the leftovers, so no practical misfire — but `"tuesday 10:00"` containing -`"tue"`-style input is not a supported format anyway). Cosmetic; note only. - -Fix: `parse_weekly` now matches day names as whole words (whitespace-token -membership) and removes only the matched word when consuming the input, -instead of `.contains()`/`.replace()` substring scans. All existing when -tests still pass unchanged. - -## 4. What's solid - -- `zfs.rs` full+incremental chain logic, keep_last gate, delegated-permission - check, compression nudge, and the `when`/cron `full_when` resolution all - check out (the dead local-file cleanup in `cleanup()` was removed — see - #1.8; the real cleanup is `Dump::drop`). -- `when.rs` (shared when-format module, fixed dow mapping + regression test), - `folders.rs`, `localhost.rs`, `service.rs`, `config.rs`, `lib.rs` -- The keep_last startup gate, `what`/`where` validation, and the zfs - `required_keep_last` upper-bound simulation are correct (conservative by ≤ 1 - when `full_when` fires between backup runs — safe direction). - -## Suggested fix order (second pass, 2026-10-05) - -1. ~~#1.8 remove the dead local-file cleanup code in zfs `cleanup()`~~ [DONE] -2. #2.1–2.4 panic sweep (all are `unwrap()` on signal/utf8/empty inputs) -3. #1.2 / #1.3 git nesting + flattening -4. #1.4 ssh multi-dir glob, #1.6 ssh atomic upload -5. ~~#3.4, #3.6, #3.7, #3.8, #3.12~~ [DONE — applied and verified with `cargo build` + `cargo test` (58 passed, 2 ignored)] diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index b1167ee..6180229 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -119,11 +119,16 @@ async fn main() -> Result<(), i32> { match config.aws { Some(aws) => { for (bucket_name, bucket_config) in aws { - remotes.insert( - format!("aws.{}", bucket_name), - Box::new(AwsBucket::new(bucket_config, &bucket_name).await.unwrap()), - ); - info!("Remote aws.{} configured", bucket_name); + match AwsBucket::new(bucket_config, &bucket_name).await { + Ok(remote) => { + remotes.insert(format!("aws.{}", bucket_name), Box::new(remote)); + info!("Remote aws.{} configured", bucket_name); + } + Err(error) => { + error!("Failed to configure remote aws.{}: {:?}", bucket_name, error); + return Err(-1); + } + } } } None => warn!("No AWS cloud configured."), @@ -132,11 +137,16 @@ async fn main() -> Result<(), i32> { match config.ssh { Some(host) => { for (hostname, config) in host { - remotes.insert( - format!("ssh.{}", hostname), - Box::new(Ssh::new(config, &hostname).await.unwrap()), - ); - info!("Remote ssh.{} configured", hostname); + match Ssh::new(config, &hostname).await { + Ok(remote) => { + remotes.insert(format!("ssh.{}", hostname), Box::new(remote)); + info!("Remote ssh.{} configured", hostname); + } + Err(error) => { + error!("Failed to configure remote ssh.{}: {}", hostname, error); + return Err(-1); + } + } } } None => warn!("No Ssh remotes configured."), @@ -145,11 +155,16 @@ async fn main() -> Result<(), i32> { match config.localhost { Some(host) => { for (name, config) in host { - remotes.insert( - format!("localhost.{}", name), - Box::new(Localhost::new(config, &name).unwrap()), - ); - info!("Remote localhost.{} configured", name); + match Localhost::new(config, &name) { + Ok(remote) => { + remotes.insert(format!("localhost.{}", name), Box::new(remote)); + info!("Remote localhost.{} configured", name); + } + Err(error) => { + error!("Failed to configure remote localhost.{}: {}", name, error); + return Err(-1); + } + } } } None => warn!("No localhost remotes configured."), @@ -158,11 +173,16 @@ async fn main() -> Result<(), i32> { match config.git { Some(host) => { for (name, config) in host { - remotes.insert( - format!("git.{}", name), - Box::new(Git::new(config, &name).await.unwrap()), - ); - info!("Remote git.{} configured", name); + match Git::new(config, &name).await { + Ok(remote) => { + remotes.insert(format!("git.{}", name), Box::new(remote)); + info!("Remote git.{} configured", name); + } + Err(error) => { + error!("Failed to configure remote git.{}: {}", name, error); + return Err(-1); + } + } } } None => warn!("No Git remotes configured."), @@ -183,7 +203,15 @@ async fn main() -> Result<(), i32> { Some(folders) => { for (location_name, folder) in folders { let key = format!("folders.{}", location_name); - services.insert(key, Box::new(Folder::new(&folder.pattern).await.unwrap())); + match Folder::new(&folder.pattern).await { + Ok(service) => { + services.insert(key, Box::new(service)); + } + Err(error) => { + error!("Failed to configure service folders.{}: {}", location_name, error); + return Err(-1); + } + } } } None => warn!("No folders to backup."), @@ -192,14 +220,15 @@ async fn main() -> Result<(), i32> { Some(postgres) => { for (service_name, instance_config) in postgres { let key = format!("postgres.{}", service_name); - services.insert( - key, - Box::new( - PostgreSql::new(instance_config, &service_name) - .await - .unwrap(), - ), - ); + match PostgreSql::new(instance_config, &service_name).await { + Ok(service) => { + services.insert(key, Box::new(service)); + } + Err(error) => { + error!("Failed to configure service postgres.{}: {}", service_name, error); + return Err(-1); + } + } } } None => warn!("No PostgreSql to backup."), @@ -208,10 +237,15 @@ async fn main() -> Result<(), i32> { Some(docker) => { for (service_name, instance_config) in docker { let key = format!("docker.{}", service_name); - services.insert( - key, - Box::new(Docker::new(instance_config, &service_name).await.unwrap()), - ); + match Docker::new(instance_config, &service_name).await { + Ok(service) => { + services.insert(key, Box::new(service)); + } + Err(error) => { + error!("Failed to configure service docker.{}: {}", service_name, error); + return Err(-1); + } + } } } None => warn!("No Docker to backup."), @@ -222,9 +256,17 @@ async fn main() -> Result<(), i32> { Some(zfs) => { for (service_name, instance_config) in zfs { let key = format!("zfs.{}", service_name); - let service = Zfs::new(&instance_config, &service_name).await.unwrap(); - zfs_compression.insert(service_name, service.compression().to_string()); - services.insert(key, Box::new(service)); + match Zfs::new(&instance_config, &service_name).await { + Ok(service) => { + zfs_compression.insert(service_name, service.compression().to_string()); + services.insert(key, Box::new(service)); + } + + Err(error) => { + error!("Failed to configure service zfs.{}: {}", service_name, error); + return Err(-1); + } + } } } None => warn!("No Zfs to backup."), @@ -252,19 +294,21 @@ async fn main() -> Result<(), i32> { return Err(-1); } - backup.insert( - backup_name.clone(), - Arc::new( - Backup::new( - &backup_name, - dyn_clone::clone_box(&*remotes[&config.r#where]), - dyn_clone::clone_box(&*services[&config.what]), - &config, - ) - .await - .unwrap(), - ), - ); + let backup_entry = match Backup::new( + &backup_name, + dyn_clone::clone_box(&*remotes[&config.r#where]), + dyn_clone::clone_box(&*services[&config.what]), + &config, + ) + .await + { + Ok(backup) => Arc::new(backup), + Err(error) => { + error!("Failed to configure backup {}: {}", backup_name, error); + return Err(-1); + } + }; + backup.insert(backup_name.clone(), backup_entry); // The dump is produced with `zfs send -c -L`: on a compressed dataset // the stream is already compressed, so gzip at upload is wasted CPU. if let Some(compression) = config @@ -289,11 +333,17 @@ async fn main() -> Result<(), i32> { info!("Backup {} -> {} configured", config.what, config.r#where); } - let mut scheduler = JobScheduler::new().await.unwrap(); + let Ok(mut scheduler) = JobScheduler::new().await else { + error!("Unable to create the job scheduler"); + return Err(-1); + }; // scheduler.shutdown_on_ctrl_c(); for (name, job) in backup { - let upcoming = job.schedule.upcoming(chrono::Utc).take(1).next().unwrap(); + let Some(upcoming) = job.schedule.upcoming(chrono::Utc).take(1).next() else { + error!("Backup {}: no upcoming run in schedule", name); + return Err(-1); + }; let schedule = job.schedule.clone(); let res = job.schedule(&mut scheduler, schedule).await; diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 0bb5092..9bc3321 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -395,7 +395,11 @@ impl remote::Remote for AwsBucket { ) -> Result<(), remote::Error> { let tot = paths.len(); - let mut local_prefix = paths.iter().min_by(|a, b| a.cmp(b)).unwrap(); + let Some(mut local_prefix) = paths.iter().min_by(|a, b| a.cmp(b)) else { + return Err(remote::Error::LocalError(io::Error::other( + "no paths to upload", + ))); + }; // The local_prefix found is the shortest path inside the folder we want to backup. // If it is a folder, we of course don't want to consider this a prefix, but its parent. @@ -409,7 +413,17 @@ impl remote::Remote for AwsBucket { // Strip local prefix from remote paths let mut remote_paths: Vec = Vec::with_capacity(tot); for path in paths.iter() { - remote_paths.push(remote_path.join(path.strip_prefix(local_prefix).unwrap())); + match path.strip_prefix(local_prefix) { + Ok(stripped) => remote_paths.push(remote_path.join(stripped)), + Err(error) => { + return Err(remote::Error::LocalError(io::Error::other(format!( + "path {} is not under prefix {}: {}", + path.display(), + local_prefix.display(), + error + )))) + } + } } // Upload all the files one by one diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index 9bd184c..a21b075 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -215,7 +215,7 @@ impl Ssh { config.username, config.host, config.port, - status.code().unwrap(), + status, )))); } } diff --git a/src/services/docker.rs b/src/services/docker.rs index 3d46cf3..3c48397 100644 --- a/src/services/docker.rs +++ b/src/services/docker.rs @@ -67,14 +67,14 @@ impl Docker { .stdout(Stdio::null()) .status() .await; - if status.is_err() { - return Err(Error::RuntimeError(status.err().unwrap())); - } - let code = status.unwrap().code().unwrap(); - if code != 0 { + let status = match status { + Err(error) => return Err(Error::RuntimeError(error)), + Ok(status) => status, + }; + if !status.success() { return Err(Error::RuntimeError(io::Error::other(format!( - "docker info failed (is the docker daemon running and accessible?), exit code {}", - code + "docker info failed (is the docker daemon running and accessible?), {}", + status )))); } @@ -138,10 +138,10 @@ impl Service for Docker { match status.success() { true => Ok(Dump { path: Some(dest) }), false => Err(Error::RuntimeError(io::Error::other(format!( - "{} {:?} failed with exit code {}", + "{} {:?} failed, {}", self.cmd.display(), self.args, - status.code().unwrap() + status ))) .into()), } diff --git a/src/services/postgresql.rs b/src/services/postgresql.rs index 6691e3e..c63ade0 100644 --- a/src/services/postgresql.rs +++ b/src/services/postgresql.rs @@ -78,14 +78,14 @@ impl PostgreSql { .stdout(Stdio::null()) .status() .await; - if status.is_err() { - return Err(Error::RuntimeError(status.err().unwrap())); - } - let code = status.unwrap().code().unwrap(); - if code != 0 { + let status = match status { + Err(error) => return Err(Error::RuntimeError(error)), + Ok(status) => status, + }; + if !status.success() { return Err(Error::RuntimeError(io::Error::other(format!( - "pg_isready failed, exit code {}", - code + "pg_isready failed, {}", + status )))); } From 7ec641cad46a67533bbf98b8e30b1a5d0661c468 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Mon, 5 Oct 2026 21:02:46 +0200 Subject: [PATCH 46/51] Ignore BUGS.md (local notes) --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index fe2aedf..87d5496 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ # Local configuration file used during development config.toml +# Local bug-tracking notes +BUGS.md + # IDE stuff .vscode/ From c29adb9d6ca11fb6aed9d1c46555404e82acec42 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Tue, 6 Oct 2026 07:08:51 +0200 Subject: [PATCH 47/51] Reject unknown config keys and make zfs full/incremental selection visible A full_when placed under [backup.] instead of [zfs.] was silently dropped by serde, so every zfs run took a full backup (and the keep_last restorability check was skipped). - config: deny_unknown_fields on every config struct, so a misplaced or misspelled key fails at startup - zfs: log full_when (or its absence) when the service is configured - zfs: log why a run takes a full backup - zfs: a failing `zfs list` is now an error instead of an empty snapshot list, which silently forced a full - zfs: the incremental->full fallback dump is named -full-, not -inc- --- src/config.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++ src/services/zfs.rs | 49 +++++++++++++++++++++++++++++-------- 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/config.rs b/src/config.rs index 73b8f4a..2cb5a4f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,6 +21,7 @@ use std::fmt; use tokio::{fs, io}; #[derive(Serialize, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct GitConfig { pub host: String, pub port: u16, @@ -31,6 +32,7 @@ pub struct GitConfig { } #[derive(Serialize, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct SshConfig { pub host: String, pub port: u16, @@ -39,6 +41,7 @@ pub struct SshConfig { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AwsConfig { pub region: String, pub endpoint: Option, @@ -48,11 +51,13 @@ pub struct AwsConfig { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct GCloudConfig { pub service_account_path: String, } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct PostgreSqlConfig { pub username: String, pub db_name: String, @@ -66,12 +71,14 @@ pub struct PostgreSqlConfig { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct DockerConfig { pub container_name: String, pub command: String, } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ZfsConfig { pub dataset: String, pub snapshot_name: String, @@ -86,11 +93,13 @@ pub struct ZfsConfig { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct FoldersConfig { pub pattern: String, } #[derive(Serialize, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct BackupConfig { pub what: String, pub r#where: String, @@ -101,11 +110,13 @@ pub struct BackupConfig { } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct LocalhostConfig { pub path: String, } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Config { // remotes pub aws: Option>, @@ -157,3 +168,51 @@ impl Config { Ok(config) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn misplaced_full_when_is_rejected() { + // full_when belongs to [zfs.], not to [backup.]: an + // unknown key must fail the parse instead of being silently ignored. + let txt = r#" + [zfs.storage] + dataset = "storage" + snapshot_name = "storage-snap" + + [backup.storage] + what = "zfs.storage" + where = "aws.bucket" + when = "daily 22:00" + remote_path = "/zfs/storage/" + compress = false + full_when = "monthly 1 22:00" + "#; + let err = toml::from_str::(txt) + .err() + .expect("unknown key rejected"); + assert!(err.to_string().contains("full_when"), "{}", err); + } + + #[test] + fn full_when_in_zfs_section_is_accepted() { + let txt = r#" + [zfs.storage] + dataset = "storage" + snapshot_name = "storage-snap" + full_when = "monthly 1 22:00" + + [backup.storage] + what = "zfs.storage" + where = "aws.bucket" + when = "daily 22:00" + remote_path = "/zfs/storage/" + compress = false + "#; + let config: Config = toml::from_str(txt).unwrap(); + let zfs = config.zfs.unwrap(); + assert_eq!(zfs["storage"].full_when.as_deref(), Some("monthly 1 22:00")); + } +} diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 171326a..1817a36 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -388,6 +388,10 @@ impl Zfs { .to_string(); debug!("new zfs service on {dataset} (snapshot base {snapshot_base})"); + match &config.full_when { + Some(expr) => info!("zfs.{name}: full backups when '{expr}', incrementals in between"), + None => info!("zfs.{name}: full_when not set, every run takes a full backup"), + } Ok(Zfs { name: name.to_string(), cmd, @@ -407,8 +411,12 @@ impl Zfs { /// List all snapshots of our chain on the dataset tree, including those /// on child datasets (created with `-r`). - async fn list_our_snapshots(&self) -> Vec { - let output = match Command::new(&self.cmd) + /// + /// A failing `zfs list` is an error: treating it as "no snapshots" would + /// silently turn the run into a full backup and destroy nothing, breaking + /// the expected full/incremental cadence without notice. + async fn list_our_snapshots(&self) -> Result, Error> { + let output = Command::new(&self.cmd) .arg("list") .arg("-r") .arg("-H") @@ -418,16 +426,20 @@ impl Zfs { .arg("snapshot") .arg(&self.dataset) .output() - .await - { - Ok(output) if output.status.success() => output, - _ => return vec![], - }; - String::from_utf8_lossy(&output.stdout) + .await?; + if !output.status.success() { + return Err(Error::String(format!( + "zfs list snapshots of {} failed (exit {}): {}", + self.dataset, + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(String::from_utf8_lossy(&output.stdout) .lines() .map(|line| line.trim().to_string()) .filter_map(|line| parse_snapshot(&line, &self.snapshot_base)) - .collect() + .collect()) } async fn create_snapshot(&self, name: &str) -> Result<(), Error> { @@ -493,9 +505,19 @@ impl Zfs { let now = Utc::now(); let ts = format_ts(now); - let snapshots = self.list_our_snapshots().await; + let snapshots = self.list_our_snapshots().await?; let plan = select_plan(now, self.full_when.as_ref(), &snapshots); let is_full = plan[0] == "full"; + if is_full { + let reason = if self.full_when.is_none() { + "full_when is not set" + } else if latest_of_kind(&snapshots, Some(SnapshotKind::Full)).is_none() { + "no previous full snapshot of the chain exists" + } else { + "the full_when schedule is due" + }; + info!("full backup selected for {}: {reason}", self.dataset); + } let kind: SnapshotKind = if is_full { SnapshotKind::Full } else { @@ -545,10 +567,17 @@ impl Zfs { } let full_name = format!("{}@{}-{FULL_MARK}-{ts}", self.dataset, self.snapshot_base); self.create_snapshot(&full_name).await?; + // The fallback is a full stream: name the dump file as such, + // or a restore would treat it as an incremental. + let _ = fs::remove_file(&dest).await; + let dest = dump_file_name(&self.name, SnapshotKind::Full, &ts); let send = send_args(SnapshotKind::Full, None, &full_name); self.send(&send, &dest).await?; info!("ZFS full checkpoint completed successfully (incremental fallback)"); self.cleanup(&snapshots, &full_name).await; + return Ok(Dump { + path: Some(dest.into()), + }); } else { info!("ZFS incremental checkpoint completed successfully (base {base})"); } From 5f170d3343c9779328a6a8f1adfcb6b212ed8f6b Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Tue, 6 Oct 2026 07:16:47 +0200 Subject: [PATCH 48/51] fmt + clippy --- src/bin/bacup.rs | 25 ++++++++++++++++++++----- src/remotes/aws.rs | 2 +- src/remotes/ssh.rs | 11 +++-------- src/services/postgresql.rs | 11 ++++++----- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/bin/bacup.rs b/src/bin/bacup.rs index 6180229..45d8738 100644 --- a/src/bin/bacup.rs +++ b/src/bin/bacup.rs @@ -125,7 +125,10 @@ async fn main() -> Result<(), i32> { info!("Remote aws.{} configured", bucket_name); } Err(error) => { - error!("Failed to configure remote aws.{}: {:?}", bucket_name, error); + error!( + "Failed to configure remote aws.{}: {:?}", + bucket_name, error + ); return Err(-1); } } @@ -208,7 +211,10 @@ async fn main() -> Result<(), i32> { services.insert(key, Box::new(service)); } Err(error) => { - error!("Failed to configure service folders.{}: {}", location_name, error); + error!( + "Failed to configure service folders.{}: {}", + location_name, error + ); return Err(-1); } } @@ -225,7 +231,10 @@ async fn main() -> Result<(), i32> { services.insert(key, Box::new(service)); } Err(error) => { - error!("Failed to configure service postgres.{}: {}", service_name, error); + error!( + "Failed to configure service postgres.{}: {}", + service_name, error + ); return Err(-1); } } @@ -242,7 +251,10 @@ async fn main() -> Result<(), i32> { services.insert(key, Box::new(service)); } Err(error) => { - error!("Failed to configure service docker.{}: {}", service_name, error); + error!( + "Failed to configure service docker.{}: {}", + service_name, error + ); return Err(-1); } } @@ -263,7 +275,10 @@ async fn main() -> Result<(), i32> { } Err(error) => { - error!("Failed to configure service zfs.{}: {}", service_name, error); + error!( + "Failed to configure service zfs.{}: {}", + service_name, error + ); return Err(-1); } } diff --git a/src/remotes/aws.rs b/src/remotes/aws.rs index 9bc3321..803e96c 100644 --- a/src/remotes/aws.rs +++ b/src/remotes/aws.rs @@ -120,7 +120,7 @@ impl Bucket { // ByteStream::from_path sets Content-Length from file metadata. let body = ByteStream::from_path(path) .await - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + .map_err(std::io::Error::other)?; let response = self .client .put_object() diff --git a/src/remotes/ssh.rs b/src/remotes/ssh.rs index a21b075..efd1f06 100644 --- a/src/remotes/ssh.rs +++ b/src/remotes/ssh.rs @@ -87,8 +87,7 @@ fn openssh_key_is_encrypted(key: &str) -> bool { .chars() .filter(|c| c.is_ascii_alphanumeric() || *c == '+' || *c == '/') .collect(); - let Ok(bytes) = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64.as_bytes()) - else { + let Ok(bytes) = base64::engine::general_purpose::STANDARD_NO_PAD.decode(b64.as_bytes()) else { return false; }; // openssh-key-v1 layout: uint32 len, magic, uint32 len, ciphername, ... @@ -212,10 +211,7 @@ impl Ssh { if !status.success() { return Err(Error::RuntimeError(io::Error::other(format!( "ssh connection to {}@{}:{} failed with status: {}", - config.username, - config.host, - config.port, - status, + config.username, config.host, config.port, status, )))); } } @@ -547,8 +543,7 @@ mod tests { let garbage = "-----BEGIN OPENSSH PRIVATE KEY-----\n!!!!not-base64!!!!\n-----END OPENSSH PRIVATE KEY-----\n"; assert!(!openssh_key_is_encrypted(garbage)); // Valid base64, but not an openssh-key-v1 blob. - let encoded = - base64::engine::general_purpose::STANDARD_NO_PAD.encode(b"not a key at all"); + let encoded = base64::engine::general_purpose::STANDARD_NO_PAD.encode(b"not a key at all"); let not_a_key = format!( "-----BEGIN OPENSSH PRIVATE KEY-----\n{encoded}\n-----END OPENSSH PRIVATE KEY-----\n" ); diff --git a/src/services/postgresql.rs b/src/services/postgresql.rs index c63ade0..96d1cce 100644 --- a/src/services/postgresql.rs +++ b/src/services/postgresql.rs @@ -122,7 +122,11 @@ impl PostgreSql { if !output.status.success() { return Err(Error::RuntimeError(io::Error::other(format!( "psql connection check failed: {}", - if stderr.is_empty() { "unknown error".to_string() } else { stderr } + if stderr.is_empty() { + "unknown error".to_string() + } else { + stderr + } )))); } @@ -192,10 +196,7 @@ impl Service for PostgreSql { if let Some(password) = &self.password { command.env("PGPASSWORD", password); } - match command - .status() - .await - { + match command.status().await { Ok(status) if status.success() => Ok(Dump { path: Some(dest) }), Ok(status) => Err(Error::RuntimeError(io::Error::other(format!( "pg_dump exited with {} for database {}", From c33d8a8ca3a3c08a7361fb1380044a869e2d0440 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Tue, 6 Oct 2026 07:41:27 +0200 Subject: [PATCH 49/51] Prune zfs dumps by age, not by name, in keep_last --- src/backup.rs | 74 ++++++++++++++++++++++++++++++++++----------- src/services/zfs.rs | 56 ++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 17 deletions(-) diff --git a/src/backup.rs b/src/backup.rs index 56a4356..35e8f6d 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -15,6 +15,7 @@ use crate::config::BackupConfig; use crate::remotes::remote; use crate::services::service::Service; +use crate::services::zfs::retention_sort_key; use crate::when; use cron::Schedule; @@ -264,23 +265,17 @@ impl Backup { if let Some(to_keep) = keep_last { let to_keep = to_keep as usize; match remote.enumerate(remote_path.parent().unwrap()).await { - Ok(mut list) => { - if list.len() > to_keep { - list.sort(); - list.reverse(); - for delete_me in &list[to_keep..] { - if let Some(error) = remote - .delete(&PathBuf::from(delete_me)) - .await - .err() - { - error!( - "[{}] Error during delete of {}: {}", - name, delete_me, error - ); - } else { - info!("[{}] Deleted {}", name, delete_me); - } + Ok(list) => { + for delete_me in &to_prune(list, to_keep) { + if let Some(error) = + remote.delete(&PathBuf::from(delete_me)).await.err() + { + error!( + "[{}] Error during delete of {}: {}", + name, delete_me, error + ); + } else { + info!("[{}] Deleted {}", name, delete_me); } } } @@ -313,11 +308,56 @@ impl Backup { } } +/// The remote objects to delete so that only the `to_keep` newest remain. +/// Objects are ordered chronologically by [retention_sort_key], so zfs +/// `full`/`inc` dump files are pruned by age and not by kind. +fn to_prune(mut list: Vec, to_keep: usize) -> Vec { + if list.len() <= to_keep { + return vec![]; + } + list.sort_by_cached_key(|name| retention_sort_key(name)); + list.reverse(); + list.split_off(to_keep) +} + #[cfg(test)] mod tests { use super::*; use croner::Cron; + #[test] + fn to_prune_keeps_the_newest_by_name() { + let list = vec![ + "p/2026-10-03-22:00-db.gz".to_string(), + "p/2026-10-05-22:00-db.gz".to_string(), + "p/2026-10-04-22:00-db.gz".to_string(), + ]; + assert_eq!(to_prune(list.clone(), 2), vec!["p/2026-10-03-22:00-db.gz"]); + assert!(to_prune(list, 3).is_empty()); + } + + #[test] + fn to_prune_zfs_chain_across_a_new_full() { + // daily backups, monthly full on the 1st, keep_last = 32: the + // October chain (full on Oct 5 + daily incs) followed by the + // November full and a week of incs. + let file = |kind: &str, month: u32, day: u32| { + format!("gtr7/zfs/storage/storage-{kind}-2026{month:02}{day:02}-220000.snapshot") + }; + let mut list = vec![file("full", 10, 5)]; + list.extend((6..=31).map(|d| file("inc", 10, d))); + list.push(file("full", 11, 1)); + list.extend((2..=7).map(|d| file("inc", 11, d))); + + let pruned = to_prune(list.clone(), 32); + // Only the oldest objects go, and the November full is kept: + // the remote stays restorable from it. + assert_eq!(pruned, vec![file("inc", 10, 6), file("full", 10, 5)]); + let kept: Vec<_> = list.iter().filter(|f| !pruned.contains(f)).collect(); + assert!(kept.contains(&&file("full", 11, 1))); + assert_eq!(kept.len(), 32); + } + fn validate_cron_expression(when: &str) { let result = when::parse_when(when); assert!( diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 1817a36..91a880d 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -89,6 +89,36 @@ pub(crate) fn dump_file_name(name: &str, kind: SnapshotKind, ts: &str) -> String format!("{name}-{mark}-{ts}{SNAPSHOT_EXT}") } +/// Chronological sort key for a remote object name, used by `keep_last` +/// pruning. +/// +/// Remote names normally start with their timestamp, so they sort +/// chronologically as plain strings. zfs dump files do not: in +/// `-{full,inc}-.snapshot` the kind comes first, so every `inc` +/// sorts after every `full` regardless of time, and pruning by name would +/// delete the fulls the incrementals depend on. For zfs dump files the kind +/// is dropped from the key (`-.snapshot`); any other name is +/// returned unchanged. +pub fn retention_sort_key(remote_name: &str) -> String { + let (dir, file) = match remote_name.rfind('/') { + Some(i) => remote_name.split_at(i + 1), + None => ("", remote_name), + }; + let Some(stem) = file.strip_suffix(SNAPSHOT_EXT) else { + return remote_name.to_string(); + }; + for mark in [FULL_MARK, INC_MARK] { + let marker = format!("-{mark}-"); + if let Some(i) = stem.rfind(&marker) { + let (name, ts) = (&stem[..i], &stem[i + marker.len()..]); + if NaiveDateTime::parse_from_str(ts, TS_FORMAT).is_ok() { + return format!("{dir}{name}-{ts}{SNAPSHOT_EXT}"); + } + } + } + remote_name.to_string() +} + /// Arguments for the `zfs send` command to dump the given snapshot. /// A full snapshot is sent with `send -R -v -c -L`, an incremental one with /// `send -R -v -c -L -i ` where `` is the name of the snapshot it @@ -716,6 +746,32 @@ mod tests { } } + #[test] + fn retention_sort_key_ignores_the_kind() { + assert_eq!( + retention_sort_key("gtr7/zfs/storage/storage-full-20261005-220000.snapshot"), + "gtr7/zfs/storage/storage-20261005-220000.snapshot" + ); + assert_eq!( + retention_sort_key("storage-inc-20261006-220000.snapshot"), + "storage-20261006-220000.snapshot" + ); + // A service name containing a marker is left alone, only the last + // marker followed by a timestamp is dropped. + assert_eq!( + retention_sort_key("my-inc-pool-full-20261005-220000.snapshot"), + "my-inc-pool-20261005-220000.snapshot" + ); + // Non zfs names are unchanged. + for name in [ + "backups/2026-10-05-22:00-db.sql.gz", + "backups/storage-full-notatimestamp.snapshot", + "backups/storage-full-20261005-220000.gz", + ] { + assert_eq!(retention_sort_key(name), name); + } + } + #[test] fn parse_snapshot_new_style() { let s = parse_snapshot("tank@snap-full-20260101-010000", "snap").unwrap(); From 6f333cf696c0f9ddbb9dbb5368d04f30095a769f Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Tue, 6 Oct 2026 13:21:18 +0200 Subject: [PATCH 50/51] zfs: take the incremental base from the root dataset, dry-run the send at startup --- src/services/zfs.rs | 123 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 13 deletions(-) diff --git a/src/services/zfs.rs b/src/services/zfs.rs index 91a880d..b097282 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -204,6 +204,23 @@ fn latest_of_kind(snapshots: &[Snapshot], kind: Option) -> Option< .max_by(|a, b| a.time.cmp(&b.time)) } +/// The snapshots of `snapshots` taken on `dataset` itself, excluding those +/// of its child datasets. +/// +/// `zfs snapshot -r` gives every dataset of the tree a snapshot with the +/// same name and timestamp, so the plan (latest full, incremental base) must +/// be computed on the root dataset only: `zfs send -R -i ` +/// requires `` to be a snapshot of the same dataset as ``, and a +/// child's snapshot (e.g. `tank/data@snap-full-...` for `tank@snap-inc-...`) +/// makes the send fail. +pub(crate) fn root_snapshots(dataset: &str, snapshots: &[Snapshot]) -> Vec { + snapshots + .iter() + .filter(|s| s.name.split('@').next() == Some(dataset)) + .cloned() + .collect() +} + /// The names of the snapshots to destroy after a new full snapshot has been /// created and sent: every snapshot of our chain except the new full /// (which, being the newest, is the only one the incremental chain needs). @@ -422,14 +439,46 @@ impl Zfs { Some(expr) => info!("zfs.{name}: full backups when '{expr}', incrementals in between"), None => info!("zfs.{name}: full_when not set, every run takes a full backup"), } - Ok(Zfs { + let zfs = Zfs { name: name.to_string(), cmd, dataset: dataset.clone(), snapshot_base: snapshot_base.clone(), full_when, compression, - }) + }; + zfs.check_send().await?; + Ok(zfs) + } + + /// Dry-run (`zfs send -n`) the send the next run builds on, so that + /// unsupported flags, a broken chain or wrong arguments are reported at + /// startup instead of at the first scheduled run. + /// + /// With at least two snapshots of the chain on the root dataset, the + /// incremental between the last two is dry-run (same arguments as an + /// incremental run); with one, its full send. Without snapshots there is + /// nothing to check: the first run is a full. + async fn check_send(&self) -> Result<(), Error> { + let mut root = root_snapshots(&self.dataset, &self.list_our_snapshots().await?); + root.sort_by_key(|s| s.time); + let mut args = match root.as_slice() { + [] => return Ok(()), + [only] => send_args(SnapshotKind::Full, None, &only.name), + [.., prev, last] => send_args(SnapshotKind::Incremental, Some(&prev.name), &last.name), + }; + args.insert(1, "-n".into()); + let output = Command::new(&self.cmd).args(&args).output().await?; + if !output.status.success() { + return Err(Error::String(format!( + "dry run `zfs {}` failed (exit {}): {}", + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + debug!("dry run `zfs {}` succeeded", args.join(" ")); + Ok(()) } /// Value of the dataset's `compression` property (`off`, `zstd`, ...). @@ -501,13 +550,15 @@ impl Zfs { Ok(()) } - async fn destroy(&self, name: &str) -> bool { - match Command::new(&self.cmd) - .arg("destroy") - .arg(name) - .status() - .await - { + /// Destroy the snapshot `name`; with `recursive`, also the snapshots with + /// the same name on the child datasets (`zfs destroy -r`). + async fn destroy(&self, name: &str, recursive: bool) -> bool { + let mut cmd = Command::new(&self.cmd); + cmd.arg("destroy"); + if recursive { + cmd.arg("-r"); + } + match cmd.arg(name).status().await { Ok(status) => status.success(), Err(e) => { warn!("failed to run zfs destroy {name}: {e}"); @@ -536,12 +587,13 @@ impl Zfs { let ts = format_ts(now); let snapshots = self.list_our_snapshots().await?; - let plan = select_plan(now, self.full_when.as_ref(), &snapshots); + let root = root_snapshots(&self.dataset, &snapshots); + let plan = select_plan(now, self.full_when.as_ref(), &root); let is_full = plan[0] == "full"; if is_full { let reason = if self.full_when.is_none() { "full_when is not set" - } else if latest_of_kind(&snapshots, Some(SnapshotKind::Full)).is_none() { + } else if latest_of_kind(&root, Some(SnapshotKind::Full)).is_none() { "no previous full snapshot of the chain exists" } else { "the full_when schedule is due" @@ -590,7 +642,9 @@ impl Zfs { "incremental send from base {base} failed; \ destroying the incremental snapshot and retrying as a full backup" ); - if self.destroy(&new_name).await { + // The snapshot was taken with `-r`: destroy it on the whole + // tree, or the children keep orphan incremental snapshots. + if self.destroy(&new_name, true).await { info!("incremental snapshot {new_name} destroyed"); } else { warn!("failed to destroy incremental snapshot {new_name}"); @@ -692,7 +746,7 @@ impl Zfs { let to_destroy = cleanup_list(new_full_name, snapshots); let mut destroyed = 0; for name in &to_destroy { - if self.destroy(name).await { + if self.destroy(name, false).await { info!("destroyed old zfs snapshot {name}"); destroyed += 1; } else { @@ -746,6 +800,49 @@ mod tests { } } + #[test] + fn incremental_base_is_on_the_root_dataset() { + // `zfs list -r` lists the root and every child with the same + // snapshot name and time; the base must be the root's snapshot, + // whatever the listing order. + let names = [ + "zroot@zroot-snap-full-20261005-100000", + "zroot/data@zroot-snap-full-20261005-100000", + "zroot/data/home@zroot-snap-full-20261005-100000", + ]; + for rotation in 0..names.len() { + let mut ordered = names.to_vec(); + ordered.rotate_left(rotation); + let snaps: Vec<_> = ordered + .iter() + .filter_map(|n| parse_snapshot(n, "zroot-snap")) + .collect(); + let root = root_snapshots("zroot", &snaps); + assert_eq!(root.len(), 1); + let cron = parse_schedule("monthly 1 10:00").unwrap(); + let now = Utc.with_ymd_and_hms(2026, 10, 6, 10, 0, 0).unwrap(); + assert_eq!( + select_plan(now, Some(&cron), &root), + vec!["inc", "zroot@zroot-snap-full-20261005-100000"] + ); + } + } + + #[test] + fn root_snapshots_does_not_match_dataset_prefixes() { + let snaps: Vec<_> = [ + "tank@snap-full-20260101-010000", + "tank2@snap-full-20260101-010000", + "tank/data@snap-full-20260101-010000", + ] + .iter() + .filter_map(|n| parse_snapshot(n, "snap")) + .collect(); + let root = root_snapshots("tank", &snaps); + assert_eq!(root.len(), 1); + assert_eq!(root[0].name, "tank@snap-full-20260101-010000"); + } + #[test] fn retention_sort_key_ignores_the_kind() { assert_eq!( From 1917e64737d201a4c6a2cf510b79f07eb75b7796 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Wed, 7 Oct 2026 17:42:46 +0200 Subject: [PATCH 51/51] zfs: handle failed uploads and destroy old chain on new full --- README.md | 3 +- src/backup.rs | 15 +++- src/services/service.rs | 6 ++ src/services/zfs.rs | 188 ++++++++++++++++++++++++++++------------ 4 files changed, 153 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index a7da28a..15ad8b4 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,8 @@ The `zfs` service backs up a dataset tree with `zfs snapshot` + `zfs send`. Ever - Without `full_when` every run is a full backup. - With `full_when` set, a full backup is taken when the schedule is due (the first run is always a full) and the runs in between are **incrementals** against the latest existing snapshot of the chain, so they only contain what changed since the last run and stay small. - If an incremental can't be sent (e.g. a child dataset was destroyed and recreated), bacup destroys the incremental snapshot and retries the run as a full backup, so a run never fails silently. -- After each **full** backup the previous chain is replaced: all older snapshots of the service (including those on child datasets) and the older local dump files are destroyed. On the remote, dump files are pruned by `keep_last` as usual. +- If a dump can't be uploaded, bacup destroys the snapshot it was sent from, so the next run never builds an incremental on a stream the remote doesn't have: it takes an incremental against the previous snapshot instead (or a full, if the failed run was the full). A failed upload also skips the `keep_last` pruning for that run. +- Before each **full** backup the previous chain is replaced: all older snapshots of the service (including those on child datasets) are destroyed, so `zfs send -R` does not stream them inside the new full (snapshots taken by other tools on the same datasets are still included). If the full can't be sent, its snapshot is destroyed too and the next run takes a full again. Local dump files are removed after the upload; on the remote, dump files are pruned by `keep_last` as usual. - The dump is produced with `zfs send -c -L`: blocks that are compressed on disk stay compressed in the dump file (the `-c` flag needs OpenZFS >= 2.1.1 on the sender). If the dataset uses compression (check with `zfs get -o value compression `), set `compress = false` on the backup — gzip'ing an already-compressed stream at upload time is wasted CPU with no size gain. Keep `compress = true` only for uncompressed datasets. bacup logs a hint at startup when the dataset is compressed. ### Restoring diff --git a/src/backup.rs b/src/backup.rs index 35e8f6d..5246d0a 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -200,6 +200,9 @@ impl Backup { local_files = vec![PathBuf::from(local_prefix)]; } + // Set when any upload fails: the service is told after the uploads. + let mut upload_failed = false; + // Special case in which we want to upload a folder without compression // If all the files share the same prefix, we upload all the files in this prefix. // The remote should handle eventual incremental backup. @@ -211,6 +214,7 @@ impl Backup { remote_path.display() ); let result = remote.upload_folder(&local_files, remote_path).await; + upload_failed |= result.is_err(); Backup::log_result( result, &name, @@ -261,8 +265,11 @@ impl Backup { remote.upload_file(&file, &remote_path).await }; - // Handle keep_last - if let Some(to_keep) = keep_last { + upload_failed |= result.is_err(); + + // Handle keep_last. Skipped when the upload failed: the + // remote did not get a new backup, so nothing old goes. + if let (Some(to_keep), true) = (keep_last, result.is_ok()) { let to_keep = to_keep as usize; match remote.enumerate(remote_path.parent().unwrap()).await { Ok(list) => { @@ -295,6 +302,10 @@ impl Backup { ); } + if upload_failed { + service.upload_failed(&dump).await; + } + info!( "[{}] Next run: {}", name, diff --git a/src/services/service.rs b/src/services/service.rs index 813b5bd..eb1598f 100644 --- a/src/services/service.rs +++ b/src/services/service.rs @@ -47,4 +47,10 @@ pub trait Service: DynClone { // list returns the list of the paths that should be uploaded to the Remote - those are created by dump(). async fn list(&self) -> Vec; + + // upload_failed is called when the files of `dump` did not all reach the + // Remote. Services whose next dump builds on this one (e.g. zfs + // incrementals) undo it here, so the next run does not depend on data + // the Remote never received. The default does nothing. + async fn upload_failed(&self, _dump: &Dump) {} } diff --git a/src/services/zfs.rs b/src/services/zfs.rs index b097282..3c9634f 100644 --- a/src/services/zfs.rs +++ b/src/services/zfs.rs @@ -119,6 +119,26 @@ pub fn retention_sort_key(remote_name: &str) -> String { remote_name.to_string() } +/// The snapshot a local dump file was sent from: `--.snapshot` +/// maps to `@--`. Returns `None` for a file +/// that is not a dump of the service `name`. +pub(crate) fn dump_snapshot_name( + file_name: &str, + name: &str, + dataset: &str, + snapshot_base: &str, +) -> Option { + let snap = parse_snapshot_name(file_name, name)?; + let mark = match snap.kind { + SnapshotKind::Full => FULL_MARK, + SnapshotKind::Incremental => INC_MARK, + }; + Some(format!( + "{dataset}@{snapshot_base}-{mark}-{}", + format_ts(snap.time) + )) +} + /// Arguments for the `zfs send` command to dump the given snapshot. /// A full snapshot is sent with `send -R -v -c -L`, an incremental one with /// `send -R -v -c -L -i ` where `` is the name of the snapshot it @@ -221,17 +241,6 @@ pub(crate) fn root_snapshots(dataset: &str, snapshots: &[Snapshot]) -> Vec Vec { - snapshots - .iter() - .map(|s| s.name.clone()) - .filter(|n| n != new_full_name) - .collect() -} - #[derive(Debug)] pub enum Error { ZfsError(std::io::Error), @@ -463,7 +472,13 @@ impl Zfs { let mut root = root_snapshots(&self.dataset, &self.list_our_snapshots().await?); root.sort_by_key(|s| s.time); let mut args = match root.as_slice() { - [] => return Ok(()), + [] => { + info!( + "zfs.{}: no snapshots of the chain yet, nothing to dry-run (the first run is a full)", + self.name + ); + return Ok(()); + } [only] => send_args(SnapshotKind::Full, None, &only.name), [.., prev, last] => send_args(SnapshotKind::Incremental, Some(&prev.name), &last.name), }; @@ -477,7 +492,7 @@ impl Zfs { String::from_utf8_lossy(&output.stderr).trim() ))); } - debug!("dry run `zfs {}` succeeded", args.join(" ")); + info!("zfs.{}: dry run `zfs {}` ok", self.name, args.join(" ")); Ok(()) } @@ -550,6 +565,24 @@ impl Zfs { Ok(()) } + /// Send the full snapshot `name` to `dest`. On failure the snapshot is + /// destroyed on the whole tree (and the partial dump removed): the old + /// chain is already gone, and a full that never reached the remote must + /// not become the base of the next incremental. The next run then finds + /// no full and takes one again. + async fn send_full(&self, args: &[String], dest: &str, name: &str) -> Result<(), Error> { + let result = self.send(args, dest).await; + if result.is_err() { + if self.destroy(name, true).await { + info!("full snapshot {name} destroyed after the failed send"); + } else { + warn!("failed to destroy full snapshot {name} after the failed send"); + } + let _ = fs::remove_file(dest).await; + } + result + } + /// Destroy the snapshot `name`; with `recursive`, also the snapshots with /// the same name on the child datasets (`zfs destroy -r`). async fn destroy(&self, name: &str, recursive: bool) -> bool { @@ -579,6 +612,41 @@ impl Service for Zfs { async fn list(&self) -> Vec { self.list_files().await } + + /// The dump of this run did not reach the remote: destroy the snapshot + /// it was sent from, on the whole tree. Otherwise the next incremental + /// would be based on it, producing a stream the remote cannot apply + /// (its base is missing there). The next run then takes an incremental + /// against the previous snapshot, or a full when this run was the full + /// (the old chain is already gone). + async fn upload_failed(&self, dump: &Dump) { + let Some(file_name) = dump + .path + .as_ref() + .and_then(|p| p.file_name()) + .map(|f| f.to_string_lossy().to_string()) + else { + return; + }; + let Some(snapshot) = + dump_snapshot_name(&file_name, &self.name, &self.dataset, &self.snapshot_base) + else { + warn!( + "upload of {file_name} failed, but it is not a dump of zfs.{}", + self.name + ); + return; + }; + warn!("upload of {file_name} failed: destroying {snapshot} so the next run does not build on it"); + if self.destroy(&snapshot, true).await { + info!("zfs snapshot {snapshot} destroyed"); + } else { + warn!( + "failed to destroy zfs snapshot {snapshot}: destroy it manually \ + (zfs destroy -r {snapshot}), or the next incremental is based on it" + ); + } + } } impl Zfs { @@ -624,15 +692,18 @@ impl Zfs { .map(|b| format!(" based on {b}")) .unwrap_or_default() ); - self.create_snapshot(&new_name).await?; - if is_full { - // A full chain replaces all previous ones: dump it and clean up. + // A full chain replaces all previous ones. Clean up before taking + // the full: `zfs send -R` streams every snapshot of the tree up + // to the new one, so leftover snapshots of the old chain would + // end up inside the full dump. + self.cleanup(&snapshots).await; + self.create_snapshot(&new_name).await?; let send = send_args(kind, base.as_deref(), &new_name); - self.send(&send, &dest).await?; + self.send_full(&send, &dest, &new_name).await?; info!("ZFS full checkpoint completed successfully"); - self.cleanup(&snapshots, &new_name).await; } else { + self.create_snapshot(&new_name).await?; // Incremental: fall back to a full if the base is gone // (e.g. a child dataset was destroyed and recreated). let base = base.expect("plan guarantees a base for incrementals"); @@ -650,15 +721,17 @@ impl Zfs { warn!("failed to destroy incremental snapshot {new_name}"); } let full_name = format!("{}@{}-{FULL_MARK}-{ts}", self.dataset, self.snapshot_base); + // As for a planned full: the old chain goes before the full + // is taken, so it is not streamed inside it. + self.cleanup(&snapshots).await; self.create_snapshot(&full_name).await?; // The fallback is a full stream: name the dump file as such, // or a restore would treat it as an incremental. let _ = fs::remove_file(&dest).await; let dest = dump_file_name(&self.name, SnapshotKind::Full, &ts); let send = send_args(SnapshotKind::Full, None, &full_name); - self.send(&send, &dest).await?; + self.send_full(&send, &dest, &full_name).await?; info!("ZFS full checkpoint completed successfully (incremental fallback)"); - self.cleanup(&snapshots, &full_name).await; return Ok(Dump { path: Some(dest.into()), }); @@ -738,12 +811,13 @@ fn parse_snapshot_name(file_name: &str, name: &str) -> Option { } impl Zfs { - /// After a successful full: destroy all older snapshots of our chain on - /// the dataset tree. Best effort: failures are logged but do not fail the - /// backup. Local dump files are not cleaned up here: the [Dump] returned - /// by `do_dump` removes its own file when dropped by the backup job. - async fn cleanup(&self, snapshots: &[Snapshot], new_full_name: &str) { - let to_destroy = cleanup_list(new_full_name, snapshots); + /// Before a new full: destroy all snapshots of the old chain on the + /// dataset tree. Best effort: failures are logged but do not fail the + /// backup (a leftover snapshot is only streamed inside the new full). + /// Local dump files are not cleaned up here: the [Dump] returned by + /// `do_dump` removes its own file when dropped by the backup job. + async fn cleanup(&self, snapshots: &[Snapshot]) { + let to_destroy: Vec = snapshots.iter().map(|s| s.name.clone()).collect(); let mut destroyed = 0; for name in &to_destroy { if self.destroy(name, false).await { @@ -843,6 +917,38 @@ mod tests { assert_eq!(root[0].name, "tank@snap-full-20260101-010000"); } + #[test] + fn dump_snapshot_name_maps_the_dump_to_its_snapshot() { + assert_eq!( + dump_snapshot_name( + "storage-inc-20261007-220000.snapshot", + "storage", + "storage", + "storage-snap" + ) + .as_deref(), + Some("storage@storage-snap-inc-20261007-220000") + ); + assert_eq!( + dump_snapshot_name( + "zroot-full-20261101-100000.snapshot", + "zroot", + "zroot", + "zroot-snap" + ) + .as_deref(), + Some("zroot@zroot-snap-full-20261101-100000") + ); + // Not a dump of this service. + assert!(dump_snapshot_name( + "storage-inc-20261007-220000.snapshot", + "zroot", + "zroot", + "zroot-snap" + ) + .is_none()); + } + #[test] fn retention_sort_key_ignores_the_kind() { assert_eq!( @@ -1007,36 +1113,6 @@ mod tests { assert_eq!(select_plan(t("20260115"), Some(&cron), &[]), vec!["full"]); } - #[test] - fn cleanup_keeps_only_new_full() { - let snaps = vec![ - snap( - "tank@snap-full-20260101-010000", - SnapshotKind::Full, - "20260101", - ), - snap( - "tank@snap-inc-20260102-010000", - SnapshotKind::Incremental, - "20260102", - ), - snap( - "tank/data@snap-inc-20260102-010000", - SnapshotKind::Incremental, - "20260102", - ), - ]; - let cleaned = cleanup_list("tank@snap-full-20260201-010000", &snaps); - assert_eq!( - cleaned, - vec![ - "tank@snap-full-20260101-010000", - "tank@snap-inc-20260102-010000", - "tank/data@snap-inc-20260102-010000", - ] - ); - } - #[test] fn parse_snapshot_name_file() { let s = parse_snapshot_name("snap-inc-20260102-020000.snapshot", "snap").unwrap();