From d542f469e8655cd3b1ab45eb6e2d6ce1de5cad35 Mon Sep 17 00:00:00 2001 From: Lancelot Marti Date: Thu, 27 Aug 2026 06:32:58 +0000 Subject: [PATCH 01/20] Add backup functionality --- .../lomas_client/tests/test_integrations.py | 9 +++ client/lomas_client/tests/test_temp.py | 13 ++++ devenv.nix | 7 ++ .../admin_database/admin_database.py | 11 +++ .../admin_database/local_database.py | 57 +++++++++++++++- server/lomas_server/models/config.py | 24 +++++++ server/lomas_server/models/responses.py | 11 +++ server/lomas_server/routes/routes_admin.py | 19 +++++- server/lomas_server/utils/backup_storage.py | 67 +++++++++++++++++++ 9 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 client/lomas_client/tests/test_temp.py create mode 100644 server/lomas_server/utils/backup_storage.py diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 6febdd0b2..9bb770912 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -15,6 +15,7 @@ from bs4 import BeautifulSoup from csvw_eo.constants import COL_NAME, TABLE_SCHEMA from diffprivlib import models +from fastapi.testclient import TestClient from opendp.mod import enable_features from sklearn.pipeline import Pipeline @@ -27,6 +28,7 @@ del_all_dex_users, ) from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup +from lomas_server.app import get_admin_app from lomas_server.models.config import AdminConfig, ServerConfig enable_features("contrib") @@ -378,6 +380,13 @@ def test_demo_diffprivlib(dex_config, demo_setup) -> None: assert len(predictions) == 2 assert predictions == pytest.approx([20, 20], abs=20) + breakpoint() + config = Config() + with TestClient( + get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"} + ) as client_admin: + response = client_admin.post("/backup") + @pytest.mark.long def test_demo_opendp_polars(dex_config, demo_setup) -> None: diff --git a/client/lomas_client/tests/test_temp.py b/client/lomas_client/tests/test_temp.py new file mode 100644 index 000000000..31c5e9b1c --- /dev/null +++ b/client/lomas_client/tests/test_temp.py @@ -0,0 +1,13 @@ +# from pathlib import Path +# from fastapi.testclient import TestClient +# from lomas_server.app import get_admin_app +# from lomas_server.models.config import Config + +# config = Config() +# config.database.wipe() +# config.database.set_bootstrap(config.bootstrap) + +# with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: +# breakpoint() +# response = client.post("/backup") +# print(response.status_code, response.json()) diff --git a/devenv.nix b/devenv.nix index 6c8e0f1e0..72209fc37 100644 --- a/devenv.nix +++ b/devenv.nix @@ -203,6 +203,13 @@ in # Too many unrelated (3party dep warnings for now) # PYTHONWARNDEFAULTENCODING = 1; + # Config for sqlite backup (S3) + LOMAS_SERVICE_backup__s3__bucket = "bucket"; + LOMAS_SERVICE_backup__s3__key_prefix = "backup"; + LOMAS_SERVICE_backup__s3__endpoint_url = "http://${config.lomas.garage.host}:${toString config.lomas.garage.port}"; + LOMAS_SERVICE_backup__s3__access_key_id = config.lomas.garage.keyId; + LOMAS_SERVICE_backup__s3__secret_access_key = config.lomas.garage.secretKey; + # Lomas Runtime LOMAS_SERVER_log_level = "INFO"; LOMAS_SERVER_lomas_log_level = "DEBUG"; diff --git a/server/lomas_server/admin_database/admin_database.py b/server/lomas_server/admin_database/admin_database.py index ddb9ce751..d78c32150 100644 --- a/server/lomas_server/admin_database/admin_database.py +++ b/server/lomas_server/admin_database/admin_database.py @@ -486,3 +486,14 @@ def get_bootstrap_disabled(self) -> bool: Returns: bool: The bootstrap disabled value. False by default if not set in the DB. """ + + @abstractmethod + def backup(self) -> bytes: + """Creates a backup of the database and returns it as a Zip archive. + + The backup is a zip archive containing snapshots of the underlying storage (db, archives, misc). + It can be stored locally or in a S3. + + Returns: + bytes: A zip archive containing the backup. + """ diff --git a/server/lomas_server/admin_database/local_database.py b/server/lomas_server/admin_database/local_database.py index c7920de44..5470e23f6 100644 --- a/server/lomas_server/admin_database/local_database.py +++ b/server/lomas_server/admin_database/local_database.py @@ -1,9 +1,13 @@ +import io import json import sqlite3 +import zipfile from collections.abc import Generator from contextlib import AbstractContextManager, closing, contextmanager, nullcontext +from datetime import UTC, datetime +from functools import wraps from pathlib import Path -from tempfile import SpooledTemporaryFile +from tempfile import SpooledTemporaryFile, TemporaryDirectory from typing import Any, BinaryIO, override from uuid import UUID @@ -964,3 +968,54 @@ def get_bootstrap_disabled(self) -> bool: case _: ADMINDB_ERROR_COUNTER.add(1, {"operation": "invalid_misc_value_return"}) raise InternalServerException("Invalid Query Returns") + + # Backup + ########################################################################### + + def _sqlite_paths_to_backup(self) -> list[Path]: + """Paths of the sqlite files that make up the database state to snapshot.""" + return [self._db_path, self._archives_db_path, self._misc_db_path] + + @staticmethod + def _snapshot_sqlite_file(src_path: Path, dest_path: Path) -> None: + """Writes a point-in-time copy of a live sqlite db to dest_path. + + Args: + src_path (Path): Path of the sqlite database to copy. + dest_path (Path): Path to write the snapshot to. + """ + with ( + closing(sqlite3.connect(src_path)) as src_conn, + closing(sqlite3.connect(dest_path)) as dest_conn, + ): + src_conn.backup(dest_conn) + + @override + @db_span("db.backup", table="admin-db") + def backup(self) -> bytes: + ADMINDB_QUERY_COUNTER.add(1, {"operation": "backup"}) + + with TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + buffer = io.BytesIO() + + with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as archive: + for src_path in self._sqlite_paths_to_backup(): + if not src_path.exists(): + # If nothing writtenm, we don't save + continue + + snapshot_path = tmp_path / src_path.name + self._snapshot_sqlite_file(src_path, snapshot_path) + archive.write(snapshot_path, arcname=src_path.name) + + return buffer.getvalue() + + def backup_filename(self) -> str: + """Generates a timestamped filename. + + Returns: + str: filename for the backup. + """ + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return f"lomas-admin-backup-{timestamp}.zip" diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index 3fe3a7f48..13fdaa0e9 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -32,6 +32,28 @@ class S3CredentialsConfig(PrivateDBCredentials): secret_access_key: str +class BackupS3Config(BaseModel): + """S3 destination for admin database backups.""" + + bucket: str + key_prefix: str = Field(default="lomas-backups/") + endpoint_url: str | None = Field(default=None) + access_key_id: str + secret_access_key: str + + +class BackupConfig(BaseModel): + """Destination configuration for admin database backups. + + If `s3` is set, backups are uploaded to S3. Otherwise, they are written + to `local_directory` (falling back to a 'backups' subdirectory of the + server's database_directory if not set). + """ + + local_directory: Path | None = Field(default=None) + s3: BackupS3Config | None = Field(default=None) + + class DexAdminConfig(BaseModel): url: Url = Field(description="Dex OIDC server addresse") @@ -65,6 +87,8 @@ class Config(BaseSettings): private_db_credentials: dict[int, Annotated[S3CredentialsConfig, Field(discriminator="db_type")]] = {} + backup: BackupConfig = Field(default_factory=BackupConfig) + opendp_features: OpenDPFeatures = Field(default=["contrib", "idealized-numerics", "honest-but-curious"]) telemetry: Telemetry = Field(default_factory=Telemetry, description=CLI_SUPPRESS) diff --git a/server/lomas_server/models/responses.py b/server/lomas_server/models/responses.py index e9146636c..29fdc4c77 100644 --- a/server/lomas_server/models/responses.py +++ b/server/lomas_server/models/responses.py @@ -11,3 +11,14 @@ class ConfigResponse(BaseModel): config: ServerConfig = Field(default_factory=ServerConfig) """The server config.""" + + +class BackupResponse(BaseModel): + """Model for response to an admin database backup request.""" + + location: str + """Where the backup was written: a local path, or an s3://bucket/key URI.""" + is_s3: bool + """Whether the backup was uploaded to S3 (True) or written locally (False).""" + size_bytes: int + """Size in bytes of the backup archive.""" diff --git a/server/lomas_server/routes/routes_admin.py b/server/lomas_server/routes/routes_admin.py index b53ecee97..ac02c23cc 100644 --- a/server/lomas_server/routes/routes_admin.py +++ b/server/lomas_server/routes/routes_admin.py @@ -23,9 +23,11 @@ ) from lomas_server.admin_database.constants import BudgetDBKey from lomas_server.admin_database.local_database import LocalAdminDatabase -from lomas_server.models.responses import ConfigResponse +from lomas_server.models.config import Config +from lomas_server.models.responses import BackupResponse, ConfigResponse from lomas_server.routes.error_handler import API_ERROR_RESPONSES from lomas_server.routes.utils import get_user_id_from_authenticator +from lomas_server.utils.backup_storage import store_backup router = APIRouter() example_get_admin_db_data_body = Body(EXAMPLE_GET_ADMIN_DB_DATA) @@ -346,6 +348,21 @@ def set_dataset_metadata_admin( db.set_dataset_metadata(dataset_name, file.file) +@router.post("/backup", responses=API_ERROR_RESPONSES) +def backup_admin_database( + request: Request, + _: Annotated[UserId, Security(get_user_id_from_authenticator, scopes=[Scopes.ADMIN])], +) -> BackupResponse: + + db: LocalAdminDatabase = request.app.state.admin_database + config = Config() + + data = db.backup() + destination = store_backup(data, db.backup_filename(), config.database_directory, config.backup) + + return BackupResponse(location=destination.location, is_s3=destination.is_s3, size_bytes=len(data)) + + @router.get("/bootstrap", responses=API_ERROR_RESPONSES) def get_bootstrap( request: Request, diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py new file mode 100644 index 000000000..4a9e3fbe1 --- /dev/null +++ b/server/lomas_server/utils/backup_storage.py @@ -0,0 +1,67 @@ +from dataclasses import dataclass +from pathlib import Path + +import boto3 + +from lomas_core.models.constants import get_lomas_logger +from lomas_server.models.config import BackupConfig, BackupS3Config + +logger = get_lomas_logger(__name__) + + +@dataclass(frozen=True) +class BackupDestination: + """Where a backup ended up being written.""" + + location: str + """Local path, or an s3://bucket/key URI.""" + is_s3: bool + + +def store_backup( + data: bytes, filename: str, database_directory: Path, config: BackupConfig +) -> BackupDestination: + """Persists a backup archive either to S3 or to a local directory. + + If `config.s3` is set, the archive is uploaded to that S3 bucket. Otherwise, + it is written to `config.local_directory`, falling back to a 'backups' + subdirectory of the server's `database_directory` if that is not set either. + + Args: + data (bytes): The backup archive content (e.g. a zip file's bytes). + filename (str): The filename to give the backup (e.g. 'lomas-admin-backup-....zip'). + database_directory (Path): The server's admin database directory, used + as a fallback base directory for local backups. + config (BackupConfig): Backup destination configuration. + + Returns: + BackupDestination: Where the backup was written. + """ + if config.s3 is not None: + return _store_backup_s3(data, filename, config.s3) + + return _store_backup_local(data, filename, config.local_directory or (database_directory / "backups")) + + +def _store_backup_local(data: bytes, filename: str, directory: Path) -> BackupDestination: + directory.mkdir(parents=True, exist_ok=True) + dest_path = directory / filename + dest_path.write_bytes(data) + logger.info(f"Wrote admin database backup to {dest_path}.") + return BackupDestination(location=str(dest_path), is_s3=False) + + +def _store_backup_s3(data: bytes, filename: str, s3_config: BackupS3Config) -> BackupDestination: + key = f"{s3_config.key_prefix.rstrip('/')}/{filename}" if s3_config.key_prefix else filename + + client = boto3.client( + "s3", + endpoint_url=str(s3_config.endpoint_url) if s3_config.endpoint_url else None, + aws_access_key_id=s3_config.access_key_id, + aws_secret_access_key=s3_config.secret_access_key, + ) + client.put_object(Bucket=s3_config.bucket, Key=key, Body=data) + + location = f"s3://{s3_config.bucket}/{key}" + logger.info(f"Uploaded admin database backup to {location}.") + return BackupDestination(location=location, is_s3=True) From e40152c41cb3203b73f935349939e85364ddec5b Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Thu, 27 Aug 2026 13:39:45 +0200 Subject: [PATCH 02/20] Add backup button in dashboard --- .../dashboard/database_administration.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/lomas_server/administration/dashboard/database_administration.py b/server/lomas_server/administration/dashboard/database_administration.py index 47a2f5d19..6b763b795 100644 --- a/server/lomas_server/administration/dashboard/database_administration.py +++ b/server/lomas_server/administration/dashboard/database_administration.py @@ -526,3 +526,14 @@ def del_all_lomas_users() -> ResultE: lambda: drop_lomas_collection(TK.ARCHIVE), "All Archives deleted.", ) + +st.divider() +st.title("Backup") +st.write("Creates a backup of the admin database (jobs, users, archives).") +if st.button("Backup now", key="btn_backup_admin_db"): + match query_lomas_auth("/backup", httpx2.post): + case Success(backup_info): + size_kb = backup_info["size_bytes"] / 1024 + st.success(f"Backup written to `{backup_info['location']}` ({size_kb:.1f} KB).") + case Failure(e): + st.error(f"Backup failed: {e}") From 6402388227cac3b7263e65c47bf77c070877abdb Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Thu, 27 Aug 2026 17:17:29 +0200 Subject: [PATCH 03/20] Add tests backup --- .../lomas_client/tests/test_integrations.py | 48 ++++++++++++++++--- client/lomas_client/tests/test_temp.py | 13 ----- 2 files changed, 41 insertions(+), 20 deletions(-) delete mode 100644 client/lomas_client/tests/test_temp.py diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 9bb770912..3a5ca1914 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -1,4 +1,5 @@ import io +import os import re import sys import time @@ -380,13 +381,6 @@ def test_demo_diffprivlib(dex_config, demo_setup) -> None: assert len(predictions) == 2 assert predictions == pytest.approx([20, 20], abs=20) - breakpoint() - config = Config() - with TestClient( - get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"} - ) as client_admin: - response = client_admin.post("/backup") - @pytest.mark.long def test_demo_opendp_polars(dex_config, demo_setup) -> None: @@ -424,3 +418,43 @@ def test_demo_opendp_polars(dex_config, demo_setup) -> None: assert response_archives is not None assert response_archives.epsilon == DEFAULT_EPSILON assert response_archives.delta == pytest.approx(0.0, abs=0.1) + + +def test_backup(): + # With S3 + config = Config() + config.database.wipe() + config.database.set_bootstrap(config.bootstrap) + + with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: + response = client.post("/backup") + body = response.json() + assert body["is_s3"] is True + assert body["location"].startswith("s3://") + + # No s3 is configured, falls back to local saved (tmp) + for var in ( + "LOMAS_SERVICE_backup__s3__bucket", + "LOMAS_SERVICE_backup__s3__key_prefix", + "LOMAS_SERVICE_backup__s3__endpoint_url", + "LOMAS_SERVICE_backup__s3__access_key_id", + "LOMAS_SERVICE_backup__s3__secret_access_key", + ): + os.environ.pop(var, None) + config = Config() + + with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: + response = client.post("/backup") + assert response.json()["is_s3"] is False + assert os.path.exists(response.json()["location"]) + + # With local_directory setup + os.environ["LOMAS_SERVICE_backup__local_directory"] = "/tmp/lomas-custom-backups" + config = Config() + + with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: + response = client.post("/backup") + body = response.json() + assert body["is_s3"] is False + assert body["location"].startswith("/tmp/lomas-custom-backups/") + assert os.path.exists(body["location"]) diff --git a/client/lomas_client/tests/test_temp.py b/client/lomas_client/tests/test_temp.py deleted file mode 100644 index 31c5e9b1c..000000000 --- a/client/lomas_client/tests/test_temp.py +++ /dev/null @@ -1,13 +0,0 @@ -# from pathlib import Path -# from fastapi.testclient import TestClient -# from lomas_server.app import get_admin_app -# from lomas_server.models.config import Config - -# config = Config() -# config.database.wipe() -# config.database.set_bootstrap(config.bootstrap) - -# with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: -# breakpoint() -# response = client.post("/backup") -# print(response.status_code, response.json()) From a1383a57164e00dad279969bbe8c571c116f9239 Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Fri, 28 Aug 2026 11:01:53 +0200 Subject: [PATCH 04/20] Add test backup --- .../lomas_client/tests/test_integrations.py | 76 ++++++++++++++++--- devenv.nix | 12 +-- server/lomas_server/routes/routes_admin.py | 4 +- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 3a5ca1914..7c4852d48 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -1,9 +1,13 @@ import io import os import re +import sqlite3 import sys +import tempfile import time +import zipfile from dataclasses import dataclass +from pathlib import Path from urllib.parse import urljoin import numpy as np @@ -422,7 +426,7 @@ def test_demo_opendp_polars(dex_config, demo_setup) -> None: def test_backup(): # With S3 - config = Config() + config = ServerConfig() config.database.wipe() config.database.set_bootstrap(config.bootstrap) @@ -434,14 +438,14 @@ def test_backup(): # No s3 is configured, falls back to local saved (tmp) for var in ( - "LOMAS_SERVICE_backup__s3__bucket", - "LOMAS_SERVICE_backup__s3__key_prefix", - "LOMAS_SERVICE_backup__s3__endpoint_url", - "LOMAS_SERVICE_backup__s3__access_key_id", - "LOMAS_SERVICE_backup__s3__secret_access_key", + "LOMAS_SERVER_backup__s3__bucket", + "LOMAS_SERVER_backup__s3__key_prefix", + "LOMAS_SERVER_backup__s3__endpoint_url", + "LOMAS_SERVER_backup__s3__access_key_id", + "LOMAS_SERVER_backup__s3__secret_access_key", ): os.environ.pop(var, None) - config = Config() + config = ServerConfig() with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: response = client.post("/backup") @@ -449,12 +453,66 @@ def test_backup(): assert os.path.exists(response.json()["location"]) # With local_directory setup - os.environ["LOMAS_SERVICE_backup__local_directory"] = "/tmp/lomas-custom-backups" - config = Config() + os.environ["LOMAS_SERVER_backup__local_directory"] = "/tmp/lomas-custom-backups" + config = ServerConfig() with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: + # Create one query to test backup content + lomas_demo_setup() + user_name = "Jack" + client_u = Client( + user_name=f"{user_name}@example.com", user_password=user_name.lower(), dataset_name="TITANIC" + ) + + context = client_u.get_context(epsilon=DEFAULT_EPSILON) + plan = context.query().select(pl.col("Age").dp.mean(bounds=(0, 120)), dp.len()) + client_u.opendp.query(plan, epsilon=DEFAULT_EPSILON) + + # Backup bew db state response = client.post("/backup") body = response.json() + + # Check if custom location works assert body["is_s3"] is False assert body["location"].startswith("/tmp/lomas-custom-backups/") assert os.path.exists(body["location"]) + + # Test that we have correct tables saved in backup + # Load each sqlite db out of the backup zip and check its tables + backup_path = Path(body["location"]) + expected_tables_by_file = { + "db.sqlite3": {"jobs", "users"}, + "archives.sqlite3": {"archives"}, + "misc.sqlite3": {"misc"}, + } + + with zipfile.ZipFile(backup_path) as archive, tempfile.TemporaryDirectory() as extract_dir: + assert set(archive.namelist()) == set(expected_tables_by_file) + + for filename, expected_tables in expected_tables_by_file.items(): + db_path = Path(extract_dir) / filename + db_path.write_bytes(archive.read(filename)) + + conn = sqlite3.connect(db_path) + try: + # Check tables are correctly saved in each db + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + } + + # Check that the client query is saved in backup + if tables == {"archives"}: + row = conn.execute( + "SELECT uid, user_name, dataset_name, status FROM archives;" + ).fetchone() + + assert row[1] == user_name + assert row[2] == "TITANIC" + assert row[3] == "complete" + + finally: + conn.close() + assert expected_tables <= tables, f"{filename} missing tables {expected_tables - tables}" diff --git a/devenv.nix b/devenv.nix index 72209fc37..f917ec9e6 100644 --- a/devenv.nix +++ b/devenv.nix @@ -204,12 +204,12 @@ in # PYTHONWARNDEFAULTENCODING = 1; # Config for sqlite backup (S3) - LOMAS_SERVICE_backup__s3__bucket = "bucket"; - LOMAS_SERVICE_backup__s3__key_prefix = "backup"; - LOMAS_SERVICE_backup__s3__endpoint_url = "http://${config.lomas.garage.host}:${toString config.lomas.garage.port}"; - LOMAS_SERVICE_backup__s3__access_key_id = config.lomas.garage.keyId; - LOMAS_SERVICE_backup__s3__secret_access_key = config.lomas.garage.secretKey; - + LOMAS_SERVER_backup__s3__bucket = "bucket"; + LOMAS_SERVER_backup__s3__key_prefix = "backup"; + LOMAS_SERVER_backup__s3__endpoint_url = "http://${config.lomas.garage.host}:${toString config.lomas.garage.port}"; + LOMAS_SERVER_backup__s3__access_key_id = config.lomas.garage.keyId; + LOMAS_SERVER_backup__s3__secret_access_key = config.lomas.garage.secretKey; + # Lomas Runtime LOMAS_SERVER_log_level = "INFO"; LOMAS_SERVER_lomas_log_level = "DEBUG"; diff --git a/server/lomas_server/routes/routes_admin.py b/server/lomas_server/routes/routes_admin.py index ac02c23cc..e3a2e64c3 100644 --- a/server/lomas_server/routes/routes_admin.py +++ b/server/lomas_server/routes/routes_admin.py @@ -23,7 +23,7 @@ ) from lomas_server.admin_database.constants import BudgetDBKey from lomas_server.admin_database.local_database import LocalAdminDatabase -from lomas_server.models.config import Config +from lomas_server.models.config import ServerConfig from lomas_server.models.responses import BackupResponse, ConfigResponse from lomas_server.routes.error_handler import API_ERROR_RESPONSES from lomas_server.routes.utils import get_user_id_from_authenticator @@ -355,7 +355,7 @@ def backup_admin_database( ) -> BackupResponse: db: LocalAdminDatabase = request.app.state.admin_database - config = Config() + config = ServerConfig() data = db.backup() destination = store_backup(data, db.backup_filename(), config.database_directory, config.backup) From ec977bb23523a9cd7fb3fbdf24e10ac68db86957 Mon Sep 17 00:00:00 2001 From: Lancelot Marti Date: Tue, 1 Sep 2026 09:37:06 +0000 Subject: [PATCH 05/20] Change pydantic model for BackupS3Config --- .../lomas_client/tests/test_integrations.py | 9 +--- devenv.nix | 15 ++++-- server/lomas_server/models/config.py | 46 +++++++++++++++++-- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 7c4852d48..d172dd498 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -437,14 +437,7 @@ def test_backup(): assert body["location"].startswith("s3://") # No s3 is configured, falls back to local saved (tmp) - for var in ( - "LOMAS_SERVER_backup__s3__bucket", - "LOMAS_SERVER_backup__s3__key_prefix", - "LOMAS_SERVER_backup__s3__endpoint_url", - "LOMAS_SERVER_backup__s3__access_key_id", - "LOMAS_SERVER_backup__s3__secret_access_key", - ): - os.environ.pop(var, None) + os.environ.pop("LOMAS_SERVER_backup__s3__uri", None) config = ServerConfig() with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: diff --git a/devenv.nix b/devenv.nix index f917ec9e6..f3d609fb9 100644 --- a/devenv.nix +++ b/devenv.nix @@ -204,11 +204,16 @@ in # PYTHONWARNDEFAULTENCODING = 1; # Config for sqlite backup (S3) - LOMAS_SERVER_backup__s3__bucket = "bucket"; - LOMAS_SERVER_backup__s3__key_prefix = "backup"; - LOMAS_SERVER_backup__s3__endpoint_url = "http://${config.lomas.garage.host}:${toString config.lomas.garage.port}"; - LOMAS_SERVER_backup__s3__access_key_id = config.lomas.garage.keyId; - LOMAS_SERVER_backup__s3__secret_access_key = config.lomas.garage.secretKey; + LOMAS_SERVER_backup__s3__uri = + "http://" + + config.lomas.garage.keyId + + ":" + + config.lomas.garage.secretKey + + "@" + + config.lomas.garage.host + + ":" + + toString config.lomas.garage.port + + "/bucket/backup"; # Lomas Runtime LOMAS_SERVER_log_level = "INFO"; diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index 13fdaa0e9..1bc3b8a2c 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -1,10 +1,13 @@ from pathlib import Path from typing import Annotated, Literal +from urllib.parse import unquote from pydantic import ( + AnyUrl, BaseModel, Field, HttpUrl, + UrlConstraints, computed_field, ) from pydantic_core import Url @@ -32,14 +35,47 @@ class S3CredentialsConfig(PrivateDBCredentials): secret_access_key: str +BackupUri = Annotated[ + AnyUrl, + UrlConstraints(allowed_schemes=["http", "https", "aws", "s3"]), +] + + class BackupS3Config(BaseModel): """S3 destination for admin database backups.""" - bucket: str - key_prefix: str = Field(default="lomas-backups/") - endpoint_url: str | None = Field(default=None) - access_key_id: str - secret_access_key: str + uri: BackupUri + + @computed_field + def access_key_id(self) -> str: + if self.uri.username is None: + raise ValueError("Backup S3 uri is missing access_key_id.") + return unquote(self.uri.username) + + @computed_field + def secret_access_key(self) -> str: + if self.uri.password is None: + raise ValueError("Backup S3 uri is missing secret_access_key.") + return unquote(self.uri.password) + + @computed_field + def endpoint_url(self) -> str: + port = f":{self.uri.port}" if self.uri.port else "" + return f"{self.uri.scheme}://{self.uri.host}{port}" + + @computed_field + def bucket(self) -> str: + path = (self.uri.path or "").lstrip("/") + bucket, _, _ = path.partition("/") + if not bucket: + raise ValueError("Backup S3 uri is missing a bucket name.") + return bucket + + @computed_field + def key_prefix(self) -> str: + path = (self.uri.path or "").lstrip("/") + _, _, prefix = path.partition("/") + return f"{prefix.rstrip('/')}/" if prefix else "lomas-backups/" class BackupConfig(BaseModel): From e84340f854bb4ce910246a21729611bb6a25f9cf Mon Sep 17 00:00:00 2001 From: Lancelot Marti Date: Tue, 1 Sep 2026 09:45:22 +0000 Subject: [PATCH 06/20] Fix config in backup route --- server/lomas_server/routes/routes_admin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/lomas_server/routes/routes_admin.py b/server/lomas_server/routes/routes_admin.py index e3a2e64c3..cd4654167 100644 --- a/server/lomas_server/routes/routes_admin.py +++ b/server/lomas_server/routes/routes_admin.py @@ -23,7 +23,6 @@ ) from lomas_server.admin_database.constants import BudgetDBKey from lomas_server.admin_database.local_database import LocalAdminDatabase -from lomas_server.models.config import ServerConfig from lomas_server.models.responses import BackupResponse, ConfigResponse from lomas_server.routes.error_handler import API_ERROR_RESPONSES from lomas_server.routes.utils import get_user_id_from_authenticator @@ -355,7 +354,7 @@ def backup_admin_database( ) -> BackupResponse: db: LocalAdminDatabase = request.app.state.admin_database - config = ServerConfig() + config = request.app.state.config data = db.backup() destination = store_backup(data, db.backup_filename(), config.database_directory, config.backup) From 99fbd3e50a49fa5e852531674f2bc1d9d649f1cc Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 09:17:40 +0200 Subject: [PATCH 07/20] Use discriminator --- .../lomas_client/tests/test_integrations.py | 3 ++- devenv.nix | 3 ++- server/lomas_server/models/config.py | 20 ++++++++++--------- server/lomas_server/utils/backup_storage.py | 4 ++-- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index d172dd498..1ec2c0d84 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -437,7 +437,8 @@ def test_backup(): assert body["location"].startswith("s3://") # No s3 is configured, falls back to local saved (tmp) - os.environ.pop("LOMAS_SERVER_backup__s3__uri", None) + os.environ.pop("LOMAS_SERVER_backup__uri", None) + os.environ["LOMAS_SERVER_backup__type"] = "local_directory" config = ServerConfig() with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: diff --git a/devenv.nix b/devenv.nix index f3d609fb9..4384e32fa 100644 --- a/devenv.nix +++ b/devenv.nix @@ -204,7 +204,8 @@ in # PYTHONWARNDEFAULTENCODING = 1; # Config for sqlite backup (S3) - LOMAS_SERVER_backup__s3__uri = + LOMAS_SERVER_backup__type = "s3"; + LOMAS_SERVER_backup__uri = "http://" + config.lomas.garage.keyId + ":" diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index 1bc3b8a2c..11afce5e5 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -44,6 +44,7 @@ class S3CredentialsConfig(PrivateDBCredentials): class BackupS3Config(BaseModel): """S3 destination for admin database backups.""" + type: Literal["s3"] = "s3" uri: BackupUri @computed_field @@ -78,16 +79,17 @@ def key_prefix(self) -> str: return f"{prefix.rstrip('/')}/" if prefix else "lomas-backups/" -class BackupConfig(BaseModel): - """Destination configuration for admin database backups. - - If `s3` is set, backups are uploaded to S3. Otherwise, they are written - to `local_directory` (falling back to a 'backups' subdirectory of the - server's database_directory if not set). - """ +class LocalBackupConfig(BaseModel): + """Local destination for admin database backups.""" + type: Literal["local_directory"] = "local_directory" local_directory: Path | None = Field(default=None) - s3: BackupS3Config | None = Field(default=None) + + +BackupConfig = Annotated[ + LocalBackupConfig | BackupS3Config, + Field(discriminator="type"), +] class DexAdminConfig(BaseModel): @@ -123,7 +125,7 @@ class Config(BaseSettings): private_db_credentials: dict[int, Annotated[S3CredentialsConfig, Field(discriminator="db_type")]] = {} - backup: BackupConfig = Field(default_factory=BackupConfig) + backup: BackupConfig = Field(default_factory=LocalBackupConfig) opendp_features: OpenDPFeatures = Field(default=["contrib", "idealized-numerics", "honest-but-curious"]) diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py index 4a9e3fbe1..c085095d9 100644 --- a/server/lomas_server/utils/backup_storage.py +++ b/server/lomas_server/utils/backup_storage.py @@ -37,8 +37,8 @@ def store_backup( Returns: BackupDestination: Where the backup was written. """ - if config.s3 is not None: - return _store_backup_s3(data, filename, config.s3) + if config.type == "s3": + return _store_backup_s3(data, filename, config) return _store_backup_local(data, filename, config.local_directory or (database_directory / "backups")) From aee792fe871ed7b0c0a1b9ea47670005c94ddd3b Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 09:27:29 +0200 Subject: [PATCH 08/20] Only use BackupResponse --- server/lomas_server/models/responses.py | 2 +- server/lomas_server/utils/backup_storage.py | 23 +++++++-------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/server/lomas_server/models/responses.py b/server/lomas_server/models/responses.py index 29fdc4c77..7355236ec 100644 --- a/server/lomas_server/models/responses.py +++ b/server/lomas_server/models/responses.py @@ -20,5 +20,5 @@ class BackupResponse(BaseModel): """Where the backup was written: a local path, or an s3://bucket/key URI.""" is_s3: bool """Whether the backup was uploaded to S3 (True) or written locally (False).""" - size_bytes: int + size_bytes: int | None = Field(default=None) """Size in bytes of the backup archive.""" diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py index c085095d9..8e04948c2 100644 --- a/server/lomas_server/utils/backup_storage.py +++ b/server/lomas_server/utils/backup_storage.py @@ -1,26 +1,17 @@ -from dataclasses import dataclass from pathlib import Path import boto3 from lomas_core.models.constants import get_lomas_logger from lomas_server.models.config import BackupConfig, BackupS3Config +from lomas_server.models.responses import BackupResponse logger = get_lomas_logger(__name__) -@dataclass(frozen=True) -class BackupDestination: - """Where a backup ended up being written.""" - - location: str - """Local path, or an s3://bucket/key URI.""" - is_s3: bool - - def store_backup( data: bytes, filename: str, database_directory: Path, config: BackupConfig -) -> BackupDestination: +) -> BackupResponse: """Persists a backup archive either to S3 or to a local directory. If `config.s3` is set, the archive is uploaded to that S3 bucket. Otherwise, @@ -35,7 +26,7 @@ def store_backup( config (BackupConfig): Backup destination configuration. Returns: - BackupDestination: Where the backup was written. + BackupResponse: Where the backup was written. """ if config.type == "s3": return _store_backup_s3(data, filename, config) @@ -43,15 +34,15 @@ def store_backup( return _store_backup_local(data, filename, config.local_directory or (database_directory / "backups")) -def _store_backup_local(data: bytes, filename: str, directory: Path) -> BackupDestination: +def _store_backup_local(data: bytes, filename: str, directory: Path) -> BackupResponse: directory.mkdir(parents=True, exist_ok=True) dest_path = directory / filename dest_path.write_bytes(data) logger.info(f"Wrote admin database backup to {dest_path}.") - return BackupDestination(location=str(dest_path), is_s3=False) + return BackupResponse(location=str(dest_path), is_s3=False) -def _store_backup_s3(data: bytes, filename: str, s3_config: BackupS3Config) -> BackupDestination: +def _store_backup_s3(data: bytes, filename: str, s3_config: BackupS3Config) -> BackupResponse: key = f"{s3_config.key_prefix.rstrip('/')}/{filename}" if s3_config.key_prefix else filename client = boto3.client( @@ -64,4 +55,4 @@ def _store_backup_s3(data: bytes, filename: str, s3_config: BackupS3Config) -> B location = f"s3://{s3_config.bucket}/{key}" logger.info(f"Uploaded admin database backup to {location}.") - return BackupDestination(location=location, is_s3=True) + return BackupResponse(location=location, is_s3=True) From f8fdd53e788aede7dc850f21700346c479a7a1c3 Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 09:30:16 +0200 Subject: [PATCH 09/20] Use get instead of post --- client/lomas_client/tests/test_integrations.py | 6 +++--- server/lomas_server/routes/routes_admin.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 1ec2c0d84..b917d4236 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -431,7 +431,7 @@ def test_backup(): config.database.set_bootstrap(config.bootstrap) with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: - response = client.post("/backup") + response = client.get("/backup") body = response.json() assert body["is_s3"] is True assert body["location"].startswith("s3://") @@ -442,7 +442,7 @@ def test_backup(): config = ServerConfig() with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: - response = client.post("/backup") + response = client.get("/backup") assert response.json()["is_s3"] is False assert os.path.exists(response.json()["location"]) @@ -463,7 +463,7 @@ def test_backup(): client_u.opendp.query(plan, epsilon=DEFAULT_EPSILON) # Backup bew db state - response = client.post("/backup") + response = client.get("/backup") body = response.json() # Check if custom location works diff --git a/server/lomas_server/routes/routes_admin.py b/server/lomas_server/routes/routes_admin.py index cd4654167..f12539142 100644 --- a/server/lomas_server/routes/routes_admin.py +++ b/server/lomas_server/routes/routes_admin.py @@ -347,7 +347,7 @@ def set_dataset_metadata_admin( db.set_dataset_metadata(dataset_name, file.file) -@router.post("/backup", responses=API_ERROR_RESPONSES) +@router.get("/backup", responses=API_ERROR_RESPONSES) def backup_admin_database( request: Request, _: Annotated[UserId, Security(get_user_id_from_authenticator, scopes=[Scopes.ADMIN])], From 9099eb04c168183d2a82af3afc133429898545fb Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 10:25:33 +0200 Subject: [PATCH 10/20] Add additional type checkers --- core/lomas_core/models/constants.py | 7 +++++++ server/lomas_server/models/config.py | 5 +++-- server/lomas_server/utils/backup_storage.py | 20 +++++++++++++------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/core/lomas_core/models/constants.py b/core/lomas_core/models/constants.py index 2cca8058a..79b23d19e 100644 --- a/core/lomas_core/models/constants.py +++ b/core/lomas_core/models/constants.py @@ -83,6 +83,13 @@ class AuthenticationType(StrEnum): OIDC = "oidc" +class BackupType(StrEnum): + """Type of storage for backup.""" + + LOCAL_DIRECTORY = "local_directory" + S3 = "s3" + + # Logging # ----------------------------------------------------------------------------- diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index 11afce5e5..b10809064 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -15,6 +15,7 @@ from lomas_core.models.config import Telemetry, TimeAttack from lomas_core.models.constants import ( + BackupType, OpenDPFeatures, PrivateDatabaseType, ) @@ -44,7 +45,7 @@ class S3CredentialsConfig(PrivateDBCredentials): class BackupS3Config(BaseModel): """S3 destination for admin database backups.""" - type: Literal["s3"] = "s3" + type: Literal[BackupType.S3] uri: BackupUri @computed_field @@ -82,7 +83,7 @@ def key_prefix(self) -> str: class LocalBackupConfig(BaseModel): """Local destination for admin database backups.""" - type: Literal["local_directory"] = "local_directory" + type: Literal[BackupType.LOCAL_DIRECTORY] local_directory: Path | None = Field(default=None) diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py index 8e04948c2..76c8a9224 100644 --- a/server/lomas_server/utils/backup_storage.py +++ b/server/lomas_server/utils/backup_storage.py @@ -2,7 +2,8 @@ import boto3 -from lomas_core.models.constants import get_lomas_logger +from lomas_core.exceptions import InternalServerException +from lomas_core.models.constants import BackupType, get_lomas_logger from lomas_server.models.config import BackupConfig, BackupS3Config from lomas_server.models.responses import BackupResponse @@ -10,7 +11,7 @@ def store_backup( - data: bytes, filename: str, database_directory: Path, config: BackupConfig + data: bytes, filename: str, database_directory: Path, backup_config: BackupConfig ) -> BackupResponse: """Persists a backup archive either to S3 or to a local directory. @@ -23,15 +24,20 @@ def store_backup( filename (str): The filename to give the backup (e.g. 'lomas-admin-backup-....zip'). database_directory (Path): The server's admin database directory, used as a fallback base directory for local backups. - config (BackupConfig): Backup destination configuration. + backup_config (BackupConfig): Backup destination configuration. Returns: BackupResponse: Where the backup was written. """ - if config.type == "s3": - return _store_backup_s3(data, filename, config) - - return _store_backup_local(data, filename, config.local_directory or (database_directory / "backups")) + match backup_config.type: + case BackupType.S3: + return _store_backup_s3(data, filename, backup_config) + case BackupType.LOCAL_DIRECTORY: + return _store_backup_local( + data, filename, backup_config.local_directory or (database_directory / "backups") + ) + case _: + raise InternalServerException(f"Backup type not supported: {backup_config.backup_type!r}") def _store_backup_local(data: bytes, filename: str, directory: Path) -> BackupResponse: From 509678be623bb57ebed5b456c7825a991dae5d0e Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 12:00:15 +0200 Subject: [PATCH 11/20] Fix docker-compose --- server/lomas_server/models/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index b10809064..f294b05bf 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -83,7 +83,7 @@ def key_prefix(self) -> str: class LocalBackupConfig(BaseModel): """Local destination for admin database backups.""" - type: Literal[BackupType.LOCAL_DIRECTORY] + type: Literal[BackupType.LOCAL_DIRECTORY] = BackupType.LOCAL_DIRECTORY local_directory: Path | None = Field(default=None) From f34a8b23a1633c06269cb4abfb2aa0b3112d82eb Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 13:12:46 +0200 Subject: [PATCH 12/20] Remove os.env in test_backup --- client/lomas_client/tests/test_integrations.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index b917d4236..d19975cf4 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -34,7 +34,7 @@ ) from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup from lomas_server.app import get_admin_app -from lomas_server.models.config import AdminConfig, ServerConfig +from lomas_server.models.config import AdminConfig, LocalBackupConfig, ServerConfig enable_features("contrib") @@ -437,9 +437,7 @@ def test_backup(): assert body["location"].startswith("s3://") # No s3 is configured, falls back to local saved (tmp) - os.environ.pop("LOMAS_SERVER_backup__uri", None) - os.environ["LOMAS_SERVER_backup__type"] = "local_directory" - config = ServerConfig() + config = ServerConfig(backup=LocalBackupConfig()) with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: response = client.get("/backup") @@ -447,9 +445,7 @@ def test_backup(): assert os.path.exists(response.json()["location"]) # With local_directory setup - os.environ["LOMAS_SERVER_backup__local_directory"] = "/tmp/lomas-custom-backups" - config = ServerConfig() - + config = ServerConfig(backup=LocalBackupConfig(local_directory="/tmp/lomas-custom-backups")) with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: # Create one query to test backup content lomas_demo_setup() From 6ae122f6350f0342b4f6ed1b5af64e46a8f3bc8d Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 14:52:33 +0200 Subject: [PATCH 13/20] Remove discriminator logic for backup --- core/lomas_core/models/constants.py | 7 ------- devenv.nix | 1 - server/lomas_server/models/config.py | 19 +++++-------------- server/lomas_server/utils/backup_storage.py | 13 +++++-------- 4 files changed, 10 insertions(+), 30 deletions(-) diff --git a/core/lomas_core/models/constants.py b/core/lomas_core/models/constants.py index 79b23d19e..2cca8058a 100644 --- a/core/lomas_core/models/constants.py +++ b/core/lomas_core/models/constants.py @@ -83,13 +83,6 @@ class AuthenticationType(StrEnum): OIDC = "oidc" -class BackupType(StrEnum): - """Type of storage for backup.""" - - LOCAL_DIRECTORY = "local_directory" - S3 = "s3" - - # Logging # ----------------------------------------------------------------------------- diff --git a/devenv.nix b/devenv.nix index 4384e32fa..8fe3911d8 100644 --- a/devenv.nix +++ b/devenv.nix @@ -204,7 +204,6 @@ in # PYTHONWARNDEFAULTENCODING = 1; # Config for sqlite backup (S3) - LOMAS_SERVER_backup__type = "s3"; LOMAS_SERVER_backup__uri = "http://" + config.lomas.garage.keyId diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index f294b05bf..fdc04f0d8 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -15,7 +15,6 @@ from lomas_core.models.config import Telemetry, TimeAttack from lomas_core.models.constants import ( - BackupType, OpenDPFeatures, PrivateDatabaseType, ) @@ -36,17 +35,13 @@ class S3CredentialsConfig(PrivateDBCredentials): secret_access_key: str -BackupUri = Annotated[ - AnyUrl, - UrlConstraints(allowed_schemes=["http", "https", "aws", "s3"]), -] - - class BackupS3Config(BaseModel): """S3 destination for admin database backups.""" - type: Literal[BackupType.S3] - uri: BackupUri + uri: Annotated[ + AnyUrl, + UrlConstraints(allowed_schemes=["http", "https", "aws", "s3"]), + ] @computed_field def access_key_id(self) -> str: @@ -83,14 +78,10 @@ def key_prefix(self) -> str: class LocalBackupConfig(BaseModel): """Local destination for admin database backups.""" - type: Literal[BackupType.LOCAL_DIRECTORY] = BackupType.LOCAL_DIRECTORY local_directory: Path | None = Field(default=None) -BackupConfig = Annotated[ - LocalBackupConfig | BackupS3Config, - Field(discriminator="type"), -] +BackupConfig = LocalBackupConfig | BackupS3Config class DexAdminConfig(BaseModel): diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py index 76c8a9224..1d2814f53 100644 --- a/server/lomas_server/utils/backup_storage.py +++ b/server/lomas_server/utils/backup_storage.py @@ -2,9 +2,8 @@ import boto3 -from lomas_core.exceptions import InternalServerException -from lomas_core.models.constants import BackupType, get_lomas_logger -from lomas_server.models.config import BackupConfig, BackupS3Config +from lomas_core.models.constants import get_lomas_logger +from lomas_server.models.config import BackupConfig, BackupS3Config, LocalBackupConfig from lomas_server.models.responses import BackupResponse logger = get_lomas_logger(__name__) @@ -29,15 +28,13 @@ def store_backup( Returns: BackupResponse: Where the backup was written. """ - match backup_config.type: - case BackupType.S3: + match backup_config: + case BackupS3Config(): return _store_backup_s3(data, filename, backup_config) - case BackupType.LOCAL_DIRECTORY: + case LocalBackupConfig(): return _store_backup_local( data, filename, backup_config.local_directory or (database_directory / "backups") ) - case _: - raise InternalServerException(f"Backup type not supported: {backup_config.backup_type!r}") def _store_backup_local(data: bytes, filename: str, directory: Path) -> BackupResponse: From f710e42e6bd8b0d66532c16f5a99ac0142173878 Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Wed, 2 Sep 2026 16:46:27 +0200 Subject: [PATCH 14/20] Use validators --- .../lomas_client/tests/test_integrations.py | 14 ++++---- server/lomas_server/models/config.py | 32 +++++++++++++------ server/lomas_server/utils/backup_storage.py | 4 ++- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index d19975cf4..1d32b8ff7 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -22,6 +22,7 @@ from diffprivlib import models from fastapi.testclient import TestClient from opendp.mod import enable_features +from pydantic import ValidationError from sklearn.pipeline import Pipeline from lomas_client import Client @@ -34,7 +35,7 @@ ) from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup from lomas_server.app import get_admin_app -from lomas_server.models.config import AdminConfig, LocalBackupConfig, ServerConfig +from lomas_server.models.config import AdminConfig, BackupS3Config, LocalBackupConfig, ServerConfig enable_features("contrib") @@ -436,13 +437,10 @@ def test_backup(): assert body["is_s3"] is True assert body["location"].startswith("s3://") - # No s3 is configured, falls back to local saved (tmp) - config = ServerConfig(backup=LocalBackupConfig()) - - with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: - response = client.get("/backup") - assert response.json()["is_s3"] is False - assert os.path.exists(response.json()["location"]) + # Raise an error if uri isn't correctly defined + # For instance missing user:password / bucket_name, etc. + with pytest.raises(ValidationError): + ServerConfig(backup=BackupS3Config(uri="https://localhost:3900/bucket")) # With local_directory setup config = ServerConfig(backup=LocalBackupConfig(local_directory="/tmp/lomas-custom-backups")) diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index fdc04f0d8..d3ca436f2 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Annotated, Literal +from typing import Annotated, Literal, Self from urllib.parse import unquote from pydantic import ( @@ -9,6 +9,7 @@ HttpUrl, UrlConstraints, computed_field, + model_validator, ) from pydantic_core import Url from pydantic_settings import CLI_SUPPRESS, BaseSettings, SettingsConfigDict @@ -43,16 +44,25 @@ class BackupS3Config(BaseModel): UrlConstraints(allowed_schemes=["http", "https", "aws", "s3"]), ] - @computed_field - def access_key_id(self) -> str: + @model_validator(mode="after") + def check_uri_content(self) -> Self: if self.uri.username is None: raise ValueError("Backup S3 uri is missing access_key_id.") + if self.uri.password is None: + raise ValueError("Backup S3 uri is missing secret_access_key.") + + path = (self.uri.path or "").lstrip("/") + bucket, _, _ = path.partition("/") + if not bucket: + raise ValueError("Backup S3 uri is missing a bucket name.") + return self + + @computed_field + def access_key_id(self) -> str: return unquote(self.uri.username) @computed_field def secret_access_key(self) -> str: - if self.uri.password is None: - raise ValueError("Backup S3 uri is missing secret_access_key.") return unquote(self.uri.password) @computed_field @@ -64,8 +74,6 @@ def endpoint_url(self) -> str: def bucket(self) -> str: path = (self.uri.path or "").lstrip("/") bucket, _, _ = path.partition("/") - if not bucket: - raise ValueError("Backup S3 uri is missing a bucket name.") return bucket @computed_field @@ -78,7 +86,13 @@ def key_prefix(self) -> str: class LocalBackupConfig(BaseModel): """Local destination for admin database backups.""" - local_directory: Path | None = Field(default=None) + @model_validator(mode="after") + def is_absolute(self) -> Self: + if not self.local_directory.is_absolute(): + raise ValueError("Use an absolute path.") + return self + + local_directory: Path BackupConfig = LocalBackupConfig | BackupS3Config @@ -117,7 +131,7 @@ class Config(BaseSettings): private_db_credentials: dict[int, Annotated[S3CredentialsConfig, Field(discriminator="db_type")]] = {} - backup: BackupConfig = Field(default_factory=LocalBackupConfig) + backup: BackupConfig = Field(default=LocalBackupConfig(local_directory="/tmp/lomas-backups")) opendp_features: OpenDPFeatures = Field(default=["contrib", "idealized-numerics", "honest-but-curious"]) diff --git a/server/lomas_server/utils/backup_storage.py b/server/lomas_server/utils/backup_storage.py index 1d2814f53..86dc4cc2b 100644 --- a/server/lomas_server/utils/backup_storage.py +++ b/server/lomas_server/utils/backup_storage.py @@ -33,7 +33,9 @@ def store_backup( return _store_backup_s3(data, filename, backup_config) case LocalBackupConfig(): return _store_backup_local( - data, filename, backup_config.local_directory or (database_directory / "backups") + data, + filename, + backup_config.local_directory or (database_directory / "backups"), # type: ignore [truthy-bool] ) From b24aca6556328e29d74a4831ade06364c4e58460 Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Thu, 3 Sep 2026 10:37:57 +0200 Subject: [PATCH 15/20] Reformat backup_uri env --- devenv.nix | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/devenv.nix b/devenv.nix index 8fe3911d8..b7c3105ca 100644 --- a/devenv.nix +++ b/devenv.nix @@ -205,15 +205,8 @@ in # Config for sqlite backup (S3) LOMAS_SERVER_backup__uri = - "http://" - + config.lomas.garage.keyId - + ":" - + config.lomas.garage.secretKey - + "@" - + config.lomas.garage.host - + ":" - + toString config.lomas.garage.port - + "/bucket/backup"; + with config.lomas.garage; + "http://${keyId}:${secretKey}@${host}:${toString port}/bucket/backup"; # Lomas Runtime LOMAS_SERVER_log_level = "INFO"; From db56e1b4f16cca682b6e53f3b36bb730c88b21e0 Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Thu, 3 Sep 2026 12:42:34 +0200 Subject: [PATCH 16/20] Remove misc, adapt tests --- client/lomas_client/tests/test_integrations.py | 7 ++++--- server/lomas_server/admin_database/admin_database.py | 2 +- server/lomas_server/admin_database/local_database.py | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index 1d32b8ff7..8678a0194 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -469,9 +469,8 @@ def test_backup(): # Load each sqlite db out of the backup zip and check its tables backup_path = Path(body["location"]) expected_tables_by_file = { - "db.sqlite3": {"jobs", "users"}, + "db.sqlite3": {"jobs", "users", "misc", "datasets"}, "archives.sqlite3": {"archives"}, - "misc.sqlite3": {"misc"}, } with zipfile.ZipFile(backup_path) as archive, tempfile.TemporaryDirectory() as extract_dir: @@ -503,4 +502,6 @@ def test_backup(): finally: conn.close() - assert expected_tables <= tables, f"{filename} missing tables {expected_tables - tables}" + assert tables == expected_tables, ( + f"{filename} table mismatch: extra={tables - expected_tables}, missing={expected_tables - tables}" + ) diff --git a/server/lomas_server/admin_database/admin_database.py b/server/lomas_server/admin_database/admin_database.py index d78c32150..187593c7d 100644 --- a/server/lomas_server/admin_database/admin_database.py +++ b/server/lomas_server/admin_database/admin_database.py @@ -491,7 +491,7 @@ def get_bootstrap_disabled(self) -> bool: def backup(self) -> bytes: """Creates a backup of the database and returns it as a Zip archive. - The backup is a zip archive containing snapshots of the underlying storage (db, archives, misc). + The backup is a zip archive containing snapshots of the underlying storage (db, archives). It can be stored locally or in a S3. Returns: diff --git a/server/lomas_server/admin_database/local_database.py b/server/lomas_server/admin_database/local_database.py index 5470e23f6..77c652e92 100644 --- a/server/lomas_server/admin_database/local_database.py +++ b/server/lomas_server/admin_database/local_database.py @@ -5,7 +5,6 @@ from collections.abc import Generator from contextlib import AbstractContextManager, closing, contextmanager, nullcontext from datetime import UTC, datetime -from functools import wraps from pathlib import Path from tempfile import SpooledTemporaryFile, TemporaryDirectory from typing import Any, BinaryIO, override @@ -974,7 +973,7 @@ def get_bootstrap_disabled(self) -> bool: def _sqlite_paths_to_backup(self) -> list[Path]: """Paths of the sqlite files that make up the database state to snapshot.""" - return [self._db_path, self._archives_db_path, self._misc_db_path] + return [self._db_path, self._archives_db_path] @staticmethod def _snapshot_sqlite_file(src_path: Path, dest_path: Path) -> None: From 924fb0527e70fb621c17548075f04d82654ee8ef Mon Sep 17 00:00:00 2001 From: Damien Date: Thu, 3 Sep 2026 14:59:01 +0200 Subject: [PATCH 17/20] integrate backup changes to helm chart --- .github/workflows/push_docker.yml | 2 +- .../lomas/templates/server/_helpers.tpl | 21 ++++++++ .../lomas/templates/server/data_pvc.yaml | 23 --------- deploy/charts/lomas/templates/server/pvc.yaml | 50 ++++++++++++++++++- .../lomas/templates/server/secrets.yaml | 11 ++++ .../templates/server/server_deployment.yaml | 19 +++++++ deploy/charts/lomas/values.yaml | 18 ++++++- server/lomas_server/models/config.py | 4 +- 8 files changed, 120 insertions(+), 28 deletions(-) delete mode 100644 deploy/charts/lomas/templates/server/data_pvc.yaml diff --git a/.github/workflows/push_docker.yml b/.github/workflows/push_docker.yml index b0fcdb8b8..3e5b7dedc 100644 --- a/.github/workflows/push_docker.yml +++ b/.github/workflows/push_docker.yml @@ -43,7 +43,7 @@ jobs: nix build .#lomas-oci-raw -o lomas-oci-raw.tar - name: Push Image - if: github.event_name == 'push' || github.event_name == 'release' + # if: github.event_name == 'push' || github.event_name == 'release' run: | for tag in $DOCKER_METADATA_OUTPUT_TAGS; do skopeo copy docker-archive:lomas-oci.tar docker://dsccadminch/lomas:${tag##*:} diff --git a/deploy/charts/lomas/templates/server/_helpers.tpl b/deploy/charts/lomas/templates/server/_helpers.tpl index 46ac7397e..bad38890a 100644 --- a/deploy/charts/lomas/templates/server/_helpers.tpl +++ b/deploy/charts/lomas/templates/server/_helpers.tpl @@ -37,6 +37,9 @@ app.kubernetes.io/component: {{ include "lomas.worker.name" . }} {{- define "lomas.server.dataPVCName" -}} {{- printf "%s-%s" (include "lomas.server.fullname" .) "data" }} {{- end}} +{{- define "lomas.server.backupPVCName" -}} +{{- printf "%s-%s" (include "lomas.server.fullname" .) "backup" }} +{{- end}} {{- define "lomas.server.dbPVCName" -}} {{- printf "%s-%s" (include "lomas.server.fullname" .) "db" }} {{- end}} @@ -79,6 +82,24 @@ app.kubernetes.io/component: {{ include "lomas.worker.name" . }} {{- end -}} {{- end -}} +{{/* s3 backup uri secret */}} +{{- define "lomas.server.s3BackupUriSecretName" -}} +{{- $secretName := .Values.server.runtime_args.s3Backup.uri.existingSecret -}} +{{- if $secretName -}} + {{- printf "%s" (tpl $secretName $) -}} +{{- else -}} + {{- printf "%s-server-s3-backup-uri-secret" (include "lomas.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "lomas.server.s3BackupUriSecretKey" -}} + {{- if and .Values.server.runtime_args.s3Backup.uri.existingSecret .Values.server.runtime_args.s3Backup.uri.existingKey -}} + {{- printf "%s" (tpl .Values.server.runtime_args.s3Backup.uri.existingKey $) -}} + {{- else -}} + {{- printf "s3-backup-uri" -}} + {{- end -}} +{{- end -}} + {{/* Private DB credentials */}} {{- define "lomas.server.private-db-credentials-secrets" -}} {{- $result := list }} diff --git a/deploy/charts/lomas/templates/server/data_pvc.yaml b/deploy/charts/lomas/templates/server/data_pvc.yaml deleted file mode 100644 index d45a7e3ff..000000000 --- a/deploy/charts/lomas/templates/server/data_pvc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.server.pvc.data.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "lomas.server.dataPVCName" . }} - labels: - {{- include "lomas.server.labels" . | nindent 4 }} - annotations: - {{- if .Values.server.pvc.data.persistence.resourcePolicy }} - "helm.sh/resource-policy": {{ .Values.server.pvc.data.persistence.resourcePolicy }} - {{- end }} -spec: - accessModes: - - {{ .Values.server.pvc.data.accessMode }} - {{- if .Values.server.pvc.data.storageClassName }} - storageClassName: {{ .Values.server.pvc.data.storageClassName | quote }} - {{- end }} - resources: - limits: - storage: {{ .Values.server.pvc.data.sizeLimit | quote }} - requests: - storage: {{ .Values.server.pvc.data.sizeRequest | quote }} -{{- end }} \ No newline at end of file diff --git a/deploy/charts/lomas/templates/server/pvc.yaml b/deploy/charts/lomas/templates/server/pvc.yaml index 572a267ac..24f55abc5 100644 --- a/deploy/charts/lomas/templates/server/pvc.yaml +++ b/deploy/charts/lomas/templates/server/pvc.yaml @@ -18,4 +18,52 @@ spec: limits: storage: {{ .Values.server.pvc.db.sizeLimit | quote }} requests: - storage: {{ .Values.server.pvc.db.sizeRequest | quote }} \ No newline at end of file + storage: {{ .Values.server.pvc.db.sizeRequest | quote }} +--- +{{- if .Values.server.pvc.data.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "lomas.server.dataPVCName" . }} + labels: + {{- include "lomas.server.labels" . | nindent 4 }} + annotations: + {{- if .Values.server.pvc.data.persistence.resourcePolicy }} + "helm.sh/resource-policy": {{ .Values.server.pvc.data.persistence.resourcePolicy }} + {{- end }} +spec: + accessModes: + - {{ .Values.server.pvc.data.accessMode }} + {{- if .Values.server.pvc.data.storageClassName }} + storageClassName: {{ .Values.server.pvc.data.storageClassName | quote }} + {{- end }} + resources: + limits: + storage: {{ .Values.server.pvc.data.sizeLimit | quote }} + requests: + storage: {{ .Values.server.pvc.data.sizeRequest | quote }} +{{- end }} +--- +{{- if .Values.server.pvc.backup.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "lomas.server.backupPVCName" . }} + labels: + {{- include "lomas.server.labels" . | nindent 4 }} + annotations: + {{- if .Values.server.pvc.backup.persistence.resourcePolicy }} + "helm.sh/resource-policy": {{ .Values.server.pvc.backup.persistence.resourcePolicy }} + {{- end }} +spec: + accessModes: + - {{ .Values.server.pvc.backup.accessMode }} + {{- if .Values.server.pvc.backup.storageClassName }} + storageClassName: {{ .Values.server.pvc.backup.storageClassName | quote }} + {{- end }} + resources: + limits: + storage: {{ .Values.server.pvc.backup.sizeLimit | quote }} + requests: + storage: {{ .Values.server.pvc.backup.sizeRequest | quote }} +{{- end }} \ No newline at end of file diff --git a/deploy/charts/lomas/templates/server/secrets.yaml b/deploy/charts/lomas/templates/server/secrets.yaml index dad8f411a..c92bacc38 100644 --- a/deploy/charts/lomas/templates/server/secrets.yaml +++ b/deploy/charts/lomas/templates/server/secrets.yaml @@ -21,3 +21,14 @@ type: Opaque data: {{ include "lomas.server.workerApiKeySecretKey" . }}: {{ required ".Values.server.runtime_args.worker_api_key.value or existing secret must be set." .Values.server.runtime_args.worker_api_key.value | b64enc }} {{- end }} +--- +{{- if and .Values.server.runtime_args.s3Backup.enabled (not .Values.server.runtime_args.s3Backup.uri.existingSecret) }} +kind: Secret +metadata: + name: {{ include "lomas.server.s3BackupUriSecretName" . }} + labels: + {{ include "lomas.labels" . | nindent 4}} +type: Opaque +data: + {{ include "lomas.server.s3BackupUriSecretKey" . }}: {{ required ".Values.server.runtime_args.s3Backup.uri.value or existing secret must be set." .Values.server.runtime_args.s3Backup.uri.value | b64enc }} +{{- end }} diff --git a/deploy/charts/lomas/templates/server/server_deployment.yaml b/deploy/charts/lomas/templates/server/server_deployment.yaml index 61d72ecd1..b33a4eb8e 100644 --- a/deploy/charts/lomas/templates/server/server_deployment.yaml +++ b/deploy/charts/lomas/templates/server/server_deployment.yaml @@ -42,6 +42,10 @@ spec: volumeMounts: - name: db mountPath: /db + {{- if .Values.server.pvc.backup.enabled }} + - name: backup + mountPath: /backup + {{- end }} {{- if .Values.server.pvc.data.enabled }} - name: data mountPath: /data @@ -84,6 +88,16 @@ spec: value: "/db/" - name: LOMAS_SERVER_CLEAN_ADMIN_DATABASE value: "{{ .Values.server.runtime_args.clean_admin_database }}" + {{- if not .Values.server.runtime_args.s3Backup.enabled }} + - name: LOMAS_SERVER_BACKUP__LOCAL_DIRECTORY + value: "/backup/" + {{- else }} + - name: LOMAS_SERVER_BACKUP__URI + valueFrom: + secretKeyRef: + name: {{ include "lomas.server.s3BackupUriSecretName" . }} + key: {{ include "lomas.server.s3BackupUriSecretKey" . }} + {{- end }} - name: LOMAS_SERVER_DATA_DIRECTORY value: "/data" - name: LOMAS_SERVER_AUTHENTICATOR__AUTHENTICATION_TYPE @@ -126,6 +140,11 @@ spec: persistentVolumeClaim: claimName: {{ include "lomas.server.dataPVCName" . }} {{- end }} + {{- if .Values.server.pvc.backup.enabled }} + - name: backup + persistentVolumeClaim: + claimName: {{ include "lomas.server.backupPVCName" . }} + {{- end }} {{- if .Values.global.configCABundle.enabled }} - name: trusted-cabundle configMap: diff --git a/deploy/charts/lomas/values.yaml b/deploy/charts/lomas/values.yaml index 62aa994e6..1cde5d412 100644 --- a/deploy/charts/lomas/values.yaml +++ b/deploy/charts/lomas/values.yaml @@ -60,6 +60,12 @@ server: query_userinfo: true authentication_type: oidc clean_admin_database: false # careful! + s3Backup: + enable: false # If not enabled, local path backup will be enabled (see pvc.backup) + uri: + existingSecret: "" + existingSecretKey: "" + value: "" pvc: db: # used for sqlite admin db storageClassName: "" # is not set if empty string @@ -70,9 +76,19 @@ server: # If set to "keep", sets the helm/resource-policy of the pvc to keep, # so that the pvc is not deleted across reinstalls. resourcePolicy: "" + backup: # used for local database backups + enabled: true + storageClassName: "" # is not set if empty string + sizeLimit: 2Gi + sizeRequest: 1Gi + accessMode: "ReadWriteOnce" + persistence: + # If set to "keep", sets the helm/resource-policy of the pvc to keep, + # so that the pvc is not deleted across reinstalls. + resourcePolicy: "" data: # used for storing data (e.g. csv), mounted to /data at both server and worker. enabled: true - storageClassName: "nas-ssd-encrypt" # is not set if empty string + storageClassName: "" # is not set if empty string sizeLimit: 2Gi sizeRequest: 1Gi accessMode: "ReadWriteMany" diff --git a/server/lomas_server/models/config.py b/server/lomas_server/models/config.py index d3ca436f2..db8fa300e 100644 --- a/server/lomas_server/models/config.py +++ b/server/lomas_server/models/config.py @@ -131,8 +131,6 @@ class Config(BaseSettings): private_db_credentials: dict[int, Annotated[S3CredentialsConfig, Field(discriminator="db_type")]] = {} - backup: BackupConfig = Field(default=LocalBackupConfig(local_directory="/tmp/lomas-backups")) - opendp_features: OpenDPFeatures = Field(default=["contrib", "idealized-numerics", "honest-but-curious"]) telemetry: Telemetry = Field(default_factory=Telemetry, description=CLI_SUPPRESS) @@ -166,6 +164,8 @@ class ServerConfig(Config): clean_admin_database: bool = Field(default=False) + backup: BackupConfig = Field(default=LocalBackupConfig(local_directory="/tmp/lomas-backups")) + data_directory: Path = Field(default=Path("../data")) @computed_field From 0afa52e9014e56e27d7b230351c4b6267730f7db Mon Sep 17 00:00:00 2001 From: Damien Date: Thu, 3 Sep 2026 15:33:26 +0200 Subject: [PATCH 18/20] bug fix for dashboard --- .../administration/dashboard/database_administration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/lomas_server/administration/dashboard/database_administration.py b/server/lomas_server/administration/dashboard/database_administration.py index 6b763b795..dc4f92a4d 100644 --- a/server/lomas_server/administration/dashboard/database_administration.py +++ b/server/lomas_server/administration/dashboard/database_administration.py @@ -531,7 +531,7 @@ def del_all_lomas_users() -> ResultE: st.title("Backup") st.write("Creates a backup of the admin database (jobs, users, archives).") if st.button("Backup now", key="btn_backup_admin_db"): - match query_lomas_auth("/backup", httpx2.post): + match query_lomas_auth("/backup", httpx2.get): case Success(backup_info): size_kb = backup_info["size_bytes"] / 1024 st.success(f"Backup written to `{backup_info['location']}` ({size_kb:.1f} KB).") From aeee56e1bb871fc63dcbf6e8ccc6480725990bac Mon Sep 17 00:00:00 2001 From: Damien Date: Thu, 3 Sep 2026 16:57:00 +0200 Subject: [PATCH 19/20] templates/server/secrets.yaml --- deploy/charts/lomas/templates/server/secrets.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/charts/lomas/templates/server/secrets.yaml b/deploy/charts/lomas/templates/server/secrets.yaml index c92bacc38..a824c7256 100644 --- a/deploy/charts/lomas/templates/server/secrets.yaml +++ b/deploy/charts/lomas/templates/server/secrets.yaml @@ -23,6 +23,7 @@ data: {{- end }} --- {{- if and .Values.server.runtime_args.s3Backup.enabled (not .Values.server.runtime_args.s3Backup.uri.existingSecret) }} +apiVersion: v1 kind: Secret metadata: name: {{ include "lomas.server.s3BackupUriSecretName" . }} From 3387849c00716960737d8bfebbc552d391f8db7c Mon Sep 17 00:00:00 2001 From: LancelotMarti Date: Thu, 3 Sep 2026 17:19:42 +0200 Subject: [PATCH 20/20] Fix aws checksum issue --- deploy/charts/lomas/templates/server/server_deployment.yaml | 4 ++++ devenv.nix | 3 +++ 2 files changed, 7 insertions(+) diff --git a/deploy/charts/lomas/templates/server/server_deployment.yaml b/deploy/charts/lomas/templates/server/server_deployment.yaml index b33a4eb8e..9acedd3de 100644 --- a/deploy/charts/lomas/templates/server/server_deployment.yaml +++ b/deploy/charts/lomas/templates/server/server_deployment.yaml @@ -97,6 +97,10 @@ spec: secretKeyRef: name: {{ include "lomas.server.s3BackupUriSecretName" . }} key: {{ include "lomas.server.s3BackupUriSecretKey" . }} + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: "when_required" + - name: AWS_RESPONSE_CHECKSUM_VALIDATION + value: "when_required" {{- end }} - name: LOMAS_SERVER_DATA_DIRECTORY value: "/data" diff --git a/devenv.nix b/devenv.nix index b7c3105ca..879f3aa00 100644 --- a/devenv.nix +++ b/devenv.nix @@ -208,6 +208,9 @@ in with config.lomas.garage; "http://${keyId}:${secretKey}@${host}:${toString port}/bucket/backup"; + AWS_REQUEST_CHECKSUM_CALCULATION = "when_required"; + AWS_RESPONSE_CHECKSUM_VALIDATION = "when_required"; + # Lomas Runtime LOMAS_SERVER_log_level = "INFO"; LOMAS_SERVER_lomas_log_level = "DEBUG";