Wip backup sqlite

.github/workflows/push_docker.yml

@@ -43,7 +43,7 @@ jobs:
           nix build .#lomas-oci-raw -o lomas-oci-raw.tar
 
       - name: Push Image
-        if: github.event_name == 'push' || github.event_name == 'release'
+        # if: github.event_name == 'push' || github.event_name == 'release'
         run: |
           for tag in $DOCKER_METADATA_OUTPUT_TAGS; do
             skopeo copy docker-archive:lomas-oci.tar docker://dsccadminch/lomas:${tag##*:}

client/lomas_client/tests/test_integrations.py

@@ -1,8 +1,13 @@
 import io
+import os
 import re
+import sqlite3
 import sys
+import tempfile
 import time
+import zipfile
 from dataclasses import dataclass
+from pathlib import Path
 from urllib.parse import urljoin
 
 import numpy as np
@@ -15,7 +20,9 @@
 from bs4 import BeautifulSoup
 from csvw_eo.constants import COL_NAME, TABLE_SCHEMA
 from diffprivlib import models
+from fastapi.testclient import TestClient
 from opendp.mod import enable_features
+from pydantic import ValidationError
 from sklearn.pipeline import Pipeline
 
 from lomas_client import Client
@@ -27,7 +34,8 @@
     del_all_dex_users,
 )
 from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup
-from lomas_server.models.config import AdminConfig, ServerConfig
+from lomas_server.app import get_admin_app
+from lomas_server.models.config import AdminConfig, BackupS3Config, LocalBackupConfig, ServerConfig
 
 enable_features("contrib")
 
@@ -415,3 +423,85 @@ def test_demo_opendp_polars(dex_config, demo_setup) -> None:
     assert response_archives is not None
     assert response_archives.epsilon == DEFAULT_EPSILON
     assert response_archives.delta == pytest.approx(0.0, abs=0.1)
+
+
+def test_backup():
+    # With S3
+    config = ServerConfig()
+    config.database.wipe()
+    config.database.set_bootstrap(config.bootstrap)
+
+    with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client:
+        response = client.get("/backup")
+        body = response.json()
+        assert body["is_s3"] is True
+        assert body["location"].startswith("s3://")
+
+    # Raise an error if uri isn't correctly defined
+    # For instance missing user:password / bucket_name, etc.
+    with pytest.raises(ValidationError):
+        ServerConfig(backup=BackupS3Config(uri="https://localhost:3900/bucket"))
+
+    # With local_directory setup
+    config = ServerConfig(backup=LocalBackupConfig(local_directory="/tmp/lomas-custom-backups"))
+    with TestClient(get_admin_app(config), headers={"Authorization": f"Bearer {config.bootstrap}"}) as client:
+        # Create one query to test backup content
+        lomas_demo_setup()
+        user_name = "Jack"
+        client_u = Client(
+            user_name=f"{user_name}@example.com", user_password=user_name.lower(), dataset_name="TITANIC"
+        )
+
+        context = client_u.get_context(epsilon=DEFAULT_EPSILON)
+        plan = context.query().select(pl.col("Age").dp.mean(bounds=(0, 120)), dp.len())
+        client_u.opendp.query(plan, epsilon=DEFAULT_EPSILON)
+
+        # Backup bew db state
+        response = client.get("/backup")
+        body = response.json()
+
+        # Check if custom location works
+        assert body["is_s3"] is False
+        assert body["location"].startswith("/tmp/lomas-custom-backups/")
+        assert os.path.exists(body["location"])
+
+        # Test that we have correct tables saved in backup
+        # Load each sqlite db out of the backup zip and check its tables
+        backup_path = Path(body["location"])
+        expected_tables_by_file = {
+            "db.sqlite3": {"jobs", "users", "misc", "datasets"},
+            "archives.sqlite3": {"archives"},
+        }
+
+        with zipfile.ZipFile(backup_path) as archive, tempfile.TemporaryDirectory() as extract_dir:
+            assert set(archive.namelist()) == set(expected_tables_by_file)
+
+            for filename, expected_tables in expected_tables_by_file.items():
+                db_path = Path(extract_dir) / filename
+                db_path.write_bytes(archive.read(filename))
+
+                conn = sqlite3.connect(db_path)
+                try:
+                    # Check tables are correctly saved in each db
+                    tables = {
+                        row[0]
+                        for row in conn.execute(
+                            "SELECT name FROM sqlite_master WHERE type='table'"
+                        ).fetchall()
+                    }
+
+                    # Check that the client query is saved in backup
+                    if tables == {"archives"}:
+                        row = conn.execute(
+                            "SELECT uid, user_name, dataset_name, status FROM archives;"
+                        ).fetchone()
+
+                        assert row[1] == user_name
+                        assert row[2] == "TITANIC"
+                        assert row[3] == "complete"
+
+                finally:
+                    conn.close()
+                assert tables == expected_tables, (
+                    f"{filename} table mismatch: extra={tables - expected_tables}, missing={expected_tables - tables}"
+                )

deploy/charts/lomas/templates/server/_helpers.tpl

@@ -37,6 +37,9 @@ app.kubernetes.io/component: {{ include "lomas.worker.name" . }}
 {{- define "lomas.server.dataPVCName" -}}
 {{- printf "%s-%s" (include "lomas.server.fullname" .) "data" }}
 {{- end}}
+{{- define "lomas.server.backupPVCName" -}}
+{{- printf "%s-%s" (include "lomas.server.fullname" .) "backup" }}
+{{- end}}
 {{- define "lomas.server.dbPVCName" -}}
 {{- printf "%s-%s" (include "lomas.server.fullname" .) "db" }}
 {{- end}}
@@ -79,6 +82,24 @@ app.kubernetes.io/component: {{ include "lomas.worker.name" . }}
     {{- end -}}
 {{- end -}}
 
+{{/* s3 backup uri secret */}}
+{{- define "lomas.server.s3BackupUriSecretName" -}}
+{{- $secretName := .Values.server.runtime_args.s3Backup.uri.existingSecret -}}
+{{- if $secretName -}}
+    {{- printf "%s" (tpl $secretName $) -}}
+{{- else -}}
+    {{- printf "%s-server-s3-backup-uri-secret" (include "lomas.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "lomas.server.s3BackupUriSecretKey" -}}
+    {{- if and .Values.server.runtime_args.s3Backup.uri.existingSecret .Values.server.runtime_args.s3Backup.uri.existingKey -}}
+        {{- printf "%s" (tpl .Values.server.runtime_args.s3Backup.uri.existingKey $) -}}
+    {{- else -}}
+        {{- printf "s3-backup-uri" -}}
+    {{- end -}}
+{{- end -}}
+
 {{/* Private DB credentials */}}
 {{- define "lomas.server.private-db-credentials-secrets" -}}
 {{- $result := list }}

deploy/charts/lomas/templates/server/pvc.yaml

@@ -18,4 +18,52 @@ spec:
     limits:
       storage: {{ .Values.server.pvc.db.sizeLimit | quote }}
     requests:
-      storage: {{ .Values.server.pvc.db.sizeRequest | quote }}
+      storage: {{ .Values.server.pvc.db.sizeRequest | quote }}
+---
+{{- if .Values.server.pvc.data.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "lomas.server.dataPVCName" . }}
+  labels:
+    {{- include "lomas.server.labels" . | nindent 4 }}
+  annotations:
+    {{- if .Values.server.pvc.data.persistence.resourcePolicy }}
+    "helm.sh/resource-policy": {{ .Values.server.pvc.data.persistence.resourcePolicy }}
+    {{- end }}
+spec:
+  accessModes:
+    - {{ .Values.server.pvc.data.accessMode }}
+  {{- if .Values.server.pvc.data.storageClassName }}
+  storageClassName: {{ .Values.server.pvc.data.storageClassName | quote }}
+  {{- end }}
+  resources:
+    limits:
+      storage: {{ .Values.server.pvc.data.sizeLimit | quote }}
+    requests:
+      storage: {{ .Values.server.pvc.data.sizeRequest | quote }}
+{{- end }}
+---
+{{- if .Values.server.pvc.backup.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "lomas.server.backupPVCName" . }}
+  labels:
+    {{- include "lomas.server.labels" . | nindent 4 }}
+  annotations:
+    {{- if .Values.server.pvc.backup.persistence.resourcePolicy }}
+    "helm.sh/resource-policy": {{ .Values.server.pvc.backup.persistence.resourcePolicy }}
+    {{- end }}
+spec:
+  accessModes:
+    - {{ .Values.server.pvc.backup.accessMode }}
+  {{- if .Values.server.pvc.backup.storageClassName }}
+  storageClassName: {{ .Values.server.pvc.backup.storageClassName | quote }}
+  {{- end }}
+  resources:
+    limits:
+      storage: {{ .Values.server.pvc.backup.sizeLimit | quote }}
+    requests:
+      storage: {{ .Values.server.pvc.backup.sizeRequest | quote }}
+{{- end }}

deploy/charts/lomas/templates/server/secrets.yaml

@@ -21,3 +21,15 @@ type: Opaque
 data:
   {{ include "lomas.server.workerApiKeySecretKey" . }}: {{ required ".Values.server.runtime_args.worker_api_key.value or existing secret must be set." .Values.server.runtime_args.worker_api_key.value | b64enc }}
 {{- end }}
+---
+{{- if and .Values.server.runtime_args.s3Backup.enabled (not .Values.server.runtime_args.s3Backup.uri.existingSecret) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "lomas.server.s3BackupUriSecretName" . }}
+  labels:
+    {{ include "lomas.labels" . | nindent 4}}
+type: Opaque
+data:
+  {{ include "lomas.server.s3BackupUriSecretKey" . }}: {{ required ".Values.server.runtime_args.s3Backup.uri.value or existing secret must be set." .Values.server.runtime_args.s3Backup.uri.value | b64enc }}
+{{- end }}

deploy/charts/lomas/templates/server/server_deployment.yaml

@@ -42,6 +42,10 @@ spec:
         volumeMounts:
           - name: db
             mountPath: /db
+          {{- if .Values.server.pvc.backup.enabled }}
+          - name: backup
+            mountPath: /backup
+          {{- end }}
           {{- if .Values.server.pvc.data.enabled }}
           - name: data
             mountPath: /data
@@ -84,6 +88,20 @@ spec:
             value: "/db/"
           - name: LOMAS_SERVER_CLEAN_ADMIN_DATABASE
             value: "{{ .Values.server.runtime_args.clean_admin_database }}"
+          {{- if not .Values.server.runtime_args.s3Backup.enabled }}
+          - name: LOMAS_SERVER_BACKUP__LOCAL_DIRECTORY
+            value: "/backup/"
+          {{- else }}
+          - name: LOMAS_SERVER_BACKUP__URI
+            valueFrom:
+              secretKeyRef:
+                name: {{ include "lomas.server.s3BackupUriSecretName" . }}
+                key: {{ include "lomas.server.s3BackupUriSecretKey" . }}
+          - name: AWS_REQUEST_CHECKSUM_CALCULATION
+            value: "when_required"
+          - name: AWS_RESPONSE_CHECKSUM_VALIDATION
+            value: "when_required"
+          {{- end }}
           - name: LOMAS_SERVER_DATA_DIRECTORY
             value: "/data"
           - name: LOMAS_SERVER_AUTHENTICATOR__AUTHENTICATION_TYPE
@@ -126,6 +144,11 @@ spec:
           persistentVolumeClaim:
             claimName: {{ include "lomas.server.dataPVCName" . }}
         {{- end }}
+        {{- if .Values.server.pvc.backup.enabled }}
+        - name: backup
+          persistentVolumeClaim:
+            claimName: {{ include "lomas.server.backupPVCName" . }}
+        {{- end }}
         {{- if .Values.global.configCABundle.enabled }}
         - name: trusted-cabundle
           configMap:

deploy/charts/lomas/values.yaml

@@ -60,6 +60,12 @@ server:
       query_userinfo: true
       authentication_type: oidc
     clean_admin_database: false # careful!
+    s3Backup:
+      enable: false # If not enabled, local path backup will be enabled (see pvc.backup)
+      uri:
+        existingSecret: ""
+        existingSecretKey: ""
+        value: ""
   pvc:
     db: # used for sqlite admin db
       storageClassName: "" # is not set if empty string
@@ -70,9 +76,19 @@ server:
         # If set to "keep", sets the helm/resource-policy of the pvc to keep,
         # so that the pvc is not deleted across reinstalls.
         resourcePolicy: ""
+    backup: # used for local database backups
+      enabled: true
+      storageClassName: "" # is not set if empty string
+      sizeLimit: 2Gi
+      sizeRequest: 1Gi
+      accessMode: "ReadWriteOnce"
+      persistence:
+        # If set to "keep", sets the helm/resource-policy of the pvc to keep,
+        # so that the pvc is not deleted across reinstalls.
+        resourcePolicy: ""
     data: # used for storing data (e.g. csv), mounted to /data at both server and worker.
       enabled: true
-      storageClassName: "nas-ssd-encrypt" # is not set if empty string
+      storageClassName: "" # is not set if empty string
       sizeLimit: 2Gi
       sizeRequest: 1Gi
       accessMode: "ReadWriteMany"

devenv.nix

@@ -203,6 +203,14 @@ in
     # Too many unrelated (3party dep warnings for now)
     # PYTHONWARNDEFAULTENCODING = 1;
 
+    # Config for sqlite backup (S3)
+    LOMAS_SERVER_backup__uri =
+      with config.lomas.garage;
+      "http://${keyId}:${secretKey}@${host}:${toString port}/bucket/backup";
+
+    AWS_REQUEST_CHECKSUM_CALCULATION = "when_required";
+    AWS_RESPONSE_CHECKSUM_VALIDATION = "when_required";
+
     # Lomas Runtime
     LOMAS_SERVER_log_level = "INFO";
     LOMAS_SERVER_lomas_log_level = "DEBUG";

server/lomas_server/admin_database/admin_database.py

@@ -486,3 +486,14 @@ def get_bootstrap_disabled(self) -> bool:
         Returns:
             bool: The bootstrap disabled value. False by default if not set in the DB.
         """
+
+    @abstractmethod
+    def backup(self) -> bytes:
+        """Creates a backup of the database and returns it as a Zip archive.
+
+        The backup is a zip archive containing snapshots of the underlying storage (db, archives).
+        It can be stored locally or in a S3.
+
+        Returns:
+            bytes: A zip archive containing the backup.
+        """