feat(deployments): switch ocis_full example to bind-mounted ./data/ volumes

.gitignore

@@ -26,6 +26,9 @@ ocis/cmd/ocis/config/
 /hugo
 /docs/services/*/_index.md
 
+# deployment example runtime data (bind-mounted volumes)
+/deployments/examples/ocis_full/data/
+
 # IDEs
 .idea
 

deployments/examples/ocis_full/.env

@@ -65,7 +65,8 @@ LOG_LEVEL=
 #
 # Define the oCIS storage location. Set the paths for config and data to a local path.
 # Note that especially the data directory can grow big.
-# Leaving it default stores data in docker internal volumes.
+# Leaving it default stores data under ./data/ocis-config and ./data/ocis-data
+# relative to the compose project directory.
 # For more details see:
 # https://doc.owncloud.com/ocis/next/deployment/general/general-info.html#default-paths
 # OCIS_CONFIG_DIR=/your/local/ocis/config
@@ -134,11 +135,11 @@ START_ADDITIONAL_SERVICES="notifications"
 # It is possible to use the oCIS Web Extensions to add custom functionality to the oCIS frontend.
 # For more details see https://github.com/owncloud/web-extensions/blob/main/README.md
 # Note: the leading colon is required to enable the service.
-# Enable this to create a new named volume
+# Enable this to store app extension files under ./data/ocis-apps
 #EXTENSIONS=:web_extensions/extensions.yml
 # Enable the desired extensions by uncommenting the following lines.
 # Note: the leading colon is required to enable the service.
-# Note: if you want to remove a web extension, you must delete the ocis-apps volume. It will be properly recreated on docker compose startup.
+# Note: if you want to remove a web extension, you must delete the ./data/ocis-apps directory. It will be properly recreated on docker compose startup.
 #UNZIP=:web_extensions/unzip.yml
 #DRAWIO=:web_extensions/drawio.yml
 #JSONVIEWER=:web_extensions/jsonviewer.yml

deployments/examples/ocis_full/README.md

@@ -15,6 +15,18 @@ This deployment example is documented in two locations for different audiences:
   Providing details which are more developer focused. This description can also be used when deviating from the default.\
   Note that this examples uses self signed certificates and is intended for testing purposes.
 
+## Data Storage
+
+All persistent runtime data (oCIS config/data, S3 storage, database, virus
+definitions, web app extensions, etc.) is bind-mounted from a `./data/`
+directory next to this compose project, rather than docker-managed named
+volumes. The oCIS config/data paths can be redirected elsewhere via the
+`OCIS_CONFIG_DIR`/`OCIS_DATA_DIR` variables in `.env`.
+
+Upgrading an existing deployment that still has data in the old named
+volumes? Run `./migrate-volumes.sh` once, before starting the stack, to copy
+data from the old volumes into the new `./data/` directories.
+
 ## Optional Services
 
 ### Keycloak

deployments/examples/ocis_full/clamav.yml

@@ -10,6 +10,9 @@ services:
       # PROXY_TLS is set to "false", the download url has no https
       STORAGE_USERS_DATA_GATEWAY_URL: http://ocis:9200/data
     volumes:
+      # a named volume is used here instead of a bind mount: it only ever
+      # holds the runtime clamd.sock and some Docker Desktop hosts don't
+      # reliably support AF_UNIX sockets on bind-mounted paths
       - "clamav-socket:/var/run/clamav"
 
   clamav:
@@ -19,11 +22,10 @@ services:
       ocis-net:
     volumes:
       - "clamav-socket:/tmp"
-      - "clamav-db:/var/lib/clamav"
+      - "./data/clamav-db:/var/lib/clamav"
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always
 
 volumes:
   clamav-socket:
-  clamav-db:

deployments/examples/ocis_full/collabora.yml

@@ -36,8 +36,8 @@ services:
       COLLABORATION_LOG_LEVEL: ${LOG_LEVEL:-info}
       OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
     volumes:
-      # configure the .env file to use own paths instead of docker internal volumes
-      - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis
+      # configure the .env file to use different host paths if desired
+      - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.collaboration.entrypoints=http"

deployments/examples/ocis_full/docker-compose.yml

@@ -39,7 +39,7 @@ services:
       - "443:443"
     volumes:
       - "${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock:ro"
-      - "certs:/certs"
+      - "./data/certs:/certs"
     labels:
       - "traefik.enable=${TRAEFIK_DASHBOARD:-false}"
        # defaults to admin:admin
@@ -53,8 +53,5 @@ services:
       driver: ${LOG_DRIVER:-local}
     restart: always
 
-volumes:
-  certs:
-
 networks:
   ocis-net:

deployments/examples/ocis_full/keycloak.yml

@@ -27,7 +27,7 @@ services:
     networks:
       ocis-net:
     volumes:
-      - keycloak_postgres_data:/var/lib/postgresql
+      - ./data/keycloak-postgres:/var/lib/postgresql
     environment:
       POSTGRES_DB: keycloak
       POSTGRES_USER: keycloak
@@ -78,6 +78,3 @@ services:
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always
-
-volumes:
-  keycloak_postgres_data:

deployments/examples/ocis_full/migrate-volumes.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+# One-time helper for upgrading an existing ocis_full deployment from the old
+# docker-managed named volumes to the new ./data/ bind mounts.
+#
+# Docker named volumes are not directly accessible from the host, so for each
+# volume that still exists we spin up a throwaway container that mounts both
+# the old volume and the new host directory, and copy the data across.
+#
+# Safe to re-run: volumes that no longer exist (fresh installs) are skipped,
+# and targets that already contain data are left untouched.
+#
+# Usage: run this from the ocis_full example directory, before starting the
+# stack with `docker compose up` for the first time after upgrading:
+#   ./migrate-volumes.sh
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+# shellcheck disable=SC1091
+[ -f .env ] && source .env
+
+MIGRATIONS=(
+  "ocis-config:${OCIS_CONFIG_DIR:-./data/ocis-config}"
+  "ocis-data:${OCIS_DATA_DIR:-./data/ocis-data}"
+  "ocis-apps:./data/ocis-apps"
+  "keycloak_postgres_data:./data/keycloak-postgres"
+  "minio-data:./data/minio"
+  "clamav-db:./data/clamav-db"
+  "companion-data:./data/companion"
+)
+
+for entry in "${MIGRATIONS[@]}"; do
+  vol="${entry%%:*}"
+  target="${entry#*:}"
+
+  if ! docker volume inspect "$vol" >/dev/null 2>&1; then
+    echo "skip ${vol}: no such docker volume (fresh install?)"
+    continue
+  fi
+
+  mkdir -p "$target"
+  if [ -n "$(ls -A "$target" 2>/dev/null)" ]; then
+    echo "skip ${vol}: ${target} already has data, not overwriting"
+    continue
+  fi
+
+  docker run --rm \
+    -v "${vol}:/from" \
+    -v "$(cd "$target" && pwd):/to" \
+    alpine sh -c "cp -a /from/. /to/"
+  echo "migrated ${vol} -> ${target}"
+done
+
+vol_names=()
+for entry in "${MIGRATIONS[@]}"; do
+  vol_names+=("${entry%%:*}")
+done
+
+echo
+echo "Done. Verify the deployment works with the new ./data/ paths, then you"
+echo "can remove the old named volumes with, e.g.:"
+echo "  docker volume rm ${vol_names[*]}"

deployments/examples/ocis_full/minio.yml

@@ -13,7 +13,7 @@ services:
         "mkdir -p /data/${S3NG_BUCKET:-ocis-bucket} && minio server --console-address ':9001' /data",
       ]
     volumes:
-      - minio-data:/data
+      - ./data/minio:/data
     environment:
       MINIO_ACCESS_KEY: ${S3NG_ACCESS_KEY:-ocis}
       MINIO_SECRET_KEY: ${S3NG_SECRET_KEY:-ocis-secret-key}
@@ -27,6 +27,3 @@ services:
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always
-
-volumes:
-  minio-data:

deployments/examples/ocis_full/ocis.yml

@@ -5,12 +5,35 @@ services:
       ocis-net:
         aliases:
           - ${OCIS_DOMAIN:-ocis.owncloud.test}
+  # chowns the host-mounted config/data dirs to the oCIS container's non-root
+  # user (uid/gid 1000) before oCIS itself starts, since bind mounts default
+  # to being owned by whoever created them on the host. Skips the recursive
+  # chown on subsequent startups (marker file) so it doesn't re-walk the
+  # whole data dir, which can grow big, on every restart.
+  ocis-data-init:
+    image: alpine:latest
+    user: root
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >-
+        for d in /etc/ocis /var/lib/ocis; do
+        mkdir -p "$$d";
+        m="$$d/.ocis-data-init-done";
+        [ -f "$$m" ] || { chown -R 1000:1000 "$$d"; touch "$$m"; };
+        done
+    volumes:
+      - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis
+      - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis
+
   ocis:
     image: ${OCIS_DOCKER_IMAGE}:${OCIS_DOCKER_TAG}
     # changelog: https://github.com/owncloud/ocis/tree/master/changelog
     # release notes: https://doc.owncloud.com/ocis_release_notes.html
     networks:
       ocis-net:
+    depends_on:
+      ocis-data-init:
+        condition: service_completed_successfully
     entrypoint:
       - /bin/sh
     # run ocis init to initialize a configuration file with random secrets
@@ -57,9 +80,9 @@ services:
       - ./config/ocis/app-registry.yaml:/etc/ocis/app-registry.yaml
       - ./config/ocis/csp.yaml:/etc/ocis/csp.yaml
       - ./config/ocis/banned-password-list.txt:/etc/ocis/banned-password-list.txt
-      # configure the .env file to use own paths instead of docker internal volumes
-      - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis
-      - ${OCIS_DATA_DIR:-ocis-data}:/var/lib/ocis
+      # configure the .env file to use different host paths if desired
+      - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis
+      - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.ocis.entrypoints=https"
@@ -70,7 +93,3 @@ services:
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always
-
-volumes:
-  ocis-config:
-  ocis-data:

deployments/examples/ocis_full/onlyoffice.yml

@@ -31,8 +31,8 @@ services:
       COLLABORATION_LOG_LEVEL: ${LOG_LEVEL:-info}
       OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test}
     volumes:
-      # configure the .env file to use own paths instead of docker internal volumes
-      - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis
+      # configure the .env file to use different host paths if desired
+      - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.collaboration-oo.entrypoints=http"

deployments/examples/ocis_full/vault-storage.yml

@@ -32,9 +32,9 @@ services:
       STORAGE_USERS_OCIS_ROOT: /var/lib/ocis/storage/users-vault
       STORAGE_USERS_EVENTS_CONSUMER_GROUP: vault-dcfs
     volumes:
-      # configure the .env file to use own paths instead of docker internal volumes
-      - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis
-      - ${OCIS_DATA_DIR:-ocis-data}:/var/lib/ocis
+      # configure the .env file to use different host paths if desired
+      - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis
+      - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always

deployments/examples/ocis_full/web_extensions/advanced-search.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:advanced-search-0.3.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/advanced-search/ /apps"]

deployments/examples/ocis_full/web_extensions/drawio.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:draw-io-0.4.1
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/draw-io/ /apps"]

deployments/examples/ocis_full/web_extensions/extensions.yml

@@ -1,7 +1,4 @@
 services:
   ocis:
     volumes:
-      - ocis-apps:/var/lib/ocis/web/assets/apps
-
-volumes:
-  ocis-apps:
+      - ./data/ocis-apps:/var/lib/ocis/web/assets/apps

deployments/examples/ocis_full/web_extensions/externalsites.yml

@@ -11,7 +11,7 @@ services:
     image: owncloud/web-extensions:external-sites-0.4.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/external-sites/ /apps"]

deployments/examples/ocis_full/web_extensions/importer.yml

@@ -17,7 +17,7 @@ services:
     image: owncloud/web-extensions:importer-0.4.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: [ "-c", "cp -R /var/lib/nginx/html/importer/ /apps" ]
@@ -38,7 +38,7 @@ services:
       COMPANION_TUS_DEFERRED_UPLOAD_LENGTH: 'false'
       COMPANION_CLIENT_ORIGINS: "https://${OCIS_DOMAIN:-ocis.owncloud.test},https://${COMPANION_DOMAIN:-companion.owncloud.test}"
     volumes:
-      - companion-data:/tmp/companion/
+      - ./data/companion:/tmp/companion/
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.companion.entrypoints=https"
@@ -49,6 +49,3 @@ services:
     logging:
       driver: ${LOG_DRIVER:-local}
     restart: always
-
-volumes:
-  companion-data:

deployments/examples/ocis_full/web_extensions/jsonviewer.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:json-viewer-0.4.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/json-viewer/ /apps"]

deployments/examples/ocis_full/web_extensions/photo-addon.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:photo-addon-0.3.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/photo-addon/ /apps"]

deployments/examples/ocis_full/web_extensions/progressbars.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:progress-bars-0.4.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/progress-bars/ /apps"]

deployments/examples/ocis_full/web_extensions/unzip.yml

@@ -9,7 +9,7 @@ services:
     image: owncloud/web-extensions:unzip-0.5.0
     user: root
     volumes:
-      - ocis-apps:/apps
+      - ./data/ocis-apps:/apps
     entrypoint:
       - /bin/sh
     command: ["-c", "cp -R /var/lib/nginx/html/unzip/ /apps"]