diff --git a/.gitignore b/.gitignore index f3617a8a08b..fd013b96582 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ ocis/cmd/ocis/config/ /hugo /docs/services/*/_index.md +# deployment example runtime data (bind-mounted volumes) +/deployments/examples/ocis_full/data/ + # IDEs .idea diff --git a/deployments/examples/ocis_full/.env b/deployments/examples/ocis_full/.env index 8e2902b3cc9..9d3f577f959 100644 --- a/deployments/examples/ocis_full/.env +++ b/deployments/examples/ocis_full/.env @@ -65,7 +65,8 @@ LOG_LEVEL= # # Define the oCIS storage location. Set the paths for config and data to a local path. # Note that especially the data directory can grow big. -# Leaving it default stores data in docker internal volumes. +# Leaving it default stores data under ./data/ocis-config and ./data/ocis-data +# relative to the compose project directory. # For more details see: # https://doc.owncloud.com/ocis/next/deployment/general/general-info.html#default-paths # OCIS_CONFIG_DIR=/your/local/ocis/config @@ -134,11 +135,11 @@ START_ADDITIONAL_SERVICES="notifications" # It is possible to use the oCIS Web Extensions to add custom functionality to the oCIS frontend. # For more details see https://github.com/owncloud/web-extensions/blob/main/README.md # Note: the leading colon is required to enable the service. -# Enable this to create a new named volume +# Enable this to store app extension files under ./data/ocis-apps #EXTENSIONS=:web_extensions/extensions.yml # Enable the desired extensions by uncommenting the following lines. # Note: the leading colon is required to enable the service. -# Note: if you want to remove a web extension, you must delete the ocis-apps volume. It will be properly recreated on docker compose startup. +# Note: if you want to remove a web extension, you must delete the ./data/ocis-apps directory. It will be properly recreated on docker compose startup. #UNZIP=:web_extensions/unzip.yml #DRAWIO=:web_extensions/drawio.yml #JSONVIEWER=:web_extensions/jsonviewer.yml diff --git a/deployments/examples/ocis_full/README.md b/deployments/examples/ocis_full/README.md index e2211b871db..57ba9edad06 100644 --- a/deployments/examples/ocis_full/README.md +++ b/deployments/examples/ocis_full/README.md @@ -15,6 +15,18 @@ This deployment example is documented in two locations for different audiences: Providing details which are more developer focused. This description can also be used when deviating from the default.\ Note that this examples uses self signed certificates and is intended for testing purposes. +## Data Storage + +All persistent runtime data (oCIS config/data, S3 storage, database, virus +definitions, web app extensions, etc.) is bind-mounted from a `./data/` +directory next to this compose project, rather than docker-managed named +volumes. The oCIS config/data paths can be redirected elsewhere via the +`OCIS_CONFIG_DIR`/`OCIS_DATA_DIR` variables in `.env`. + +Upgrading an existing deployment that still has data in the old named +volumes? Run `./migrate-volumes.sh` once, before starting the stack, to copy +data from the old volumes into the new `./data/` directories. + ## Optional Services ### Keycloak diff --git a/deployments/examples/ocis_full/clamav.yml b/deployments/examples/ocis_full/clamav.yml index 39d34e0166d..bc80ae00e45 100644 --- a/deployments/examples/ocis_full/clamav.yml +++ b/deployments/examples/ocis_full/clamav.yml @@ -10,6 +10,9 @@ services: # PROXY_TLS is set to "false", the download url has no https STORAGE_USERS_DATA_GATEWAY_URL: http://ocis:9200/data volumes: + # a named volume is used here instead of a bind mount: it only ever + # holds the runtime clamd.sock and some Docker Desktop hosts don't + # reliably support AF_UNIX sockets on bind-mounted paths - "clamav-socket:/var/run/clamav" clamav: @@ -19,11 +22,10 @@ services: ocis-net: volumes: - "clamav-socket:/tmp" - - "clamav-db:/var/lib/clamav" + - "./data/clamav-db:/var/lib/clamav" logging: driver: ${LOG_DRIVER:-local} restart: always volumes: clamav-socket: - clamav-db: diff --git a/deployments/examples/ocis_full/collabora.yml b/deployments/examples/ocis_full/collabora.yml index c52c032d6f1..a77b9929229 100644 --- a/deployments/examples/ocis_full/collabora.yml +++ b/deployments/examples/ocis_full/collabora.yml @@ -36,8 +36,8 @@ services: COLLABORATION_LOG_LEVEL: ${LOG_LEVEL:-info} OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test} volumes: - # configure the .env file to use own paths instead of docker internal volumes - - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis + # configure the .env file to use different host paths if desired + - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis labels: - "traefik.enable=true" - "traefik.http.routers.collaboration.entrypoints=http" diff --git a/deployments/examples/ocis_full/docker-compose.yml b/deployments/examples/ocis_full/docker-compose.yml index 0cc1c98e37e..14c0388a50d 100644 --- a/deployments/examples/ocis_full/docker-compose.yml +++ b/deployments/examples/ocis_full/docker-compose.yml @@ -39,7 +39,7 @@ services: - "443:443" volumes: - "${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock:ro" - - "certs:/certs" + - "./data/certs:/certs" labels: - "traefik.enable=${TRAEFIK_DASHBOARD:-false}" # defaults to admin:admin @@ -53,8 +53,5 @@ services: driver: ${LOG_DRIVER:-local} restart: always -volumes: - certs: - networks: ocis-net: diff --git a/deployments/examples/ocis_full/keycloak.yml b/deployments/examples/ocis_full/keycloak.yml index 05d75316bc3..e2ba801211c 100644 --- a/deployments/examples/ocis_full/keycloak.yml +++ b/deployments/examples/ocis_full/keycloak.yml @@ -27,7 +27,7 @@ services: networks: ocis-net: volumes: - - keycloak_postgres_data:/var/lib/postgresql + - ./data/keycloak-postgres:/var/lib/postgresql environment: POSTGRES_DB: keycloak POSTGRES_USER: keycloak @@ -78,6 +78,3 @@ services: logging: driver: ${LOG_DRIVER:-local} restart: always - -volumes: - keycloak_postgres_data: diff --git a/deployments/examples/ocis_full/migrate-volumes.sh b/deployments/examples/ocis_full/migrate-volumes.sh new file mode 100755 index 00000000000..b6041545ad6 --- /dev/null +++ b/deployments/examples/ocis_full/migrate-volumes.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# One-time helper for upgrading an existing ocis_full deployment from the old +# docker-managed named volumes to the new ./data/ bind mounts. +# +# Docker named volumes are not directly accessible from the host, so for each +# volume that still exists we spin up a throwaway container that mounts both +# the old volume and the new host directory, and copy the data across. +# +# Safe to re-run: volumes that no longer exist (fresh installs) are skipped, +# and targets that already contain data are left untouched. +# +# Usage: run this from the ocis_full example directory, before starting the +# stack with `docker compose up` for the first time after upgrading: +# ./migrate-volumes.sh + +set -euo pipefail + +cd "$(dirname "$0")" + +# shellcheck disable=SC1091 +[ -f .env ] && source .env + +MIGRATIONS=( + "ocis-config:${OCIS_CONFIG_DIR:-./data/ocis-config}" + "ocis-data:${OCIS_DATA_DIR:-./data/ocis-data}" + "ocis-apps:./data/ocis-apps" + "keycloak_postgres_data:./data/keycloak-postgres" + "minio-data:./data/minio" + "clamav-db:./data/clamav-db" + "companion-data:./data/companion" +) + +for entry in "${MIGRATIONS[@]}"; do + vol="${entry%%:*}" + target="${entry#*:}" + + if ! docker volume inspect "$vol" >/dev/null 2>&1; then + echo "skip ${vol}: no such docker volume (fresh install?)" + continue + fi + + mkdir -p "$target" + if [ -n "$(ls -A "$target" 2>/dev/null)" ]; then + echo "skip ${vol}: ${target} already has data, not overwriting" + continue + fi + + docker run --rm \ + -v "${vol}:/from" \ + -v "$(cd "$target" && pwd):/to" \ + alpine sh -c "cp -a /from/. /to/" + echo "migrated ${vol} -> ${target}" +done + +vol_names=() +for entry in "${MIGRATIONS[@]}"; do + vol_names+=("${entry%%:*}") +done + +echo +echo "Done. Verify the deployment works with the new ./data/ paths, then you" +echo "can remove the old named volumes with, e.g.:" +echo " docker volume rm ${vol_names[*]}" diff --git a/deployments/examples/ocis_full/minio.yml b/deployments/examples/ocis_full/minio.yml index 9c91b0c4489..fabfcd3c3a2 100644 --- a/deployments/examples/ocis_full/minio.yml +++ b/deployments/examples/ocis_full/minio.yml @@ -13,7 +13,7 @@ services: "mkdir -p /data/${S3NG_BUCKET:-ocis-bucket} && minio server --console-address ':9001' /data", ] volumes: - - minio-data:/data + - ./data/minio:/data environment: MINIO_ACCESS_KEY: ${S3NG_ACCESS_KEY:-ocis} MINIO_SECRET_KEY: ${S3NG_SECRET_KEY:-ocis-secret-key} @@ -27,6 +27,3 @@ services: logging: driver: ${LOG_DRIVER:-local} restart: always - -volumes: - minio-data: diff --git a/deployments/examples/ocis_full/ocis.yml b/deployments/examples/ocis_full/ocis.yml index 8ef91e31172..bc8247d5efa 100644 --- a/deployments/examples/ocis_full/ocis.yml +++ b/deployments/examples/ocis_full/ocis.yml @@ -5,12 +5,35 @@ services: ocis-net: aliases: - ${OCIS_DOMAIN:-ocis.owncloud.test} + # chowns the host-mounted config/data dirs to the oCIS container's non-root + # user (uid/gid 1000) before oCIS itself starts, since bind mounts default + # to being owned by whoever created them on the host. Skips the recursive + # chown on subsequent startups (marker file) so it doesn't re-walk the + # whole data dir, which can grow big, on every restart. + ocis-data-init: + image: alpine:latest + user: root + entrypoint: ["/bin/sh", "-c"] + command: + - >- + for d in /etc/ocis /var/lib/ocis; do + mkdir -p "$$d"; + m="$$d/.ocis-data-init-done"; + [ -f "$$m" ] || { chown -R 1000:1000 "$$d"; touch "$$m"; }; + done + volumes: + - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis + - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis + ocis: image: ${OCIS_DOCKER_IMAGE}:${OCIS_DOCKER_TAG} # changelog: https://github.com/owncloud/ocis/tree/master/changelog # release notes: https://doc.owncloud.com/ocis_release_notes.html networks: ocis-net: + depends_on: + ocis-data-init: + condition: service_completed_successfully entrypoint: - /bin/sh # run ocis init to initialize a configuration file with random secrets @@ -57,9 +80,9 @@ services: - ./config/ocis/app-registry.yaml:/etc/ocis/app-registry.yaml - ./config/ocis/csp.yaml:/etc/ocis/csp.yaml - ./config/ocis/banned-password-list.txt:/etc/ocis/banned-password-list.txt - # configure the .env file to use own paths instead of docker internal volumes - - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis - - ${OCIS_DATA_DIR:-ocis-data}:/var/lib/ocis + # configure the .env file to use different host paths if desired + - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis + - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis labels: - "traefik.enable=true" - "traefik.http.routers.ocis.entrypoints=https" @@ -70,7 +93,3 @@ services: logging: driver: ${LOG_DRIVER:-local} restart: always - -volumes: - ocis-config: - ocis-data: diff --git a/deployments/examples/ocis_full/onlyoffice.yml b/deployments/examples/ocis_full/onlyoffice.yml index 0483a413f1d..3c57603ad81 100644 --- a/deployments/examples/ocis_full/onlyoffice.yml +++ b/deployments/examples/ocis_full/onlyoffice.yml @@ -31,8 +31,8 @@ services: COLLABORATION_LOG_LEVEL: ${LOG_LEVEL:-info} OCIS_URL: https://${OCIS_DOMAIN:-ocis.owncloud.test} volumes: - # configure the .env file to use own paths instead of docker internal volumes - - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis + # configure the .env file to use different host paths if desired + - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis labels: - "traefik.enable=true" - "traefik.http.routers.collaboration-oo.entrypoints=http" diff --git a/deployments/examples/ocis_full/vault-storage.yml b/deployments/examples/ocis_full/vault-storage.yml index c79b16b4d03..a60e47958a3 100644 --- a/deployments/examples/ocis_full/vault-storage.yml +++ b/deployments/examples/ocis_full/vault-storage.yml @@ -32,9 +32,9 @@ services: STORAGE_USERS_OCIS_ROOT: /var/lib/ocis/storage/users-vault STORAGE_USERS_EVENTS_CONSUMER_GROUP: vault-dcfs volumes: - # configure the .env file to use own paths instead of docker internal volumes - - ${OCIS_CONFIG_DIR:-ocis-config}:/etc/ocis - - ${OCIS_DATA_DIR:-ocis-data}:/var/lib/ocis + # configure the .env file to use different host paths if desired + - ${OCIS_CONFIG_DIR:-./data/ocis-config}:/etc/ocis + - ${OCIS_DATA_DIR:-./data/ocis-data}:/var/lib/ocis logging: driver: ${LOG_DRIVER:-local} restart: always diff --git a/deployments/examples/ocis_full/web_extensions/advanced-search.yml b/deployments/examples/ocis_full/web_extensions/advanced-search.yml index b5bb6eb75fd..8201b81b765 100644 --- a/deployments/examples/ocis_full/web_extensions/advanced-search.yml +++ b/deployments/examples/ocis_full/web_extensions/advanced-search.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:advanced-search-0.3.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/advanced-search/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/drawio.yml b/deployments/examples/ocis_full/web_extensions/drawio.yml index 637bd32ac72..95445422e44 100644 --- a/deployments/examples/ocis_full/web_extensions/drawio.yml +++ b/deployments/examples/ocis_full/web_extensions/drawio.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:draw-io-0.4.1 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/draw-io/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/extensions.yml b/deployments/examples/ocis_full/web_extensions/extensions.yml index d2a60a039b9..46d248a27eb 100644 --- a/deployments/examples/ocis_full/web_extensions/extensions.yml +++ b/deployments/examples/ocis_full/web_extensions/extensions.yml @@ -1,7 +1,4 @@ services: ocis: volumes: - - ocis-apps:/var/lib/ocis/web/assets/apps - -volumes: - ocis-apps: + - ./data/ocis-apps:/var/lib/ocis/web/assets/apps diff --git a/deployments/examples/ocis_full/web_extensions/externalsites.yml b/deployments/examples/ocis_full/web_extensions/externalsites.yml index a6a9d74e5f4..329c0e7ca8c 100644 --- a/deployments/examples/ocis_full/web_extensions/externalsites.yml +++ b/deployments/examples/ocis_full/web_extensions/externalsites.yml @@ -11,7 +11,7 @@ services: image: owncloud/web-extensions:external-sites-0.4.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/external-sites/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/importer.yml b/deployments/examples/ocis_full/web_extensions/importer.yml index 49f571b9392..848be13952b 100644 --- a/deployments/examples/ocis_full/web_extensions/importer.yml +++ b/deployments/examples/ocis_full/web_extensions/importer.yml @@ -17,7 +17,7 @@ services: image: owncloud/web-extensions:importer-0.4.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: [ "-c", "cp -R /var/lib/nginx/html/importer/ /apps" ] @@ -38,7 +38,7 @@ services: COMPANION_TUS_DEFERRED_UPLOAD_LENGTH: 'false' COMPANION_CLIENT_ORIGINS: "https://${OCIS_DOMAIN:-ocis.owncloud.test},https://${COMPANION_DOMAIN:-companion.owncloud.test}" volumes: - - companion-data:/tmp/companion/ + - ./data/companion:/tmp/companion/ labels: - "traefik.enable=true" - "traefik.http.routers.companion.entrypoints=https" @@ -49,6 +49,3 @@ services: logging: driver: ${LOG_DRIVER:-local} restart: always - -volumes: - companion-data: diff --git a/deployments/examples/ocis_full/web_extensions/jsonviewer.yml b/deployments/examples/ocis_full/web_extensions/jsonviewer.yml index 85e4b94d6a4..be403f14c8b 100644 --- a/deployments/examples/ocis_full/web_extensions/jsonviewer.yml +++ b/deployments/examples/ocis_full/web_extensions/jsonviewer.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:json-viewer-0.4.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/json-viewer/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/photo-addon.yml b/deployments/examples/ocis_full/web_extensions/photo-addon.yml index fead2b7a7f2..6b9f7b3f128 100644 --- a/deployments/examples/ocis_full/web_extensions/photo-addon.yml +++ b/deployments/examples/ocis_full/web_extensions/photo-addon.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:photo-addon-0.3.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/photo-addon/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/progressbars.yml b/deployments/examples/ocis_full/web_extensions/progressbars.yml index 029e50ef941..8511d62ee44 100644 --- a/deployments/examples/ocis_full/web_extensions/progressbars.yml +++ b/deployments/examples/ocis_full/web_extensions/progressbars.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:progress-bars-0.4.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/progress-bars/ /apps"] diff --git a/deployments/examples/ocis_full/web_extensions/unzip.yml b/deployments/examples/ocis_full/web_extensions/unzip.yml index ff8336852e4..acd4ee07fd7 100644 --- a/deployments/examples/ocis_full/web_extensions/unzip.yml +++ b/deployments/examples/ocis_full/web_extensions/unzip.yml @@ -9,7 +9,7 @@ services: image: owncloud/web-extensions:unzip-0.5.0 user: root volumes: - - ocis-apps:/apps + - ./data/ocis-apps:/apps entrypoint: - /bin/sh command: ["-c", "cp -R /var/lib/nginx/html/unzip/ /apps"] diff --git a/docs/ocis/deployment/ocis_full.md b/docs/ocis/deployment/ocis_full.md index 744c7bb0bac..91eb2587a48 100644 --- a/docs/ocis/deployment/ocis_full.md +++ b/docs/ocis/deployment/ocis_full.md @@ -74,6 +74,11 @@ See also [example server setup]({{< ref "preparing_server" >}}) `cd ocis/deployments/examples/ocis_full` +* Upgrading an existing deployment? Persistent data now lives under `./data/` + bind mounts instead of docker-managed named volumes. Run + `./migrate-volumes.sh` once, before starting the stack, to copy data from + the old named volumes into the new `./data/` directories. + * Open the `.env` file in a text editor. The file by default looks like this: @@ -144,7 +149,8 @@ See also [example server setup]({{< ref "preparing_server" >}}) # # Define the oCIS storage location. Set the paths for config and data to a local path. # Note that especially the data directory can grow big. - # Leaving it default stores data in docker internal volumes. + # Leaving it default stores data under ./data/ocis-config and ./data/ocis-data + # relative to the compose project directory. # For more details see: # https://doc.owncloud.com/ocis/next/deployment/general/general-info.html#default-paths # OCIS_CONFIG_DIR=/your/local/ocis/config