diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index 7dc69d84dc6..246cdd05780 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -62,7 +62,8 @@ jobs: - suite: apiAuthApp # - suite: apiAntivirus - suite: apiOcm - # - suite: apiCollaboration + - suite: apiCollaboration + - suite: apiVault # - suite: "coreApiAuth,coreApiCapabilities,coreApiFavorites,coreApiMain,coreApiVersions" - suite: "coreApiShareManagementBasicToShares,coreApiShareManagementToShares" - suite: "coreApiSharees" @@ -103,30 +104,32 @@ jobs: - name: Prepare hosts run: | - echo "127.0.0.1 ocis-server clamav email collabora onlyoffice fakeoffice tika federation-ocis-server" \ + echo "127.0.0.1 ocis-server clamav email collabora onlyoffice fakeoffice tika federation-ocis-server keycloak" \ | sudo tee -a /etc/hosts - name: Spin up K3d Cluster run: make -C tests/config/k8s create-cluster - - name: Prepare Helm Charts & Deploy oCIS + - name: Prepare Helm Charts env: ENABLE_ANTIVIRUS: ${{ matrix.suite == 'apiAntivirus' }} ENABLE_EMAIL: ${{ matrix.suite == 'apiNotification' || matrix.suite == 'apiSettings' || matrix.suite == 'apiOcm' }} - ENABLE_TIKA: ${{ matrix.suite == 'apiSearchContent' }} + ENABLE_TIKA: ${{ matrix.suite == 'apiSearchContent' || matrix.suite == 'apiVault' }} ENABLE_WOPI: ${{ matrix.suite == 'apiCollaboration' }} ENABLE_OCM: ${{ matrix.suite == 'apiOcm' }} ENABLE_AUTH_APP: ${{ matrix.suite == 'apiAuthApp' }} + ENABLE_VAULT: ${{ matrix.suite == 'apiVault' }} run: | cd tests/config/k8s make prepare-charts - kubectl get pods -n ocis-server -Aw & - make deploy-ocis if [[ "${{ matrix.suite }}" == "apiOcm" ]]; then OCM=true make prepare-charts - OCM=true make deploy-ocis fi + # Keycloak/postgres (and the other suite-specific backends) must be up and + # exposed to the cluster before oCIS is deployed: the proxy validates the + # OCIS_OIDC_ISSUER well-known endpoint against Keycloak at startup in vault mode, + # so `helm install --wait` would time out waiting for the proxy pod otherwise. - name: Deploy Suite-Specific External Backends run: | if [[ "${{ matrix.suite }}" == "apiNotification" || \ @@ -147,13 +150,80 @@ jobs: bash tests/config/k8s/expose-external-svc.sh clamav:3310 fi - if [[ "${{ matrix.suite }}" == "apiSearchContent" ]]; then + if [[ "${{ matrix.suite }}" == "apiSearchContent" || "${{ matrix.suite }}" == "apiVault" ]]; then docker run -d \ -p 9998:9998 \ --name tika \ apache/tika:3.2.2.0-full bash tests/config/k8s/expose-external-svc.sh tika:9998 fi + + if [[ "${{ matrix.suite }}" == "apiVault" ]]; then + # GitHub runners ship PostgreSQL pre-started on 5432; stop it so our + # container (needed by keycloak) can bind the same port. + sudo systemctl stop postgresql || true + + mkdir -p keycloak-certs + openssl req -x509 -newkey rsa:2048 \ + -keyout keycloak-certs/keycloakkey.pem \ + -out keycloak-certs/keycloakcrt.pem \ + -nodes -days 365 -subj "/CN=keycloak" + chmod 777 keycloak-certs/* + + # patch the realm so the "web" client's redirect/origin URLs match + # the k8s ingress domain instead of the non-k8s "localhost:9200" one + sed 's|https://localhost:9200|https://ocis-server|g' \ + tests/config/ci/ocis-mfa-ci-realm.dist.json > /tmp/ocis-realm.json + + docker run -d --name postgres --network host \ + -e POSTGRES_DB=keycloak \ + -e POSTGRES_USER=keycloak \ + -e POSTGRES_PASSWORD=keycloak \ + postgres:alpine3.18 + + for i in {1..30}; do + docker exec postgres pg_isready -U keycloak && break + echo "Waiting for postgres... ($i/30)" + sleep 2 + done + + docker run -d --name keycloak --network host \ + -e OCIS_DOMAIN=https://ocis-server \ + -e KC_HOSTNAME=keycloak \ + -e KC_PORT=8443 \ + -e KC_DB=postgres \ + -e KC_DB_URL=jdbc:postgresql://localhost:5432/keycloak \ + -e KC_DB_USERNAME=keycloak \ + -e KC_DB_PASSWORD=keycloak \ + -e KC_FEATURES=impersonation \ + -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + -e KC_HTTPS_CERTIFICATE_FILE=/keycloak-certs/keycloakcrt.pem \ + -e KC_HTTPS_CERTIFICATE_KEY_FILE=/keycloak-certs/keycloakkey.pem \ + -v "$(pwd)/keycloak-certs:/keycloak-certs:ro" \ + -v /tmp/ocis-realm.json:/opt/keycloak/data/import/ocis-mfa-ci-realm.dist.json:ro \ + quay.io/keycloak/keycloak:26.5.6 \ + start-dev --proxy-headers xforwarded \ + --spi-connections-http-client-default-disable-trust-manager=true \ + --import-realm --health-enabled=true + + for i in {1..60}; do + curl -skf https://localhost:9000/health/ready && break + echo "Waiting for keycloak... ($i/60)" + sleep 5 + done + + bash tests/config/k8s/expose-external-svc.sh keycloak:8443 + fi + + - name: Deploy oCIS + run: | + cd tests/config/k8s + kubectl get pods -n ocis-server -Aw & + make deploy-ocis + if [[ "${{ matrix.suite }}" == "apiOcm" ]]; then + OCM=true make deploy-ocis + fi - name: Wait for oCIS to be ready env: @@ -175,49 +245,90 @@ jobs: kubectl get pods -n ocis-server - echo "Creating test file in oCIS..." - - request() { - local method=$1 - shift - curl -ks -o /dev/null -w "%{http_code}" \ - -X "$method" \ - -u admin:admin \ - "$@" \ - "$FILE_URL" - } - - retry_http() { - local expected=$1 + retry() { + local label=$1 shift for i in {1..30}; do - status=$("$@") - if [ "$status" = "$expected" ]; then - echo "Succeeded (HTTP $status)" + if "$@"; then + echo "$label succeeded" return 0 fi - echo "Attempt $i failed (HTTP $status), retrying in 10s..." + echo "$label attempt $i failed, retrying in 10s..." sleep 10 done - echo "Failed after 30 attempts" + echo "$label failed after 30 attempts" return 1 } - put_file() { - request PUT \ - -H "Content-Type: text/plain" \ - --data "Hello from GitHub Actions!" - } - delete_file() { - request DELETE - } - echo "Creating test file..." - retry_http 201 put_file || exit 1 - echo "Deleting test file..." - retry_http 204 delete_file || exit 1 + + if [[ "${{ matrix.suite }}" == "apiVault" ]]; then + # In vault mode IDM_CREATE_DEMO_USERS=false, so there is no "admin" + # LDAP user to authenticate the basic-auth check below against. + # Poll the proxy's unauthenticated debug readyz endpoint instead, + # reached via port-forward since it isn't exposed by any Service. + # local port 19205 (not 9205): 9100-9399 is already bound on the + # host by the k3d loadbalancer's NodePort range, see create-cluster. + kubectl -n ocis-server port-forward deployment/proxy 19205:9205 & + PORT_FORWARD_PID=$! + trap 'kill $PORT_FORWARD_PID 2>/dev/null' EXIT + + proxy_ready() { + curl -sf http://localhost:19205/readyz > /dev/null + } + retry "proxy readyz" proxy_ready || exit 1 + else + echo "Creating test file in oCIS..." + + request() { + local method=$1 + shift + curl -ks -o /dev/null -w "%{http_code}" \ + -X "$method" \ + -u admin:admin \ + "$@" \ + "$FILE_URL" + } + put_file() { + request PUT \ + -H "Content-Type: text/plain" \ + --data "Hello from GitHub Actions!" + } + delete_file() { + request DELETE + } + check_status() { + local expected=$1 + shift + status=$("$@") + [ "$status" = "$expected" ] + } + echo "Creating test file..." + retry "create file (HTTP 201)" check_status 201 put_file || exit 1 + echo "Deleting test file..." + retry "delete file (HTTP 204)" check_status 204 delete_file || exit 1 + fi - name: Expose debug ports run: bash tests/config/k8s/expose-debug-svc.sh - + + # env-config scenarios (OcisConfigContext::waitForOcisProxyReady) poll the proxy's + # unauthenticated /readyz endpoint after every config-triggered restart. Local port 9205 + # is unavailable on the runner (k3d's loadbalancer reserves 9100-9399, see create-cluster), + # so forward it to 19205 instead, for the whole remainder of the job - unlike the earlier + # one-off port-forward in "Wait for oCIS to be ready", this one is intentionally left + # running (not killed via trap) since env-config scenarios can restart oCIS at any point + # during the test run. Wrapped in a reconnect loop: a config change that restarts the + # proxy pod itself would kill a plain `kubectl port-forward` for good, since it targets a + # specific pod and doesn't follow a deployment through a rollout. + - name: Expose proxy readyz + if: matrix.suite == 'apiVault' + run: | + ( + while true; do + kubectl -n ocis-server port-forward deployment/proxy 19205:9205 >> /tmp/proxy-readyz-portforward.log 2>&1 + sleep 1 + done + ) & + - name: Build ociswrapper run: make -C tests/ociswrapper/ @@ -230,6 +341,17 @@ jobs: --skip-ocis-run \ -n ocis-server & + - name: Install Playwright for vault tests + if: matrix.suite == 'apiVault' + run: | + composer install --no-progress + composer bin behat install --no-progress + vendor-php/bin/playwright-install + vendor-php/bin/playwright-install --browsers + env: + COMPOSER_NO_INTERACTION: "1" + COMPOSER_NO_AUDIT: "1" + - name: Prepare expected failures if: startsWith( matrix.suite, 'core' ) env: @@ -253,8 +375,11 @@ jobs: OCIS_WRAPPER_URL: ${{ env.OCIS_WRAPPER_URL }} COLLABORATION_SERVICE_URL: http://ocis-server:9304 K8S: ${{ env.K8S }} + KEYCLOAK: ${{ matrix.suite == 'apiVault' }} + KC_URL: https://keycloak:8443 + PROXY_READYZ_URL: ${{ matrix.suite == 'apiVault' && 'http://localhost:19205/readyz' || '' }} run: make test-acceptance-api - + - name: Run Core ${{ matrix.suite }} tests if: startsWith( matrix.suite, 'core' ) env: diff --git a/tests/acceptance/bootstrap/OcisConfigContext.php b/tests/acceptance/bootstrap/OcisConfigContext.php index 11276f6c977..81667d35f4d 100644 --- a/tests/acceptance/bootstrap/OcisConfigContext.php +++ b/tests/acceptance/bootstrap/OcisConfigContext.php @@ -391,7 +391,10 @@ private function assertOcisRestarted(ResponseInterface $response, string $errorM * @throws GuzzleException */ private function waitForOcisProxyReady(int $timeoutSeconds = 60): void { - $readyzUrl = 'http://localhost:9205/readyz'; + // In k8s, port 9205 on the runner's localhost is unavailable (k3d's loadbalancer + // reserves that range), so the proxy's debug port is forwarded to a different local + // port for the lifetime of the job. See k8s.yml's "Expose proxy readyz" step. + $readyzUrl = getenv('PROXY_READYZ_URL') ?: 'http://localhost:9205/readyz'; $deadline = time() + $timeoutSeconds; while (time() < $deadline) { try { diff --git a/tests/acceptance/bootstrap/Provisioning.php b/tests/acceptance/bootstrap/Provisioning.php index dfb4bb9c775..4e2a67c045c 100644 --- a/tests/acceptance/bootstrap/Provisioning.php +++ b/tests/acceptance/bootstrap/Provisioning.php @@ -81,6 +81,27 @@ public function getOcisUserToken(string $userId): array { return $this->userTokens[$userId]; } + /** + * Finds the oidc-client-ts user entry (key "oc_oAuth.user::") + * in a Playwright browser storage state and decodes its token data. + * The exact position of this entry among the other localStorage keys is not + * guaranteed, so it must be located by name rather than by a fixed index. + * + * @param array $state + * + * @return mixed + * @throws Exception + */ + public function extractOidcTokenDataFromStorageState(array $state): mixed { + $localStorage = $state['origins'][0]['localStorage'] ?? []; + foreach ($localStorage as $entry) { + if (\str_starts_with($entry['name'] ?? '', 'oc_oAuth.user:')) { + return \json_decode($entry['value']); + } + } + throw new Exception('Could not find an "oc_oAuth.user:" entry in the browser storage state.'); + } + /** * Check if this is the admin group. That group is always a local group in * ownCloud10, even if other groups come from LDAP. @@ -711,7 +732,7 @@ public function setAccessTokenForAdmin(): void { $adminUser["actualUsername"], $adminUser["password"], ); - $tokenData = \json_decode($state['origins'][0]['localStorage'][2]['value']); + $tokenData = $this->extractOidcTokenDataFromStorageState($state); $this->setOcisUserToken($adminUser, $tokenData); } @@ -735,7 +756,7 @@ public function userHasLoggedInViaWebUI(string $user): void { $userAttribute["actualUsername"], $userAttribute["password"], ); - $stateData = \json_decode($state['origins'][0]['localStorage'][2]['value']); + $stateData = $this->extractOidcTokenDataFromStorageState($state); $this->setOcisUserToken($userAttribute, $stateData); $response = $this->graphContext->adminHasRetrievedUserUsingTheGraphApi($user); $userAttribute['id'] = $this->getJsonDecodedResponse($response)['id']; diff --git a/tests/acceptance/features/apiVault/vault.feature b/tests/acceptance/features/apiVault/vault.feature index ad60348f956..a9cf7b42b55 100644 --- a/tests/acceptance/features/apiVault/vault.feature +++ b/tests/acceptance/features/apiVault/vault.feature @@ -57,7 +57,13 @@ Feature: vault @env-config @keycloak-config Scenario: user can set custom auth level names Given the administrator has set the Keycloak realm attribute "acr.loa.map" to '{"regular":"1","testing":"2"}' - And the config "OCIS_MFA_AUTH_LEVEL_NAMES" has been set to "testing" + # OCIS_MFA_AUTH_LEVEL_NAMES is read by both the proxy (mfa.go, gates access) and the + # frontend (exposed via the capabilities endpoint, which tells the web app which acr_values + # to request during step-up). In k8s each is a separate deployment, so both must be + # reconfigured explicitly - unlike the single-binary setup, there is no "just set it + # globally" here. + And the config "OCIS_MFA_AUTH_LEVEL_NAMES" has been set to "testing" for "proxy" service + And the config "OCIS_MFA_AUTH_LEVEL_NAMES" has been set to "testing" for "frontend" service And user "Alice" has logged in via web UI When user "Alice" uploads a file inside space "Personal" with content "some content" to "vaultFile.txt" in vault using the WebDAV API Then the HTTP status code should be "201" diff --git a/tests/config/k8s/README.md b/tests/config/k8s/README.md index 3f8bf5a7648..1fdc7a09538 100644 --- a/tests/config/k8s/README.md +++ b/tests/config/k8s/README.md @@ -57,6 +57,7 @@ > - `ENABLE_WOPI=true`: WOPI test suites > - `ENABLE_OCM=true`: OCM test suites > - `ENABLE_AUTH_APP=true`: auth-app test suites + > - `ENABLE_VAULT=true`: Vault test suites (needs `ENABLE_TIKA=true` too) > > ⚠️ When using the above environment variables, > make sure you run the necessary external services and expose them to the cluster. @@ -218,6 +219,55 @@ in a separate namespace on the same cluster, alongside the ocis server: make test-acceptance-api ``` +### Run Vault tests + +Vault mode requires Keycloak (as an external OIDC provider that can assert MFA/acr +claims) backed by postgres, plus Tika for full text search: + +1. Check if setup [step 3](#deploy-ocis-in-k8s) is done correctly. (`ENABLE_VAULT=true ENABLE_TIKA=true`) +2. Start postgres and Keycloak (self-signed cert, `CN=keycloak`, importing + `tests/config/ci/ocis-mfa-ci-realm.dist.json` with `https://localhost:9200` + replaced by `https://ocis-server`), and start tika: + + ```bash + docker run -d --name postgres --network host \ + -e POSTGRES_DB=keycloak -e POSTGRES_USER=keycloak -e POSTGRES_PASSWORD=keycloak \ + postgres:alpine3.18 + + docker run -d --name keycloak --network host \ + -e OCIS_DOMAIN=https://ocis-server -e KC_HOSTNAME=keycloak -e KC_PORT=8443 \ + -e KC_DB=postgres -e KC_DB_URL=jdbc:postgresql://localhost:5432/keycloak \ + -e KC_DB_USERNAME=keycloak -e KC_DB_PASSWORD=keycloak \ + -e KC_BOOTSTRAP_ADMIN_USERNAME=admin -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + -e KC_HTTPS_CERTIFICATE_FILE=/keycloak-certs/keycloakcrt.pem \ + -e KC_HTTPS_CERTIFICATE_KEY_FILE=/keycloak-certs/keycloakkey.pem \ + -v "$(pwd)/keycloak-certs:/keycloak-certs:ro" \ + -v /tmp/ocis-realm.json:/opt/keycloak/data/import/ocis-mfa-ci-realm.dist.json:ro \ + quay.io/keycloak/keycloak:26.5.6 \ + start-dev --import-realm --health-enabled=true + + docker run -d -p 9998:9998 --name tika apache/tika:3.2.2.0-full + ``` + +3. Expose them to the cluster + + ```bash + bash tests/config/k8s/expose-external-svc.sh keycloak:8443 + bash tests/config/k8s/expose-external-svc.sh tika:9998 + ``` + +4. Run the tests (Playwright is required for the `@javascript`/web-UI-login + scenarios; run `vendor-php/bin/playwright-install --browsers` once first) + + ```bash + TEST_SERVER_URL=https://ocis-server \ + K8S=true \ + KEYCLOAK=true \ + KC_URL=https://keycloak:8443 \ + BEHAT_FEATURE=/apiVault/vault.feature \ + make test-acceptance-api + ``` + ## Cleanup the Setup To delete the cluster and all the setup resources, run the following command: diff --git a/tests/config/k8s/setup.sh b/tests/config/k8s/setup.sh index d8f5fa2b08c..5a95043ea14 100644 --- a/tests/config/k8s/setup.sh +++ b/tests/config/k8s/setup.sh @@ -79,5 +79,9 @@ if [[ "$ENABLE_AUTH_APP" == "true" ]]; then sed -i '/authapp:/{n;s|false|true|}' $CFG_DIR/values.yaml fi +if [[ "$ENABLE_VAULT" == "true" ]]; then + sed -i '/vault:/{n;s|false|true|}' $CFG_DIR/values.yaml +fi + # copy custom values file cp $CFG_DIR/values.yaml "$CHT_DIR/ci/deployment-values.yaml" diff --git a/tests/config/k8s/templates/extra.tpl b/tests/config/k8s/templates/extra.tpl index a1327e675e9..2f26521784b 100644 --- a/tests/config/k8s/templates/extra.tpl +++ b/tests/config/k8s/templates/extra.tpl @@ -50,4 +50,104 @@ - name: ANTIVIRUS_CLAMAV_SOCKET value: "tcp://clamav:3310" {{- end -}} +{{- if .Values.features.vault.enabled }} +{{- if eq .appName "proxy" }} +- name: OCIS_ENABLE_VAULT_MODE + value: "true" +- name: OCIS_MFA_ENABLED + value: "true" +- name: PROXY_OIDC_ISSUER + value: "https://keycloak:8443/realms/oCIS" +- name: PROXY_OIDC_REWRITE_WELLKNOWN + value: "true" +- name: PROXY_AUTOPROVISION_ACCOUNTS + value: "true" +- name: PROXY_ROLE_ASSIGNMENT_DRIVER + value: oidc +- name: PROXY_USER_OIDC_CLAIM + value: preferred_username +- name: PROXY_USER_CS3_CLAIM + value: username +{{- end -}} +{{- if eq .appName "frontend" }} +- name: OCIS_ENABLE_VAULT_MODE + value: "true" +- name: OCIS_MFA_ENABLED + value: "true" +{{- end -}} +{{- if eq .appName "graph" }} +- name: OCIS_ENABLE_VAULT_MODE + value: "true" +- name: GRAPH_ASSIGN_DEFAULT_USER_ROLE + value: "false" +- name: GRAPH_USERNAME_MATCH + value: none +{{- end -}} +{{- if eq .appName "gateway" }} +- name: OCIS_ENABLE_VAULT_MODE + value: "true" +{{- end -}} +{{- if eq .appName "web" }} +{{/* WEB_OIDC_AUTHORITY intentionally left as the chart default (the oCIS + domain): PROXY_OIDC_REWRITE_WELLKNOWN transparently proxies the + well-known document from Keycloak under that same origin, matching + how the non-k8s vault test setup (run-github.py) configures this. */}} +- name: WEB_OIDC_CLIENT_ID + value: web +- name: WEB_OIDC_SCOPE + value: "openid profile email acr" +{{- end -}} +{{- if eq .appName "webfinger" }} +- name: WEBFINGER_OIDC_ISSUER + value: "https://keycloak:8443/realms/oCIS" +{{- end -}} +{{- if eq .appName "users" }} +- name: USERS_IDP_URL + value: "https://keycloak:8443/realms/oCIS" +{{- end -}} +{{- if eq .appName "groups" }} +- name: GROUPS_IDP_URL + value: "https://keycloak:8443/realms/oCIS" +{{- end -}} +{{- if eq .appName "ocs" }} +- name: OCS_IDM_ADDRESS + value: "https://keycloak:8443/realms/oCIS" +{{- end -}} +{{- if eq .appName "idm" }} +- name: OCIS_OIDC_ISSUER + value: "https://keycloak:8443/realms/oCIS" +- name: IDM_CREATE_DEMO_USERS + value: "false" +{{- end -}} +{{- if eq .appName "storageusers-vault" }} +- name: STORAGE_USERS_ENABLE_VAULT_MODE + value: "true" +- name: STORAGE_USERS_SERVICE_NAME + value: storage-users-vault +- name: STORAGE_USERS_EVENTS_CONSUMER_GROUP + value: vault-dcfs +{{/* Without these overrides this instance falls back to the shared OCIS_CACHE_STORE=nats-js-kv + used by the regular storage-users service. Since personal-space IDs are just the user's + opaque ID (identical in both instances), the vault instance's existence check on + CreateStorageSpace sees a cache hit from whatever the regular instance already created for + that user and returns AlreadyExists without ever writing its own space - the vault personal + space then never actually exists on this instance, even though every check claims it does. */}} +- name: STORAGE_USERS_FILEMETADATA_CACHE_STORE + value: memory +- name: STORAGE_USERS_ID_CACHE_STORE + value: memory +{{/* services/storage-users/pkg/config/parser/parse.go calls EnsureDefaults() (which forces + MountID to the vault constant when EnableVaultMode is set) BEFORE envdecode.Decode() + applies env vars - so EnableVaultMode is still false at that point and the override never + fires. Without this, MountID falls back to whatever STORAGE_USERS_MOUNT_ID resolves to + (the "storage-uuid" ConfigMap, shared with the regular storage-users instance), so spaces + created here come back with the wrong storage id embedded in their space id. That id still + lists fine (gateway's registry matches on its own static rule, not this value), but any + later ID-based lookup (e.g. creating a folder inside a newly created project space) fails + to route back to this instance. Setting it directly here sidesteps the ordering bug + entirely, since env vars are applied regardless of EnsureDefaults. */}} +- name: STORAGE_USERS_MOUNT_ID + value: "1a01c2c4-4309-4483-a845-842fd56d8622" +{{- end -}} +{{- end -}} {{- end -}} diff --git a/tests/config/k8s/templates/storageusersvault/deployment.yaml b/tests/config/k8s/templates/storageusersvault/deployment.yaml new file mode 100644 index 00000000000..77e68c8f7d0 --- /dev/null +++ b/tests/config/k8s/templates/storageusersvault/deployment.yaml @@ -0,0 +1,161 @@ +{{- if .Values.features.vault.enabled }} +{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameStorageUsers" "appNameSuffix" "vault") -}} +apiVersion: apps/v1 +kind: Deployment +{{ include "ocis.metadata" . }} +spec: + {{- include "ocis.selector" . | nindent 2 }} + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} + replicas: {{ .Values.replicas }} + {{- end }} + {{- include "ocis.deploymentStrategy" . | nindent 2 }} + template: + {{- include "ocis.templateMetadata" (dict "scope" $ "configCheck" false) | nindent 4 }} + spec: + {{- include "ocis.affinity" $ | nindent 6 }} + {{- include "ocis.securityContextAndtopologySpreadConstraints" . | nindent 6 }} + {{- include "ocis.priorityClassName" $.priorityClassName | nindent 6 }} + {{- include "ocis.hostAliases" $ | nindent 6 }} + nodeSelector: {{ toYaml $.nodeSelector | nindent 8 }} + containers: + - name: {{ .appName }} + {{- include "ocis.image" $ | nindent 10 }} + command: ["ocis"] + args: ["storage-users", "server"] + {{- include "ocis.containerSecurityContext" . | nindent 10 }} + env: + {{- include "ocis.serviceRegistry" . | nindent 12 }} + {{- include "ocis.events" . | nindent 12 }} + {{- include "ocis.cacheStore" . | nindent 12 }} + {{- include "ocis.cors" . | nindent 12 }} + + # this is a secondary storage-users instance, dedicated to vault storage + # see tests/config/k8s/templates/extra.tpl for STORAGE_USERS_ENABLE_VAULT_MODE + # and STORAGE_USERS_SERVICE_NAME + + - name: STORAGE_USERS_GATEWAY_GRPC_ADDR + value: {{ .appNameGateway }}:9142 + + - name: STORAGE_USERS_LOG_COLOR + value: {{ .Values.logging.color | quote }} + - name: STORAGE_USERS_LOG_LEVEL + value: {{ .Values.logging.level | quote }} + - name: STORAGE_USERS_LOG_PRETTY + value: {{ .Values.logging.pretty | quote }} + + - name: STORAGE_USERS_TRACING_ENABLED + value: "{{ .Values.tracing.enabled }}" + - name: STORAGE_USERS_TRACING_TYPE + value: {{ .Values.tracing.type | quote }} + - name: STORAGE_USERS_TRACING_ENDPOINT + value: {{ .Values.tracing.endpoint | quote }} + - name: STORAGE_USERS_TRACING_COLLECTOR + value: {{ .Values.tracing.collector | quote }} + + - name: STORAGE_USERS_DEBUG_PPROF + value: {{ .Values.debug.profiling | quote }} + + - name: STORAGE_USERS_GRPC_ADDR + value: 0.0.0.0:9157 + - name: STORAGE_USERS_DEBUG_ADDR + value: 0.0.0.0:9159 + + - name: STORAGE_USERS_HTTP_ADDR + value: 0.0.0.0:9158 + - name: STORAGE_USERS_DATA_SERVER_URL + value: "http://{{ .appName }}:9158/data" + + - name: STORAGE_USERS_DRIVER + value: ocis + - name: STORAGE_USERS_OCIS_MAX_CONCURRENCY + value: {{ .Values.services.storageusers.storageBackend.driverConfig.ocis.maxConcurrency | quote }} + + - name: STORAGE_USERS_UPLOAD_EXPIRATION + value: {{ .Values.services.storageusers.maintenance.cleanUpExpiredUploads.uploadExpiration | quote }} + + - name: STORAGE_USERS_PURGE_TRASH_BIN_PERSONAL_DELETE_BEFORE + value: {{ .Values.services.storageusers.maintenance.purgeExpiredTrashBinItems.personalDeleteBefore | quote}} + - name: STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE + value: {{ .Values.services.storageusers.maintenance.purgeExpiredTrashBinItems.projectDeleteBefore | quote }} + + - name: STORAGE_USERS_SERVICE_ACCOUNT_ID + valueFrom: + configMapKeyRef: + name: {{ include "config.authService" . }} + key: service-account-id + - name: STORAGE_USERS_SERVICE_ACCOUNT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "secrets.serviceAccountSecret" . }} + key: service-account-secret + + - name: STORAGE_USERS_STAT_CACHE_STORE + value: noop + + - name: STORAGE_USERS_MOUNT_ID + valueFrom: + configMapKeyRef: + name: {{ include "config.storageUsers" . }} + key: storage-uuid + + - name: STORAGE_USERS_JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "secrets.jwtSecret" . }} + key: jwt-secret + + - name: OCIS_TRANSFER_SECRET + valueFrom: + secretKeyRef: + name: {{ include "secrets.transferSecret" . }} + key: transfer-secret + + - name: OCIS_ASYNC_UPLOADS + value: "true" + - name: STORAGE_USERS_EVENTS_NUM_CONSUMERS + value: {{ .Values.services.storageusers.events.consumer.concurrency | quote }} + + - name: STORAGE_USERS_DATA_GATEWAY_URL + value: "http://{{ .appNameFrontend }}:9140/data/" + + {{- include "ocis.caEnv" $ | nindent 12}} + {{- include "ocis.extraEnvs" . | nindent 12}} + + {{- include "ocis.livenessProbe" . | nindent 10 }} + + resources: {{ toYaml .resources | nindent 12 }} + + ports: + - name: grpc + containerPort: 9157 + - name: http + containerPort: 9158 + - name: metrics-debug + containerPort: 9159 + + volumeMounts: + - name: tmp-volume + mountPath: /tmp + - name: messaging-system-ca + mountPath: /etc/ocis/messaging-system-ca + readOnly: true + - name: {{ include "ocis.persistence.dataVolumeName" . }} + mountPath: /var/lib/ocis + {{- include "ocis.caPath" $ | nindent 12}} + {{- include "ocis.extraVolMounts" . | nindent 12}} + + {{- include "ocis.imagePullSecrets" $ | nindent 6 }} + volumes: + - name: tmp-volume + emptyDir: {} + - name: messaging-system-ca + {{ if and (.Values.messagingSystem.external.enabled) (not .Values.messagingSystem.external.tls.certTrusted) }} + secret: + secretName: {{ include "secrets.messagingSystemCASecret" . }} + {{ else }} + emptyDir: {} + {{ end }} + {{- include "ocis.caVolume" $ | nindent 8}} + {{- include "ocis.persistence.dataVolume" . | nindent 8 }} + {{- include "ocis.extraVolumes" . | nindent 8}} +{{- end -}} diff --git a/tests/config/k8s/templates/storageusersvault/services.yaml b/tests/config/k8s/templates/storageusersvault/services.yaml new file mode 100644 index 00000000000..c6b7549d977 --- /dev/null +++ b/tests/config/k8s/templates/storageusersvault/services.yaml @@ -0,0 +1,28 @@ +{{- if .Values.features.vault.enabled }} +{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameStorageUsers" "appNameSuffix" "vault") -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ .appName }} + namespace: {{ template "ocis.namespace" . }} + labels: + app: {{ .appName }} + ocis-metrics: enabled + {{- include "ocis.labels" . | nindent 4 }} +spec: + selector: + app: {{ .appName }} + ports: + - name: grpc + port: 9157 + protocol: TCP + appProtocol: {{ .Values.service.appProtocol.grpc | quote}} + - name: http + port: 9158 + protocol: TCP + appProtocol: {{ .Values.service.appProtocol.http | quote}} + - name: metrics-debug + port: 9159 + protocol: TCP + appProtocol: {{ .Values.service.appProtocol.http | quote}} +{{- end -}} diff --git a/tests/config/k8s/values.yaml b/tests/config/k8s/values.yaml index f999e3bb88a..5b4c69ec660 100644 --- a/tests/config/k8s/values.yaml +++ b/tests/config/k8s/values.yaml @@ -16,6 +16,8 @@ insecure: features: authapp: enabled: false + vault: + enabled: false emailNotifications: enabled: false smtp: @@ -95,6 +97,7 @@ http: - 'blob:' - 'https://raw.githubusercontent.com/owncloud/awesome-ocis/' - 'https://marketplace.owncloud.com/' + - 'https://keycloak:8443/' defaultSrc: - "'none'" fontSrc: @@ -106,6 +109,7 @@ http: - "'self'" - 'blob:' - 'https://embed.diagrams.net/' + - 'https://keycloak:8443/' imgSrc: - "'self'" - 'data:' @@ -147,6 +151,9 @@ services: enabled: true accessModes: - ReadWriteOnce + storageusers-vault: + persistence: + enabled: false ocm: persistence: enabled: true diff --git a/tests/ociswrapper/ocis/k8s.go b/tests/ociswrapper/ocis/k8s.go index 7c1fb3e1b73..635e75a5eee 100644 --- a/tests/ociswrapper/ocis/k8s.go +++ b/tests/ociswrapper/ocis/k8s.go @@ -61,6 +61,15 @@ func K8sUpdateEnv(service string, envMap []string) (bool, string) { K8sOcisInitEnv[service].CurrentPod = podName } + // envMap may introduce vars that have no prior explicit value on the pod at all (e.g. only + // a code-level default was in effect) - the tracked baseline, just established/updated + // above by either branch, has no entry for those, so on its own it is not a rollback target + // that removes them; kubectl set env is additive and never strips a var it isn't told + // about. Mark any such brand-new var for removal now, while we still know it is new, or it + // silently survives every future rollback. + newlyIntroduced := diffEnvs(K8sOcisInitEnv[service].Envs, envMap) + K8sOcisInitEnv[service].Envs = append(K8sOcisInitEnv[service].Envs, newlyIntroduced...) + envSet, skipWaitForService, err := setServiceEnv(service, envMap, "Failed to set env") if err != nil { return false, "error setting env" @@ -126,7 +135,29 @@ func getInitialEnvs(service string) ([]string, error) { flatEnvVars = append(flatEnvVars, fmt.Sprintf("%s=%s", env.Name, env.Value)) } } - return flatEnvVars, nil + // The chart legitimately renders some vars twice in the pod spec (a default value, then a + // later override for the same name - Go's own os.Environ() takes the last one, which is + // what the running process actually observes). Deduping here, keeping the last occurrence, + // ensures this list is safe to replay through a single `kubectl set env` call later (e.g. + // during rollback): passing the same key twice in one invocation against a spec that + // already has two entries for it has been observed to drop the variable entirely instead of + // converging on one value, rather than raising an error. + return dedupeEnvs(flatEnvVars), nil +} + +func dedupeEnvs(envs []string) []string { + indexByKey := make(map[string]int, len(envs)) + deduped := make([]string, 0, len(envs)) + for _, env := range envs { + key := strings.SplitN(env, "=", 2)[0] + if idx, ok := indexByKey[key]; ok { + deduped[idx] = env + continue + } + indexByKey[key] = len(deduped) + deduped = append(deduped, env) + } + return deduped } func waitForService(service string, waitDeletion bool) (bool, error) { @@ -185,6 +216,37 @@ func waitForService(service string, waitDeletion bool) (bool, error) { } func setServiceEnv(service string, envMap []string, errMsgPrefix string) (bool, bool, error) { + // kubectl set env has been observed to behave unreliably (silently dropping the variable + // entirely, or picking an arbitrary one of the existing values) when the target pod spec + // already has multiple pre-existing entries for a key being set - which happens here + // because the chart legitimately renders some vars twice (a default value, then a later + // vault-mode override, relying on the running process's own last-one-wins env handling, + // not on kubectl ever reconciling it). Strip any existing entries for every key this call + // touches first, in its own invocation, so the actual set below always starts from a clean + // (zero-or-one-entry) state instead of leaving kubectl to reconcile a pre-existing + // duplicate on its own. Removing a key that isn't set is a no-op, so this is safe to do + // unconditionally. + removalArgs := []string{} + seenKeys := map[string]bool{} + for _, env := range envMap { + key := strings.TrimSuffix(strings.SplitN(env, "=", 2)[0], "-") + if !seenKeys[key] { + seenKeys[key] = true + removalArgs = append(removalArgs, key+"-") + } + } + if len(removalArgs) > 0 { + removeCmdArgs := append([]string{"set", "env", "-n", config.Get("namespace"), "deployment", service}, removalArgs...) + if _, err := exec.Command("kubectl", removeCmdArgs...).Output(); err != nil { + errMsg := "" + if exitErr, ok := err.(*exec.ExitError); ok { + errMsg = strings.TrimSpace(string(exitErr.Stderr)) + } + log.Println(fmt.Sprintf("[%s] Failed to pre-remove existing envs before setting them. %s", service, errMsg)) + return false, true, fmt.Errorf("error removing existing env before set") + } + } + cmdArgs := append([]string{"set", "env", "-n", config.Get("namespace"), "deployment", service}, envMap...) cmd := exec.Command("kubectl", cmdArgs...) output, err := cmd.Output()