Allow zenith to run in traefik ingress-nginx compatability mode

sync/chart/templates/_helpers.tpl

@@ -30,7 +30,11 @@ Annotations for the ingress resource.
 */}}
 {{- define "zenith-service.ingress.annotations" -}}
 {{ toYaml .Values.ingress.annotations }}
-nginx.ingress.kubernetes.io/backend-protocol: {{ quote .Values.protocol }}
+{{- /*
+  With OIDC, the ingress forwards to the OAuth2 proxy, which listens on HTTP and
+  forwards authenticated requests to the service using the service protocol
+*/}}
+nginx.ingress.kubernetes.io/backend-protocol: {{ ternary "http" .Values.protocol .Values.oidc.enabled | quote }}
 {{- with .Values.readTimeout }}
 nginx.ingress.kubernetes.io/proxy-read-timeout: {{ int64 . | quote }}
 {{- end }}
@@ -40,9 +44,6 @@ nginx.ingress.kubernetes.io/proxy-read-timeout: {{ int64 . | quote }}
 {{- if .Values.externalAuth.enabled }}
 {{- include "zenith-service.ingress.auth.external.annotations" . }}
 {{- end }}
-{{- if .Values.oidc.enabled }}
-{{- include "zenith-service.ingress.auth.oidc.annotations" . }}
-{{- end }}
 {{- end }}
 
 {{/*
@@ -72,8 +73,7 @@ Annotations for external auth.
 {{- with .Values.externalAuth }}
 nginx.ingress.kubernetes.io/auth-url: {{ .url | required "external auth URL is required" | quote }}
 {{- if .signinUrl }}
-nginx.ingress.kubernetes.io/auth-signin: {{ quote .signinUrl }}
-nginx.ingress.kubernetes.io/auth-signin-redirect-param: {{ .nextUrlParam }}
+nginx.ingress.kubernetes.io/auth-signin: {{ include "zenith-service.ingress.auth.external.signinUrl" . | quote }}
 {{- end }}
 {{- if or .requestHeaders .params }}
 nginx.ingress.kubernetes.io/auth-snippet: |
@@ -91,43 +91,20 @@ nginx.ingress.kubernetes.io/auth-response-headers: {{ join "," .responseHeaders
 {{- end }}
 
 {{/*
-Annotations for OIDC auth.
+The signin URL for external auth, including the parameter for the original URL.
 */}}
-{{- define "zenith-service.ingress.auth.oidc.annotations" -}}
-{{- $scheme := ternary "https" "http" .Values.global.secure }}
-{{- $host := include "zenith-service.ingress.host" . }}
-{{- $oidcReleaseName := printf "%s-oidc" .Release.Name }}
-{{- $prefix := tpl (index .Values.oidc.extraArgs "proxy-prefix") . }}
-nginx.ingress.kubernetes.io/auth-url: >-
-  http://{{ $oidcReleaseName }}.{{ .Release.Namespace }}.svc.cluster.local{{ $prefix }}/auth
-nginx.ingress.kubernetes.io/auth-signin: >-
-  {{ $scheme }}://{{ $host }}{{ $prefix }}/start?rd=$escaped_request_uri&$args
-{{- with .Values.oidc.alphaConfig.configData.injectResponseHeaders }}
-nginx.ingress.kubernetes.io/auth-response-headers: >-
-  {{ range $i, $rh := . }}{{ if $i }},{{ end }}{{ $rh.name }}{{ end }}
-{{- end }}
-nginx.ingress.kubernetes.io/configuration-snippet: |
-  auth_request_set $auth_cookie__oauth2_proxy_1 $upstream_cookie__oauth2_proxy_1;
-  auth_request_set $auth_cookie__oauth2_proxy_2 $upstream_cookie__oauth2_proxy_2;
-  auth_request_set $auth_cookie__oauth2_proxy_3 $upstream_cookie__oauth2_proxy_3;
-
-  access_by_lua_block {
-    local auth_set_cookie = ngx.var.auth_cookie
-
-    if ngx.var.auth_cookie__oauth2_proxy_1 ~= "" then
-      auth_set_cookie = "_oauth2_proxy_1=" .. ngx.var.auth_cookie__oauth2_proxy_1 .. "; " .. auth_set_cookie
-    end
-    if ngx.var.auth_cookie__oauth2_proxy_2 ~= "" then
-      auth_set_cookie = "_oauth2_proxy_2=" .. ngx.var.auth_cookie__oauth2_proxy_2 .. "; " .. auth_set_cookie
-    end
-    if ngx.var.auth_cookie__oauth2_proxy_3 ~= "" then
-      auth_set_cookie = "_oauth2_proxy_3=" .. ngx.var.auth_cookie__oauth2_proxy_3 .. "; " .. auth_set_cookie
-    end
-
-    if auth_set_cookie ~= "" then
-      ngx.header["Set-Cookie"] = auth_set_cookie
-    end
-  }
+{{- define "zenith-service.ingress.auth.external.signinUrl" -}}
+{{- $query := get (urlParse .signinUrl) "query" }}
+{{- if regexMatch (printf "(^|&)%s=" (regexQuoteMeta .nextUrlParam)) $query }}
+{{- .signinUrl }}
+{{- else }}
+{{-
+  printf "%s%s%s=$scheme://$best_http_host$escaped_request_uri"
+    .signinUrl
+    (ternary "&" "?" (contains "?" .signinUrl))
+    .nextUrlParam
+}}
+{{- end }}
 {{- end }}
 
 {{/*

sync/chart/templates/ingress.yaml

@@ -26,7 +26,14 @@ spec:
             pathType: Prefix
             backend:
               service:
+                {{- if .Values.oidc.enabled }}
+                # With OIDC, the OAuth2 proxy receives all traffic for the service
+                name: {{ .Release.Name }}-oidc
+                port:
+                  name: http
+                {{- else }}
                 name: {{ .Release.Name }}
                 port:
                   name: dynamic
+                {{- end }}
 {{- end }}

sync/chart/tests/__snapshot__/auth_test.yaml.snap

@@ -0,0 +1,75 @@
+client CA ingress should match snapshot:
+  1: |
+    apiVersion: networking.k8s.io/v1
+    kind: Ingress
+    metadata:
+      annotations:
+        nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true"
+        nginx.ingress.kubernetes.io/auth-tls-secret: NAMESPACE/tls-client-ca-RELEASE-NAME
+        nginx.ingress.kubernetes.io/auth-tls-verify-client: optional
+        nginx.ingress.kubernetes.io/backend-protocol: http
+        nginx.ingress.kubernetes.io/proxy-body-size: "0"
+        nginx.ingress.kubernetes.io/proxy-buffer-size: 16k
+        nginx.ingress.kubernetes.io/proxy-buffering: "off"
+      labels:
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/version: main
+        helm.sh/chart: zenith-service-0.1.0
+        zenith.stackhpc.com/service-name: RELEASE-NAME
+      name: RELEASE-NAME
+    spec:
+      ingressClassName: nginx
+      rules:
+        - host: bar.foo.example.com
+          http:
+            paths:
+              - backend:
+                  service:
+                    name: RELEASE-NAME
+                    port:
+                      name: dynamic
+                path: /
+                pathType: Prefix
+      tls:
+        - hosts:
+            - bar.foo.example.com
+          secretName: tls-RELEASE-NAME
+external auth ingress with signin URL should match snapshot:
+  1: |
+    apiVersion: networking.k8s.io/v1
+    kind: Ingress
+    metadata:
+      annotations:
+        nginx.ingress.kubernetes.io/auth-response-headers: X-Remote-User,X-Remote-Group
+        nginx.ingress.kubernetes.io/auth-signin: https://auth.example.com/login?next=$scheme://$best_http_host$escaped_request_uri
+        nginx.ingress.kubernetes.io/auth-snippet: |
+          proxy_set_header accept application/json;
+          proxy_set_header tenancy-id abcdef;
+        nginx.ingress.kubernetes.io/auth-url: http://auth.auth-system.svc.cluster.local/verify/
+        nginx.ingress.kubernetes.io/backend-protocol: http
+        nginx.ingress.kubernetes.io/proxy-body-size: "0"
+        nginx.ingress.kubernetes.io/proxy-buffer-size: 16k
+        nginx.ingress.kubernetes.io/proxy-buffering: "off"
+      labels:
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/version: main
+        helm.sh/chart: zenith-service-0.1.0
+        zenith.stackhpc.com/service-name: RELEASE-NAME
+      name: RELEASE-NAME
+    spec:
+      ingressClassName: nginx
+      rules:
+        - host: bar.foo.example.com
+          http:
+            paths:
+              - backend:
+                  service:
+                    name: RELEASE-NAME
+                    port:
+                      name: dynamic
+                path: /
+                pathType: Prefix
+      tls:
+        - hosts:
+            - bar.foo.example.com
+          secretName: tls-RELEASE-NAME

sync/chart/tests/__snapshot__/oidc_proxy_test.yaml.snap

@@ -0,0 +1,33 @@
+ingress should match snapshot:
+  1: |
+    apiVersion: networking.k8s.io/v1
+    kind: Ingress
+    metadata:
+      annotations:
+        nginx.ingress.kubernetes.io/backend-protocol: http
+        nginx.ingress.kubernetes.io/proxy-body-size: "0"
+        nginx.ingress.kubernetes.io/proxy-buffer-size: 16k
+        nginx.ingress.kubernetes.io/proxy-buffering: "off"
+      labels:
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/version: main
+        helm.sh/chart: zenith-service-0.1.0
+        zenith.stackhpc.com/service-name: svc1
+      name: svc1
+    spec:
+      ingressClassName: nginx
+      rules:
+        - host: bar.foo.example.com
+          http:
+            paths:
+              - backend:
+                  service:
+                    name: svc1-oidc
+                    port:
+                      name: http
+                path: /
+                pathType: Prefix
+      tls:
+        - hosts:
+            - bar.foo.example.com
+          secretName: tls-svc1

sync/chart/tests/auth_test.yaml

@@ -0,0 +1,100 @@
+# Tests for the external auth and TLS annotations on the ingress
+# OIDC is tested in oidc_proxy_test.yaml
+# These annotations must be supported by both ingress-nginx and Traefik's NGINX
+# compatibility (kubernetesIngressNGINX provider)
+# To update manifest snapshots run helm unittest plugin with -u option:
+# docker run -i --rm -v $(pwd):/apps helmunittest/helm-unittest -u /path/to/chart
+suite: Ingress auth annotations
+values:
+  - values_test_overrides.yaml
+templates:
+  - templates/ingress.yaml
+tests:
+  - it: external auth ingress with signin URL should match snapshot
+    set:
+      externalAuth: &externalAuth
+        enabled: true
+        url: http://auth.auth-system.svc.cluster.local/verify/
+        signinUrl: https://auth.example.com/login
+        requestHeaders:
+          accept: application/json
+        responseHeaders:
+          - X-Remote-User
+          - X-Remote-Group
+        params:
+          tenancy-id: abcdef
+    asserts:
+      - matchSnapshot: {}
+
+  - it: external auth signin URL should embed the next URL parameter
+    set:
+      externalAuth:
+        <<: *externalAuth
+        nextUrlParam: next_url
+    asserts:
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin"]
+          value: https://auth.example.com/login?next_url=$scheme://$best_http_host$escaped_request_uri
+      - notExists:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin-redirect-param"]
+
+  - it: external auth signin URL should append to an existing query string
+    set:
+      externalAuth:
+        <<: *externalAuth
+        signinUrl: https://auth.example.com/login?kc_idp_hint=azimuth
+    asserts:
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin"]
+          value: https://auth.example.com/login?kc_idp_hint=azimuth&next=$scheme://$best_http_host$escaped_request_uri
+
+  - it: external auth signin URL should not override an existing next URL parameter
+    set:
+      externalAuth:
+        <<: *externalAuth
+        signinUrl: https://auth.example.com/login?next=https://other.example.com
+    asserts:
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin"]
+          value: https://auth.example.com/login?next=https://other.example.com
+
+  - it: external auth signin URL should not treat a similar parameter as the next URL parameter
+    set:
+      externalAuth:
+        <<: *externalAuth
+        signinUrl: https://auth.example.com/login?nextpage=1
+    asserts:
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin"]
+          value: https://auth.example.com/login?nextpage=1&next=$scheme://$best_http_host$escaped_request_uri
+
+  - it: external auth without signin URL should not set signin annotations
+    set:
+      externalAuth:
+        <<: *externalAuth
+        signinUrl:
+    asserts:
+      - notExists:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin"]
+      - notExists:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-signin-redirect-param"]
+
+  - it: client CA ingress should match snapshot
+    set:
+      ingress:
+        tls:
+          clientCA: |
+            -----BEGIN CERTIFICATE-----
+            notarealcertificate
+            -----END CERTIFICATE-----
+    asserts:
+      - matchSnapshot: {}
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-tls-secret"]
+          value: NAMESPACE/tls-client-ca-RELEASE-NAME
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-tls-verify-client"]
+          value: optional
+      - equal:
+          path: metadata.annotations["nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream"]
+          value: "true"