diff --git a/config/crd/bases/ascender.ansible.com_ascendermeshingresses.yaml b/config/crd/bases/ascender.ansible.com_ascendermeshingresses.yaml index b22f013a..f9dbf7c2 100644 --- a/config/crd/bases/ascender.ansible.com_ascendermeshingresses.yaml +++ b/config/crd/bases/ascender.ansible.com_ascendermeshingresses.yaml @@ -56,6 +56,12 @@ spec: external_ipaddress: description: External IP address to use for the Mesh Ingress. type: string + external_port: + description: Port remote nodes dial on external_hostname, the port of the listener the Mesh Ingress is served on. Defaults to 443. + type: integer + format: int32 + minimum: 1 + maximum: 65535 ingress_type: description: The ingress type to use to reach the deployed instance type: string @@ -67,6 +73,8 @@ spec: - ingressroutetcp - Route - route + - TLSRoute + - tlsroute ingress_api_version: description: The Ingress API version to use type: string @@ -82,6 +90,33 @@ spec: ingress_controller: description: Special configuration for specific Ingress Controllers type: string + gateway_parent_refs: + description: Gateways the TLSRoute attaches to when ingress_type is tlsroute. Each entry is a Gateway API ParentReference. + type: array + items: + type: object + required: + - name + properties: + group: + type: string + kind: + type: string + namespace: + type: string + name: + type: string + minLength: 1 + sectionName: + type: string + port: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + gateway_annotations: + description: Annotations to add to the TLSRoute + type: string node_selector: description: Assign the Mesh Ingress Pod to the specified node. type: string diff --git a/config/crd/bases/ascender.ansible.com_ascenders.yaml b/config/crd/bases/ascender.ansible.com_ascenders.yaml index f465759f..4ed79301 100644 --- a/config/crd/bases/ascender.ansible.com_ascenders.yaml +++ b/config/crd/bases/ascender.ansible.com_ascenders.yaml @@ -111,6 +111,8 @@ spec: - ingress - Route - route + - HTTPRoute + - httproute ingress_api_version: description: The Ingress API version to use type: string @@ -144,6 +146,33 @@ spec: ingress_controller: description: Special configuration for specific Ingress Controllers type: string + gateway_parent_refs: + description: Gateways the HTTPRoute attaches to when ingress_type is httproute. Each entry is a Gateway API ParentReference. + type: array + items: + type: object + required: + - name + properties: + group: + type: string + kind: + type: string + namespace: + type: string + name: + type: string + minLength: 1 + sectionName: + type: string + port: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + gateway_annotations: + description: Annotations to add to the HTTPRoute + type: string api_urlpattern_prefix: description: An optional configuration to add a prefix in the API URL path type: string @@ -2046,6 +2075,9 @@ spec: URL: description: URL to access the deployed instance type: string + ingressType: + description: The ingress_type last reconciled, used to remove what a previous ingress_type created when it changes + type: string adminUser: description: Admin user of the deployed instance type: string diff --git a/config/crd/bases/awx.ansible.com_awxmeshingresses.yaml b/config/crd/bases/awx.ansible.com_awxmeshingresses.yaml index 4520e30a..1a2e3bbd 100644 --- a/config/crd/bases/awx.ansible.com_awxmeshingresses.yaml +++ b/config/crd/bases/awx.ansible.com_awxmeshingresses.yaml @@ -56,6 +56,12 @@ spec: external_ipaddress: description: External IP address to use for the Mesh Ingress. type: string + external_port: + description: Port remote nodes dial on external_hostname, the port of the listener the Mesh Ingress is served on. Defaults to 443. + type: integer + format: int32 + minimum: 1 + maximum: 65535 ingress_type: description: The ingress type to use to reach the deployed instance type: string @@ -67,6 +73,8 @@ spec: - ingressroutetcp - Route - route + - TLSRoute + - tlsroute ingress_api_version: description: The Ingress API version to use type: string @@ -82,6 +90,33 @@ spec: ingress_controller: description: Special configuration for specific Ingress Controllers type: string + gateway_parent_refs: + description: Gateways the TLSRoute attaches to when ingress_type is tlsroute. Each entry is a Gateway API ParentReference. + type: array + items: + type: object + required: + - name + properties: + group: + type: string + kind: + type: string + namespace: + type: string + name: + type: string + minLength: 1 + sectionName: + type: string + port: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + gateway_annotations: + description: Annotations to add to the TLSRoute + type: string node_selector: description: Assign the Mesh Ingress Pod to the specified node. type: string diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index c32062de..57effedb 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -111,6 +111,8 @@ spec: - ingress - Route - route + - HTTPRoute + - httproute ingress_api_version: description: The Ingress API version to use type: string @@ -144,6 +146,33 @@ spec: ingress_controller: description: Special configuration for specific Ingress Controllers type: string + gateway_parent_refs: + description: Gateways the HTTPRoute attaches to when ingress_type is httproute. Each entry is a Gateway API ParentReference. + type: array + items: + type: object + required: + - name + properties: + group: + type: string + kind: + type: string + namespace: + type: string + name: + type: string + minLength: 1 + sectionName: + type: string + port: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + gateway_annotations: + description: Annotations to add to the HTTPRoute + type: string api_urlpattern_prefix: description: An optional configuration to add a prefix in the API URL path type: string @@ -2046,6 +2075,9 @@ spec: URL: description: URL to access the deployed instance type: string + ingressType: + description: The ingress_type last reconciled, used to remove what a previous ingress_type created when it changes + type: string adminUser: description: Admin user of the deployed instance type: string diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index 7fa5d9c9..32144266 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -35,6 +35,10 @@ spec: path: external_ipaddress x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text + - displayName: External Port + path: external_port + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - displayName: Ingress Type path: ingress_type x-descriptors: @@ -42,6 +46,16 @@ spec: - urn:alm:descriptor:com.tectonic.ui:select:Ingress - urn:alm:descriptor:com.tectonic.ui:select:IngressRouteTCP - urn:alm:descriptor:com.tectonic.ui:select:Route + - urn:alm:descriptor:com.tectonic.ui:select:TLSRoute + - displayName: Gateway Parent Refs + path: gateway_parent_refs + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:TLSRoute + - displayName: Gateway Annotations + path: gateway_annotations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:TLSRoute - displayName: Ingress API Version path: ingress_api_version x-descriptors: @@ -362,6 +376,18 @@ spec: - urn:alm:descriptor:com.tectonic.ui:select:none - urn:alm:descriptor:com.tectonic.ui:select:Ingress - urn:alm:descriptor:com.tectonic.ui:select:Route + - urn:alm:descriptor:com.tectonic.ui:select:HTTPRoute + - displayName: Gateway Parent Refs + path: gateway_parent_refs + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:HTTPRoute + - displayName: Gateway Annotations + path: gateway_annotations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:HTTPRoute - displayName: Ingress Class Name path: ingress_class_name x-descriptors: @@ -1201,6 +1227,10 @@ spec: path: external_ipaddress x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text + - displayName: External Port + path: external_port + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:number - displayName: Ingress Type path: ingress_type x-descriptors: @@ -1208,6 +1238,16 @@ spec: - urn:alm:descriptor:com.tectonic.ui:select:Ingress - urn:alm:descriptor:com.tectonic.ui:select:IngressRouteTCP - urn:alm:descriptor:com.tectonic.ui:select:Route + - urn:alm:descriptor:com.tectonic.ui:select:TLSRoute + - displayName: Gateway Parent Refs + path: gateway_parent_refs + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:TLSRoute + - displayName: Gateway Annotations + path: gateway_annotations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:TLSRoute - displayName: Ingress API Version path: ingress_api_version x-descriptors: @@ -1528,6 +1568,18 @@ spec: - urn:alm:descriptor:com.tectonic.ui:select:none - urn:alm:descriptor:com.tectonic.ui:select:Ingress - urn:alm:descriptor:com.tectonic.ui:select:Route + - urn:alm:descriptor:com.tectonic.ui:select:HTTPRoute + - displayName: Gateway Parent Refs + path: gateway_parent_refs + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:HTTPRoute + - displayName: Gateway Annotations + path: gateway_annotations + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:ingress_type:HTTPRoute - displayName: Ingress Class Name path: ingress_class_name x-descriptors: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f4ed0805..bfd247d0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -152,3 +152,16 @@ rules: - patch - update - watch + - apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + - tlsroutes + verbs: + - get + - list + - create + - delete + - patch + - update + - watch diff --git a/docs/user-guide/advanced-configuration/mesh-ingress.md b/docs/user-guide/advanced-configuration/mesh-ingress.md index 9932f0fb..4e2393da 100644 --- a/docs/user-guide/advanced-configuration/mesh-ingress.md +++ b/docs/user-guide/advanced-configuration/mesh-ingress.md @@ -88,6 +88,53 @@ spec: If you can't see any IngressRouteTCP resources by `kubectl` command after deploying mesh ingress, you should fully qualify the resource name with API group, `kubectl get ingressroutetcp.traefik.io` or `kubectl get ingressroutetcp.traefik.containo.us` for example. +### On Kubernetes with Operator managed Gateway API (TLSRoute) + +Ascender Operator can create a [Gateway API](https://gateway-api.sigs.k8s.io/) `TLSRoute` (`gateway.networking.k8s.io/v1`) and attach it to an existing Gateway. + +It requires the Gateway API CRDs v1.5.0 or later, a Gateway API implementation that supports TLSRoute, and a Gateway listener with `protocol: TLS` and `tls.mode: Passthrough` that admits TLSRoutes from the mesh ingress namespace. Receptor terminates TLS itself. + +```yaml +listeners: + - name: mesh # matches sectionName + protocol: TLS + port: 443 + hostname: + tls: + mode: Passthrough + allowedRoutes: + kinds: + - kind: TLSRoute + namespaces: + from: Selector + selector: + matchLabels: + kubernetes.io/metadata.name: +``` + +Example: + +```yaml +--- +apiVersion: ascender.ansible.com/v1alpha1 +kind: AscenderMeshIngress +metadata: + name: +spec: + deployment_name: + + ingress_type: tlsroute + external_hostname: + gateway_parent_refs: + - name: + namespace: + sectionName: mesh +``` + +`external_hostname` is required; the Gateway routes passthrough traffic by TLS SNI. Remote nodes connect to `external_port`, which defaults to 443 and must match the listener port. + +Changing `ingress_type` follows [the same rules as the Ascender instance](../network-and-tls-configuration.md#changing-ingress_type). When switching to `tlsroute`, the previous object is deleted only after the TLSRoute is accepted. + ### On Kubernetes with User managed Ingress To deploy a mesh ingress on Kubernetes cluster, create the AscenderMeshIngress resource on the namespace where your Ascender instance is running on. @@ -205,13 +252,16 @@ AscenderMeshIngressSpec is the description of the configuration for AscenderMesh | Name | Description | Default | | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | | **`deployment_name`** (string), required | Name of the Ascender deployment to create the Mesh Ingress for. | `awx` | -| **`ingress_type`** (string) | Ingress type for ingress managed by the operator. Options: `none`, `Ingress`, `IngressRouteTCP`, `Route` | `Route` (on OpenShift), `none` (on Kubernetes) | +| **`ingress_type`** (string) | Ingress type for ingress managed by the operator. Options: `none`, `Ingress`, `IngressRouteTCP`, `Route`, `TLSRoute`, in any case. See [Changing ingress_type](../network-and-tls-configuration.md#changing-ingress_type) | `Route` (on OpenShift), `none` (on Kubernetes) | | **`external_hostname`** (string) | External hostname is an optional field used for specifying the external hostname defined in an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). This parameter is automatically generated on OpenShift | N/A | | **`external_ipaddress`** (string) | External IP Address is an optional field used for specifying the external IP address defined in an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) | N/A | +| **`external_port`** (integer) | Port remote nodes dial on `external_hostname`: the port of the listener the mesh ingress is served on | `443` | | **`ingress_api_version`** (string) | API Version for ingress managed by the operator. This parameter is ignored when `ingress_type` is `Route` | `networking.k8s.io/v1` | | **`ingress_annotations`** (string) | Additional annotation on the ingress managed by the operator. This parameter is ignored when `ingress_type` is `Route` | `""` | | **`ingress_controller`** (string) | Special configuration for specific Ingress Controllers. This parameter is ignored when `ingress_type` is `Route` | `""` | | **`ingress_class_name`** (string) | The name of ingress class to use instead of the cluster default. see [IngressSpec](https://kubernetes.io/docs/reference/kubernetes-api/service-resources/ingress-v1/#IngressSpec). This parameter is ignored when `ingress_type` is `Route` | `""` | +| **`gateway_parent_refs`** (array) | Gateways the TLSRoute attaches to, each a Gateway API [ParentReference](https://gateway-api.sigs.k8s.io/reference/api-spec/#parentreference). Required when `ingress_type` is `TLSRoute` | `[]` | +| **`gateway_annotations`** (string) | Additional annotations on the TLSRoute | `""` | #### AscenderMeshIngressStatus diff --git a/docs/user-guide/network-and-tls-configuration.md b/docs/user-guide/network-and-tls-configuration.md index 5f68c035..fd0dde4f 100644 --- a/docs/user-guide/network-and-tls-configuration.md +++ b/docs/user-guide/network-and-tls-configuration.md @@ -73,7 +73,7 @@ spec: By default, the Ascender Operator is not opinionated and won't force a specific ingress type on you. So, when the `ingress_type` is not specified, it will default to `none` and nothing ingress-wise will be created. -The `ingress_type` supported options are: `none`, `ingress` and `route`. To toggle between these options, you can add the following to your AWX CRD: +The `ingress_type` supported options are: `none`, `ingress`, `route` and `httproute`. To toggle between these options, you can add the following to your Ascender CR. See [Changing ingress_type](#changing-ingress_type) for what happens to the old object when you switch. * None @@ -132,6 +132,61 @@ spec: ingress_controller: contour ``` + * Gateway API (HTTPRoute) + +The `httproute` type creates a [Gateway API](https://gateway-api.sigs.k8s.io/) `HTTPRoute` (`gateway.networking.k8s.io/v1`) named after the instance and attaches it to an existing Gateway. The operator does not create the Gateway, its listeners or their TLS certificates. It requires the Gateway API CRDs v1.0 or later and a Gateway API implementation. + +| Name | Description | Default | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | +| gateway_parent_refs | Required. The Gateways to attach to, each a Gateway API [ParentReference](https://gateway-api.sigs.k8s.io/reference/api-spec/#parentreference). `sectionName` picks a listener by its `name` | `[]` | +| gateway_annotations | HTTPRoute annotations | Empty string | +| ingress_hosts | The `hostname` of each entry becomes an HTTPRoute hostname. `tls_secret` is ignored: TLS is configured on the Gateway listener | Empty string | +| hostname | The HTTPRoute hostname when `ingress_hosts` is empty. With neither set, the route answers for every hostname the listener accepts | Empty string | +| ingress_path | Path prefix the route matches | / | + +`ingress_class_name`, `ingress_controller`, `ingress_annotations` and `ingress_path_type` do not apply to an HTTPRoute. The route targets port 80 of the `-service` Service, which the default `ClusterIP` `service_type` provides. + +```yaml +--- +spec: + ... + ingress_type: httproute + ingress_hosts: + - hostname: ascender-demo.example.com + gateway_parent_refs: + - name: shared-gateway + namespace: gateway-system + sectionName: https + gateway_annotations: | + environment: testing +``` + +TLS is terminated on the Gateway listener. An example listener for the route above: + +```yaml +listeners: + - name: https # matches sectionName + protocol: HTTPS + port: 443 + hostname: ascender-demo.example.com + tls: + mode: Terminate + certificateRefs: + - name: ascender-demo-tls + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + kubernetes.io/metadata.name: +``` + +A listener admits routes only from its own namespace (`from: Same`) unless `allowedRoutes` says otherwise. The route's status shows whether each Gateway accepted it: + +``` +kubectl -n get httproute -o jsonpath='{.status.parents[*].conditions}' +``` + * Route The following variables are customizable when `ingress_type=route` @@ -152,3 +207,14 @@ spec: route_tls_termination_mechanism: Passthrough route_tls_secret: custom-route-tls-secret-name ``` + +##### Changing ingress_type + +When `ingress_type` changes between `ingress`, `route` and `httproute`, the operator creates the new object and then deletes the object it created for the previous value. + +- Only objects owned by this Ascender CR are deleted. +- The previous value is read from `status.ingressType`. If it is not set, nothing is deleted. +- Changing to or from `none` deletes nothing. +- When switching to `httproute`, the previous object is deleted only after every Gateway in `gateway_parent_refs` has accepted the route and resolved its backend. The operator waits up to two minutes at the end of the reconcile. If the route is not accepted, the reconcile fails, the previous object is kept, and the change is retried on the next reconcile. To back out, set `ingress_type` to its previous value and delete the HTTPRoute with `kubectl -n delete httproute `. + +To switch without downtime, create an HTTPRoute with a different name, for example `-cutover`, the same `parentRefs` and `hostnames`, and a backendRef to the `-service` Service on port 80. Point DNS at the Gateway, then change `ingress_type` and delete the cutover route. diff --git a/hack/migrate-awx-to-ascender.sh b/hack/migrate-awx-to-ascender.sh index d30b260c..6610c3e3 100755 --- a/hack/migrate-awx-to-ascender.sh +++ b/hack/migrate-awx-to-ascender.sh @@ -78,6 +78,15 @@ command -v kubectl >/dev/null || die "kubectl is not on PATH" KC="kubectl -n $NAMESPACE" +# The installer's Route and HTTPRoute kinds exist only on some clusters, and +# naming one the cluster does not serve fails the whole get, so each is looked +# for only where it is served. +for kind in route.route.openshift.io httproute.gateway.networking.k8s.io; do + if $KC get "$kind" >/dev/null 2>&1; then + KINDS="$KINDS,$kind" + fi +done + # resource.group, kubectl's fully qualified form. It reads as a stutter only # because each group is named after its product: the resource awx in the group # awx.ansible.com, the resource ascender in ascender.ansible.com. diff --git a/roles/common/tasks/check_gateway_api.yml b/roles/common/tasks/check_gateway_api.yml new file mode 100644 index 00000000..7d07dae5 --- /dev/null +++ b/roles/common/tasks/check_gateway_api.yml @@ -0,0 +1,25 @@ +--- +# Fail early when the cluster does not serve the route kind (_route_kind, +# HTTPRoute or TLSRoute) at v1, or no Gateway is named. +# +# Included by path rather than with include_role, so the caller's variables +# are not shadowed by the common role's defaults. +- name: Look up the route kind {{ _route_kind }} + kubernetes.core.k8s_info: + api_version: gateway.networking.k8s.io/v1 + kind: '{{ _route_kind }}' + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + register: _route_api + +- name: Check a Gateway is named and the cluster serves v1 {{ _route_kind }} + ansible.builtin.assert: + that: + - _route_api.api_found + - gateway_parent_refs | default([], true) | length > 0 + fail_msg: >- + ingress_type {{ ingress_type }} needs gateway.networking.k8s.io/v1 {{ _route_kind }} + served by the cluster (Gateway API v1.0 or later for HTTPRoute, v1.5 or later + for TLSRoute) and at least one entry in gateway_parent_refs naming the + Gateway to attach to. + quiet: true diff --git a/roles/common/tasks/remove_previous_ingress.yml b/roles/common/tasks/remove_previous_ingress.yml new file mode 100644 index 00000000..e48fd33d --- /dev/null +++ b/roles/common/tasks/remove_previous_ingress.yml @@ -0,0 +1,109 @@ +--- +# When ingress_type changes, delete the object the previous value created, so +# one path into the deployment is left rather than two. +# +# The caller passes +# _ingress_objects every object the role can create, each as +# { ingress_type, api_version, kind, name } +# _owner_api_version the CR being reconciled +# _owner_kind +# +# The value last reconciled is kept in status.ingressType. A CR with none +# recorded deletes nothing, and neither does a change to none, which leaves +# the way in to the user. +# +# Included by path rather than with include_role, so the caller's variables +# are not shadowed by the common role's defaults. +- name: Read the ingress_type this CR was last reconciled with + kubernetes.core.k8s_info: + api_version: '{{ _owner_api_version }}' + kind: '{{ _owner_kind }}' + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + register: _owner + +- name: Remove the object the previous ingress_type created + vars: + _previous_type: "{{ (_owner.resources | first).status.ingressType | default('none') }}" + _current_type: '{{ ingress_type | lower }}' + _current: "{{ _ingress_objects | selectattr('ingress_type', 'equalto', _current_type) | first }}" + # Conditions written for an older generation of the route are dropped. + _route_generation: "{{ _route.resources[0].metadata.generation | default(0) | int }}" + _all_conditions: "{{ _route.resources[0].status.parents | default([]) | map(attribute='conditions') | flatten }}" + _route_conditions: >- + {{ (_all_conditions | rejectattr('observedGeneration', 'defined') | list) + + (_all_conditions | selectattr('observedGeneration', 'defined') + | selectattr('observedGeneration', 'ge', _route_generation | int) | list) }} + # Every Gateway named has accepted the route and no backend is unresolved. + _route_ready: >- + {{ _route_conditions | selectattr('type', 'equalto', 'Accepted') + | selectattr('status', 'equalto', 'True') | list | length + >= gateway_parent_refs | length + and _route_conditions | selectattr('type', 'equalto', 'ResolvedRefs') + | rejectattr('status', 'equalto', 'True') | list | length == 0 }} + when: _previous_type not in ['none', _current_type] and _current_type != 'none' + block: + # The old object keeps serving until the Gateway accepts the new route. + # Ingress and Route report nothing comparable, so only a route waits. + - name: Wait for the Gateway to accept the new route + kubernetes.core.k8s_info: + api_version: '{{ _current.api_version }}' + kind: '{{ _current.kind }}' + name: '{{ _current.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + register: _route + until: _route_ready | bool + retries: 12 + delay: 10 + # The assert below reports a route that is never accepted. + failed_when: false + when: _current.api_version.startswith('gateway.networking.k8s.io/') + + - name: Keep the previous object until the route is accepted + ansible.builtin.assert: + that: _route is skipped or _route_ready | bool + fail_msg: >- + {{ _current.kind }} {{ _current.name }} is not accepted by every Gateway in + gateway_parent_refs, so the {{ _previous_type }} object is kept and this is + retried on the next reconcile. Conditions: {{ _route_conditions | to_json }} + {{ _route.msg | default('') }} + To back out instead, set ingress_type back to {{ _previous_type }} and delete + {{ _current.kind }} {{ _current.name }}. + quiet: true + + - name: Look up the object the previous ingress_type created + kubernetes.core.k8s_info: + api_version: '{{ item.api_version }}' + kind: '{{ item.kind }}' + name: '{{ item.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + loop: "{{ _ingress_objects | selectattr('ingress_type', 'equalto', _previous_type) | list }}" + loop_control: + label: '{{ item.kind }}/{{ item.name }}' + register: _previous + + # Only an object this CR owns is deleted, never one made by hand. + - name: Delete the object the previous ingress_type created + kubernetes.core.k8s: + state: absent + api_version: '{{ item.apiVersion }}' + kind: '{{ item.kind }}' + name: '{{ item.metadata.name }}' + namespace: '{{ item.metadata.namespace }}' + loop: "{{ _previous.results | map(attribute='resources') | flatten }}" + loop_control: + label: '{{ item.kind }}/{{ item.metadata.name }}' + when: >- + item.metadata.ownerReferences | default([]) + | selectattr('kind', 'equalto', _owner_kind) + | selectattr('name', 'equalto', ansible_operator_meta.name) + | list | length > 0 + +- name: Record the ingress_type this CR is reconciled with + operator_sdk.util.k8s_status: + api_version: '{{ _owner_api_version }}' + kind: '{{ _owner_kind }}' + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + status: + ingressType: '{{ ingress_type | lower }}' diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 10a94ff8..c3f841f2 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -90,6 +90,18 @@ route_host: '' hostname: '' +# Gateway API, used when ingress_type is httproute. The operator creates only +# the HTTPRoute; the Gateway, its listeners and their TLS certificates belong +# to whoever runs it. Each entry is a Gateway API ParentReference. E.g.: +# gateway_parent_refs: +# - name: shared-gateway +# namespace: gateway-system +# sectionName: https +gateway_parent_refs: [] + +# Add annotations to the HTTPRoute. Specify as literal block. +gateway_annotations: '' + # Add a nodeSelector for the AWX pods. It must match a node's labels for the pod # to be scheduled on that node. Specify as literal block. E.g.: # node_selector: | diff --git a/roles/installer/tasks/install.yml b/roles/installer/tasks/install.yml index 838c1f17..7b3deba8 100644 --- a/roles/installer/tasks/install.yml +++ b/roles/installer/tasks/install.yml @@ -41,6 +41,12 @@ when: this_awx['resources'][0]['metadata']['labels'] when: additional_labels | length +- name: Check the Gateway API is usable + include_tasks: '{{ role_path }}/../common/tasks/check_gateway_api.yml' + vars: + _route_kind: HTTPRoute + when: ingress_type | lower == 'httproute' + - name: Include secret key configuration tasks include_tasks: secret_key_configuration.yml @@ -115,3 +121,23 @@ - name: Cleanup & Set garbage collection refs include_tasks: cleanup.yml + +- name: Remove what a previous ingress_type created + include_tasks: '{{ role_path }}/../common/tasks/remove_previous_ingress.yml' + # Last, so a route the Gateway has not accepted yet holds up nothing else. + vars: + _owner_api_version: '{{ api_version }}' + _owner_kind: '{{ kind }}' + _ingress_objects: + - ingress_type: ingress + api_version: networking.k8s.io/v1 + kind: Ingress + name: '{{ ansible_operator_meta.name }}-ingress' + - ingress_type: route + api_version: route.openshift.io/v1 + kind: Route + name: '{{ ansible_operator_meta.name }}' + - ingress_type: httproute + api_version: gateway.networking.k8s.io/v1 + kind: HTTPRoute + name: '{{ ansible_operator_meta.name }}' diff --git a/roles/installer/templates/networking/ingress.yaml.j2 b/roles/installer/templates/networking/ingress.yaml.j2 index 65de2e9f..10100423 100644 --- a/roles/installer/templates/networking/ingress.yaml.j2 +++ b/roles/installer/templates/networking/ingress.yaml.j2 @@ -86,6 +86,36 @@ spec: {% endif %} {% endif %} +{% if ingress_type|lower == "httproute" %} +{# The ingress_hosts hostnames, else hostname, else none, so the route answers for whatever the listener does #} +{% set _hostnames = (ingress_hosts | default([], true) | selectattr('hostname', 'defined') | map(attribute='hostname') | unique | list) or ([hostname] if hostname else []) %} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ ansible_operator_meta.namespace }}' + labels: + {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} +{% if gateway_annotations %} + annotations: + {{ gateway_annotations | indent(width=4) }} +{% endif %} +spec: + parentRefs: {{ gateway_parent_refs | to_json }} +{% if _hostnames %} + hostnames: {{ _hostnames | to_json }} +{% endif %} + rules: + - matches: + - path: + type: PathPrefix + value: '{{ ingress_path }}' + backendRefs: + - name: '{{ ansible_operator_meta.name }}-service' + port: 80 +{% endif %} + {% if ingress_type|lower == "route" %} --- {% if route_api_version is defined %} diff --git a/roles/mesh_ingress/defaults/main.yml b/roles/mesh_ingress/defaults/main.yml index df6cd24e..2f530ada 100644 --- a/roles/mesh_ingress/defaults/main.yml +++ b/roles/mesh_ingress/defaults/main.yml @@ -18,6 +18,18 @@ ingress_class_name: '' ingress_controller: '' route_annotations: '' +# Gateway API, used when ingress_type is tlsroute. The operator creates only +# the TLSRoute; the Gateway listener must use tls.mode Passthrough, because +# receptor terminates its own TLS. Each entry is a Gateway API ParentReference. +gateway_parent_refs: [] + +# Add annotations to the TLSRoute. Specify as literal block. +gateway_annotations: '' + +# The port remote nodes are told to dial on external_hostname: the port of the +# listener the Ingress, IngressRouteTCP, Route or TLSRoute is served on. +external_port: 443 + set_self_owneref: true _control_plane_ee_image: "ghcr.io/ctrliq/ascender-ee:{{ lookup('env', 'DEFAULT_AWX_VERSION') or 'latest' }}" diff --git a/roles/mesh_ingress/tasks/creation.yml b/roles/mesh_ingress/tasks/creation.yml index f1fcdc40..c6e0de1c 100644 --- a/roles/mesh_ingress/tasks/creation.yml +++ b/roles/mesh_ingress/tasks/creation.yml @@ -63,6 +63,19 @@ ingress_type: route when: is_openshift | bool and ingress_type | lower == 'none' +- name: Check the Gateway API is usable + include_tasks: '{{ role_path }}/../common/tasks/check_gateway_api.yml' + vars: + _route_kind: TLSRoute + when: ingress_type | lower == 'tlsroute' + +- name: Check a TLSRoute has a hostname to match on + assert: + that: external_hostname | default('') | length > 0 + fail_msg: ingress_type tlsroute needs external_hostname, the SNI name the Gateway routes on. + quiet: true + when: ingress_type | lower == 'tlsroute' + - name: Apply Ingress resource k8s: apply: yes @@ -147,4 +160,33 @@ ascender-manage add_receptor_address --instance {{ ansible_operator_meta.name }} --address {{ external_hostname }} - --port 443 --protocol ws + --port {{ external_port }} --protocol ws + +- name: Remove what a previous ingress_type created + include_tasks: '{{ role_path }}/../common/tasks/remove_previous_ingress.yml' + # Last, so a route the Gateway has not accepted yet holds up nothing else. + vars: + _owner_api_version: '{{ api_group }}/v1alpha1' + _owner_kind: '{{ deployment_kind }}MeshIngress' + _ingress_objects: + - ingress_type: ingress + api_version: networking.k8s.io/v1 + kind: Ingress + name: '{{ ansible_operator_meta.name }}' + # Both Traefik groups, since ingress_api_version may have picked either. + - ingress_type: ingressroutetcp + api_version: traefik.io/v1alpha1 + kind: IngressRouteTCP + name: '{{ ansible_operator_meta.name }}' + - ingress_type: ingressroutetcp + api_version: traefik.containo.us/v1alpha1 + kind: IngressRouteTCP + name: '{{ ansible_operator_meta.name }}' + - ingress_type: route + api_version: route.openshift.io/v1 + kind: Route + name: '{{ ansible_operator_meta.name }}' + - ingress_type: tlsroute + api_version: gateway.networking.k8s.io/v1 + kind: TLSRoute + name: '{{ ansible_operator_meta.name }}' diff --git a/roles/mesh_ingress/templates/ingress.yml.j2 b/roles/mesh_ingress/templates/ingress.yml.j2 index 70af4997..fda9b9dd 100644 --- a/roles/mesh_ingress/templates/ingress.yml.j2 +++ b/roles/mesh_ingress/templates/ingress.yml.j2 @@ -60,6 +60,26 @@ spec: passthrough: true {% endif %} +{% if ingress_type|lower == "tlsroute" %} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: TLSRoute +metadata: + name: {{ ansible_operator_meta.name }} + namespace: "{{ ansible_operator_meta.namespace }}" +{% if gateway_annotations %} + annotations: + {{ gateway_annotations | indent(width=4) }} +{% endif %} +spec: + parentRefs: {{ gateway_parent_refs | to_json }} + hostnames: {{ [external_hostname] | to_json }} + rules: + - backendRefs: + - name: {{ ansible_operator_meta.name }} + port: 27199 +{% endif %} + {% if ingress_type|lower == "route" %} --- apiVersion: route.openshift.io/v1