diff --git a/deploy/helm/cassandra/helm/templates/statefulset.yaml b/deploy/helm/cassandra/helm/templates/statefulset.yaml index f451842d73..975bad4258 100644 --- a/deploy/helm/cassandra/helm/templates/statefulset.yaml +++ b/deploy/helm/cassandra/helm/templates/statefulset.yaml @@ -41,6 +41,9 @@ spec: {{- with .Values.cassandra.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.cassandra.topologySpreadConstraints }} + topologySpreadConstraints: {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.cassandra.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/helm/cassandra/helm/values.yaml b/deploy/helm/cassandra/helm/values.yaml index 35e1a26d4a..4c331947e2 100644 --- a/deploy/helm/cassandra/helm/values.yaml +++ b/deploy/helm/cassandra/helm/values.yaml @@ -156,6 +156,11 @@ cassandra: nodeSelector: {} tolerations: [] affinity: {} + # Pod topology spread constraints. Empty by default; the self-managed Helmfile + # sets these (zone spread across topology.kubernetes.io/zone) via + # highAvailability.tier2.topologySpread when HA is on. Requires a + # WaitForFirstConsumer StorageClass so each zonal PV binds in the pod's zone. + topologySpreadConstraints: [] hooks: initializeCluster: diff --git a/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml b/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..37878fa917 --- /dev/null +++ b/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.api.podDisruptionBudget.enabled }} +{{- $pdbMinAvail := .Values.api.podDisruptionBudget.minAvailable | toString }} +{{- $pdbMaxUnavail := .Values.api.podDisruptionBudget.maxUnavailable | toString }} +{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }} +{{- end }} +{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }} +{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }} +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "nvcf-api.fullname" . }} + namespace: {{ include "nvcf-api.namespace" . }} + labels: {{- include "nvcf-api.labels" . | nindent 4 }} +spec: + {{- if ne $pdbMinAvail "" }} + minAvailable: {{ .Values.api.podDisruptionBudget.minAvailable }} + {{- else }} + maxUnavailable: {{ .Values.api.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: {{- include "nvcf-api.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/cloud-functions/nvcf-api/values.yaml b/deploy/helm/cloud-functions/nvcf-api/values.yaml index b10d4043f7..b98265cc8b 100644 --- a/deploy/helm/cloud-functions/nvcf-api/values.yaml +++ b/deploy/helm/cloud-functions/nvcf-api/values.yaml @@ -26,6 +26,16 @@ api: # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ replicaCount: 1 + # PodDisruptionBudget for the API. Disabled by default; the self-managed + # Helmfile enables it (minAvailable 1) via highAvailability.stateless when HA + # is on. See deploy/stacks/self-managed/global.yaml.gotmpl. + podDisruptionBudget: + enabled: false + # minAvailable and maxUnavailable are mutually exclusive; set exactly one. + # Accepts an integer or a percentage string (e.g. 1 or "50%"). + minAvailable: "" + maxUnavailable: "" + # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: registry: "" # must be supplied diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile index ee727878f0..639aad87d5 100644 --- a/deploy/stacks/self-managed/Makefile +++ b/deploy/stacks/self-managed/Makefile @@ -27,6 +27,7 @@ test: @tests/nats-placement-tags.sh @tests/nats-tls-wiring.sh @tests/notary-image-repository.sh + @tests/ha-value-wiring.sh test-published-charts: @: "$${NVCF_PUBLISHED_CHART_REGISTRY:?NVCF_PUBLISHED_CHART_REGISTRY is required}" diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 4dcf4282cf..0ce81cc0b3 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -714,3 +714,50 @@ grpcproxy: # an external compute-worker endpoint. natsServiceURL: "" workerConnectBaseURL: "" + +# ============================================================================= +# Control-plane high availability +# ============================================================================= +# Resilience settings are exposed as a single highAvailability: block in +# self-managed Helmfile environment values. +# deploy/stacks/self-managed/global.yaml.gotmpl maps highAvailability.* onto +# chart values (replicaCount, affinity, PDB, update strategy). +# +# deploy/stacks/self-managed/ +# ├── environments/.yaml # highAvailability: configuration +# └── global.yaml.gotmpl # value mapping to charts +# +# mode: none (default) — keep existing env/chart replica, affinity, +# and PDB values (local / CI / BDD). +# preferred — HA sizing and placement below; hostname +# anti-affinity and zone topology spread are +# soft (still schedule if capacity is short). +# Requires ≥3 schedulable nodes. +# enforced — same HA sizing; hostname anti-affinity and zone +# topology spread are hard (Pending rather than +# packing / ignoring the constraint). Requires +# ≥3 schedulable nodes in ≥3 zones for the +# zone constraint to actually schedule. +# +# There is no per-component subtree beneath highAvailability: sizing and +# placement are derived uniformly from mode for every in-scope release (see +# global.yaml.gotmpl). Shared scheduling tuning is available under +# global.affinity / global.topologySpreadConstraints (class -> "all" -> the +# mode convention, same fallback pattern as global.nodeSelectors); an +# explicit {} / [] there suppresses the generated policy for that class. +# Component-shaped values elsewhere in this file remain the final per-release +# escape hatch for replica counts and PDBs. +# +# In-scope charts MUST expose the required value hooks. Leave mode: none +# for single-node installs. +# ============================================================================= +highAvailability: + mode: none + nats: + jetstream: + # JetStream stream replica factor (RF), applied under HA to the stream + # creators (nvcf-api, invocation-service) via env so the worker/result + # streams they declare are replicated across the NATS cluster. Default + # 3: JetStream streams use Raft quorum, so RF=2 loses quorum on a + # single replica failure and provides no HA benefit over RF=1. + replicaFactor: 3 diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 3eb2cad732..5094b4aad2 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -24,6 +24,221 @@ tolerations: {{- end -}} {{- end -}} +{{/* +highAvailability helpers. + +global.yaml.gotmpl maps highAvailability.mode onto chart values (replicaCount, +affinity, topology spread, PDB, update strategy). There is no per-component HA +subtree (stateless/hotPath/tier2/nats/openbao/cassandra) beneath +highAvailability: sizing and placement are derived uniformly from mode for +every in-scope release, per the HA values design review. + +highAvailability.mode: + none — keep existing env/chart defaults (local / CI / BDD). + preferred — HA sizing; soft (preferred/ScheduleAnyway) hostname + anti-affinity and zone topology spread. + enforced — HA sizing; hard (required/DoNotSchedule) hostname + anti-affinity and zone topology spread. + +Requires >= 3 schedulable nodes unless mode is none. + +Shared scheduling tuning: global.affinity and global.topologySpreadConstraints +resolve class-specific override -> "all" -> the mode-derived convention above, +the same fallback pattern as the existing global.nodeSelectors / +global.tolerations. An explicit {} (affinity) or [] (topology spread) for a +class — or for "all" — suppresses the generated policy for that class; +presence is the signal, so this is NOT equivalent to omitting the key. +Component-shaped values in the selected environment remain the final +per-release escape hatch and are applied by each release block below, not by +these helpers. + +Replica-safe Deployments (api, invocation, grpcproxy, adminIssuerProxy, +rateLimiter, natsAuthCalloutService, llmApiGateway): 2 replicas under HA (see +NOTE below for invocation/grpcproxy), hostname anti-affinity, zone topology +spread, PDB minAvailable 1. + NOTE: invocation-service and grpc-proxy remain single-replica for now + (anti-affinity/zone-spread still applied but no-ops at 1 replica, no PDB). + Their multi-replica scaling is deferred until Envoy support lands in the + self-hosted stack (worker-callback host binding; see #987/#989 review). + +Quorum services (Cassandra, NATS, OpenBao): 3 replicas, hostname +anti-affinity and zone topology spread so peers land on distinct +nodes/zones, PDB sized to tolerate exactly one voluntary disruption. NATS +JetStream RF is applied to the stream creators (nvcf-api, invocation) via +their env. + +In-scope charts MUST expose the required value hooks. +*/}} +{{- define "nvcf.ha.replicaSafeReplicaCount" -}} +{{- if ne (dig "highAvailability" "mode" "none" . | toString) "none" -}}2{{- end -}} +{{- end -}} + +{{/* Internal: minAvailable-shaped PodDisruptionBudget body, empty under mode + none. Context: dict "haMode" "" "minAvailable" */}} +{{- define "nvcf.ha._podDisruptionBudget" -}} +{{- if ne .haMode "none" -}} +enabled: true +minAvailable: {{ .minAvailable }} +{{- end -}} +{{- end -}} + +{{/* PodDisruptionBudget for a replica-safe Tier-1 Deployment (2 replicas + under HA): minAvailable 1, so a voluntary disruption can never remove + both. Context: .Values (called as nvcf.ha.replicaSafeReplicaCount is) */}} +{{- define "nvcf.ha.replicaSafePodDisruptionBudget" -}} +{{- include "nvcf.ha._podDisruptionBudget" (dict "haMode" (dig "highAvailability" "mode" "none" . | toString) "minAvailable" 1) -}} +{{- end -}} + +{{/* PodDisruptionBudget for a 3-member quorum service (Cassandra): tolerates + exactly one voluntary disruption (minAvailable 2 of 3). Context: .Values */}} +{{- define "nvcf.ha.quorumPodDisruptionBudget" -}} +{{- include "nvcf.ha._podDisruptionBudget" (dict "haMode" (dig "highAvailability" "mode" "none" . | toString) "minAvailable" 2) -}} +{{- end -}} + +{{/* PodDisruptionBudget for the NATS quorum, in the shape the NATS chart's + podDisruptionBudget.merge.spec values expect. Tolerates exactly one + voluntary disruption (minAvailable 2 of 3). Context: .Values */}} +{{- define "nvcf.ha.natsQuorumPodDisruptionBudget" -}} +{{- if ne (dig "highAvailability" "mode" "none" . | toString) "none" -}} +enabled: true +merge: + spec: + minAvailable: 2 +{{- end -}} +{{- end -}} + +{{/* JetStream stream replica factor (RF) under HA. Applied to the stream + creators — nvcf-api (Java, nvcf.nats.replicas) and invocation (Rust, + nats_properties.replicas) — via their env so the streams they declare are + replicated across the NATS cluster. Defaults to 3: JetStream streams use + Raft quorum, and RF=2 provides no failure tolerance (loses quorum on a + single replica loss) — see the NATS JetStream replication guidance and + the HA values design review. */}} +{{- define "nvcf.ha.natsStreamReplicas" -}} +{{- dig "highAvailability" "nats" "jetstream" "replicaFactor" 3 . -}} +{{- end -}} + +{{/* Mode-derived hostname pod anti-affinity body (no podAntiAffinity: key), + hard under enforced, soft under preferred, empty under none. Internal: + called only from nvcf.ha.podAntiAffinity. Context: dict "haMode" "" + "instance" "" */}} +{{- define "nvcf.ha._generatedPodAntiAffinity" -}} +{{- if eq .haMode "enforced" -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .instance | quote }} + topologyKey: kubernetes.io/hostname +{{- else if eq .haMode "preferred" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .instance | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} +{{- end -}} + +{{/* Mode-derived zone topology-spread constraint list items (no + topologySpreadConstraints: key), ScheduleAnyway under preferred, + DoNotSchedule under enforced, empty under none. Internal: called only + from nvcf.ha.topologySpreadItems. Context: dict "haMode" "" + "instance" "" */}} +{{- define "nvcf.ha._generatedTopologySpread" -}} +{{- if or (eq .haMode "preferred") (eq .haMode "enforced") -}} +- maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: {{ if eq .haMode "enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }} + labelSelector: + matchLabels: + app.kubernetes.io/instance: {{ .instance | quote }} +{{- end -}} +{{- end -}} + +{{/* Resolves the content of a workload's affinity: block: global.affinity. + -> global.affinity.all -> the mode-derived convention. Emits content ready + to nest directly under an affinity: key (map value), or to wrap as a + string (chart values that take affinity as templated YAML text, e.g. + OpenBao) — never the affinity: key itself, so callers control that + nesting. An operator override IS the full affinity content (may set + podAntiAffinity, podAffinity, and/or nodeAffinity) and is emitted as-is, + not merged with the generated convention. The generated convention emits + only podAntiAffinity, matching that same "content under affinity:" + contract. Context: dict "Values" $.Values "instance" "" + "class" "" */}} +{{- define "nvcf.ha.podAntiAffinity" -}} +{{- $byClass := dig "global" "affinity" dict .Values -}} +{{- if hasKey $byClass .class -}} +{{- with index $byClass .class -}}{{- toYaml . -}}{{- end -}} +{{- else if hasKey $byClass "all" -}} +{{- with index $byClass "all" -}}{{- toYaml . -}}{{- end -}} +{{- else -}} +{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}} +{{- with include "nvcf.ha._generatedPodAntiAffinity" (dict "haMode" $haMode "instance" .instance) }} +podAntiAffinity: +{{- . | nindent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Resolves topologySpreadConstraints for a workload: same fallback as + nvcf.ha.podAntiAffinity via global.topologySpreadConstraints. Emits the + constraint list items only (no topologySpreadConstraints: key), for the + same nesting/string reasons as nvcf.ha.podAntiAffinity. Context: dict + "Values" $.Values "instance" "" "class" "" */}} +{{- define "nvcf.ha.topologySpreadItems" -}} +{{- $byClass := dig "global" "topologySpreadConstraints" dict .Values -}} +{{- if hasKey $byClass .class -}} +{{- with index $byClass .class -}}{{- toYaml . -}}{{- end -}} +{{- else if hasKey $byClass "all" -}} +{{- with index $byClass "all" -}}{{- toYaml . -}}{{- end -}} +{{- else -}} +{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}} +{{- include "nvcf.ha._generatedTopologySpread" (dict "haMode" $haMode "instance" .instance) -}} +{{- end -}} +{{- end -}} + +{{/* Full affinity: block wrapper around nvcf.ha.podAntiAffinity, for release + values that take affinity as a plain map with no other keys nvcf.yaml + needs to set. Context: same as nvcf.ha.podAntiAffinity */}} +{{- define "nvcf.ha.affinity" -}} +{{- with include "nvcf.ha.podAntiAffinity" . -}} +affinity: +{{- . | nindent 2 }} +{{- end -}} +{{- end -}} + +{{/* Full topologySpreadConstraints: block wrapper around + nvcf.ha.topologySpreadItems. Context: same as nvcf.ha.topologySpreadItems */}} +{{- define "nvcf.ha.topologySpreadConstraints" -}} +{{- with include "nvcf.ha.topologySpreadItems" . -}} +topologySpreadConstraints: +{{- . | nindent 2 }} +{{- end -}} +{{- end -}} + +{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }} +{{- if not (kindIs "string" $haMode) }} +{{- fail "highAvailability.mode must be a string" }} +{{- end }} +{{- if not (has $haMode (list "none" "preferred" "enforced")) }} +{{- fail (printf "highAvailability.mode must be none, preferred, or enforced, got %q" $haMode) }} +{{- end }} +{{- $haEnabled := or (eq $haMode "preferred") (eq $haMode "enforced") }} +{{- range $key := list "affinity" "topologySpreadConstraints" }} +{{- $byClass := dig "global" $key dict $.Values }} +{{- if not (kindIs "map" $byClass) }} +{{- fail (printf "global.%s must be a map keyed by workload class (or \"all\")" $key) }} +{{- end }} +{{- end }} + cassandra: global: {{- if .Values.global.imagePullSecrets }} @@ -36,13 +251,20 @@ cassandra: defaultStorageClass: {{ .Values.global.storageClass }} {{- end }} - replicaCount: {{ dig "cassandra" "replicaCount" 3 .Values }} + replicaCount: {{ if $haEnabled }}3{{ else }}{{ dig "cassandra" "replicaCount" 3 .Values }}{{ end }} resourcesPreset: {{ dig "cassandra" "resourcesPreset" "xlarge" .Values }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.quorumPodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "cassandra" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- with include "nvcf.nodeSelector" (dict "type" "cassandra" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} @@ -50,6 +272,16 @@ cassandra: {{- with include "nvcf.tolerations" (dict "type" "cassandra" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "cassandra" "class" "cassandra") }} + affinity: + {{- . | nindent 4 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "cassandra" "class" "cassandra") }} + topologySpreadConstraints: + {{- . | nindent 4 }} + {{- end }} + {{- end }} persistence: size: {{ .Values.global.storageSize | default "10Gi" }} @@ -155,7 +387,7 @@ openbao: {{- with include "nvcf.tolerations" (dict "type" "vault" "tolerations" .Values.global.tolerations) }} {{- . | nindent 4 }} {{- end }} - replicas: {{ .Values.openbao.injector.replicas }} + replicas: {{ if $haEnabled }}2{{ else }}{{ .Values.openbao.injector.replicas }}{{ end }} {{- $nvcfUiEnabled := dig "addons" "nvcfUi" "enabled" false .Values }} {{- with dig "openbao" "injector" "webhook" dict .Values }} webhook: @@ -176,6 +408,20 @@ openbao: podDisruptionBudget: minAvailable: {{ dig "openbao" "injector" "podDisruptionBudget" "minAvailable" 1 .Values }} server: + {{- if $haEnabled }} + {{- with include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "openbao-server" "class" "vault") }} + # String form: the OpenBao chart tpl-renders server.affinity. The wrapper + # sets it to "" for single-node installs; HA re-enables hostname spread. + affinity: | + {{- . | nindent 6 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "openbao-server" "class" "vault") }} + # String form: the OpenBao chart tpl-renders server.topologySpreadConstraints + # under a topologySpreadConstraints: key. + topologySpreadConstraints: | + {{- . | nindent 6 }} + {{- end }} + {{- end }} image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/nvcf-openbao @@ -193,7 +439,19 @@ openbao: size: {{ .Values.global.storageSize | default "10Gi" }} ha: - {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }} + {{- if $haEnabled }} + enabled: true + replicas: 3 + {{- end }} + {{- $openbaoDisruptionBudget := dict }} + {{- if $haEnabled }} + {{/* Default: allow at most 1 of the 3 Raft peers to be voluntarily + disrupted at a time, so a rolling drain/upgrade can never take the + quorum below 2 of 3. Without this, HA renders 3 replicas but the + Raft cluster has no disruption protection. */}} + {{- $openbaoDisruptionBudget = dict "enabled" true "maxUnavailable" 1 }} + {{- end }} + {{- with dig "openbao" "server" "ha" "disruptionBudget" $openbaoDisruptionBudget .Values }} disruptionBudget: {{- toYaml . | nindent 8 }} {{- end }} @@ -235,7 +493,13 @@ nats: repository: {{ .Values.global.image.repository }}/alpine-k8s {{- $natsNs := include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) -}} {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) -}} - {{- if or $natsNs $natsTol }} + {{- $natsAffinity := "" -}} + {{- $natsTopoSpread := "" -}} + {{- if $haEnabled -}} + {{- $natsAffinity = include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "nats" "class" "controlplane") -}} + {{- $natsTopoSpread = include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "nats" "class" "controlplane") -}} + {{- end -}} + {{- if or $natsNs $natsTol $natsAffinity $natsTopoSpread }} podTemplate: merge: spec: @@ -245,6 +509,14 @@ nats: {{- with $natsTol }} {{- . | nindent 8 }} {{- end }} + {{- with $natsAffinity }} + affinity: + {{- . | nindent 10 }} + {{- end }} + {{- with $natsTopoSpread }} + topologySpreadConstraints: + {{- . | nindent 10 }} + {{- end }} {{- end }} # One merged config: block -- a second config: key under nats: would be @@ -262,6 +534,11 @@ nats: {{- if kindIs "bool" $allowNonTls }} allow_non_tls: {{ $allowNonTls }} {{- end }} + {{- if $haEnabled }} + cluster: + enabled: true + replicas: 3 + {{- end }} {{- if .Values.global.storageClass }} jetstream: fileStore: @@ -274,10 +551,17 @@ nats: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.natsQuorumPodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "nats" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} apikeys: fullnameOverride: api-keys @@ -308,6 +592,19 @@ apikeys: {{- end }} natsAuthCalloutService: + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }} + {{- end }} + {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "nats-auth-callout-service" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "nats-auth-callout-service" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -483,6 +780,9 @@ natsAuthCalloutService: {{- $apiRemoteConfigData = mergeOverwrite $apiRemoteConfigData $stackApiRemoteConfigData }} api: fullnameOverride: nvcf-api + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -501,6 +801,23 @@ api: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "api" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "api" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} + {{- with dig "api" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} {{- if $apiRemoteConfigData }} remoteConfig: configData: @@ -582,6 +899,11 @@ api: {{- if .Values.global.observability.tracing.enabled }} {{- $_ := set $apiEnv "MANAGEMENT_OTLP_TRACING_ENDPOINT" (printf "%s://%s:%v/v1/traces" .Values.global.observability.tracing.collectorProtocol .Values.global.observability.tracing.collectorEndpoint .Values.global.observability.tracing.collectorPort) }} {{- end }} + {{- if $haEnabled }} + {{- /* JetStream RF for the worker/result streams this service declares + (nvcf.nats.replicas → NVCF_NATS_REPLICAS via Spring relaxed binding). */}} + {{- $_ := set $apiEnv "NVCF_NATS_REPLICAS" (include "nvcf.ha.natsStreamReplicas" .Values) }} + {{- end }} {{- $apiEnv = mergeOverwrite $apiEnv $renderedApiEnv }} env: {{- toYaml $apiEnv | nindent 4 }} @@ -605,8 +927,20 @@ invocation: {{- with $invocationWorkerBaseURL }} {{- $_ := set $invocationEnv "WORKER_STREAM_PROPERTIES__SELF_ADDRESS" . }} {{- end }} + {{- if $haEnabled }} + {{- /* JetStream RF for the streams this service declares + (nats_properties.replicas → NATS_PROPERTIES__REPLICAS). */}} + {{- $_ := set $invocationEnv "NATS_PROPERTIES__REPLICAS" (include "nvcf.ha.natsStreamReplicas" .Values) }} + {{- end }} {{- $invocationEnv = mergeOverwrite $invocationEnv (deepCopy $configuredInvocationEnv) }} fullnameOverride: invocation-service + {{- if $haEnabled }} + {{- /* HA replica scaling for invocation-service is deferred until Envoy + support lands in the self-hosted stack (worker-callback host binding, + see #987/#989 review). Pin to a single replica for now; the + anti-affinity / zone-spread below are no-ops at one replica. */}} + replicaCount: 1 + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -620,6 +954,12 @@ invocation: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "invocation-service" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "invocation-service" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} metrics: enabled: {{ or (eq (dig "observability" "profile" "disabled" .Values) "control") (eq (dig "observability" "profile" "disabled" .Values) "all") }} @@ -634,6 +974,9 @@ invocation: baggageAttributeAllowlist: {{- toYaml . | nindent 6 }} {{- end }} + {{- /* No HA PodDisruptionBudget while invocation-service runs a single + replica (deferred until Envoy). A minAvailable:1 PDB on a singleton + would block node drains. */}} {{- with dig "invocation" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} @@ -685,6 +1028,13 @@ nvctApi: grpcproxy: fullnameOverride: grpc-proxy + {{- if $haEnabled }} + {{- /* HA replica scaling for grpc-proxy is deferred until Envoy support + lands in the self-hosted stack (worker-callback host binding, see + #987/#989 review). Pin to a single replica for now; the anti-affinity + / zone-spread below are no-ops at one replica. */}} + replicaCount: 1 + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -698,6 +1048,19 @@ grpcproxy: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- if $haEnabled }} + {{- $grpcAffinity := include "nvcf.ha.affinity" (dict "Values" .Values "instance" "grpc-proxy" "class" "controlplane") }} + {{- $grpcTopologySpread := include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "grpc-proxy" "class" "controlplane") }} + {{- if or $grpcAffinity $grpcTopologySpread }} + deployment: + {{- with $grpcAffinity }} + {{- . | nindent 4 }} + {{- end }} + {{- with $grpcTopologySpread }} + {{- . | nindent 4 }} + {{- end }} + {{- end }} + {{- end }} {{- with $grpcProxyWorkerConnectBaseURL }} workerConnectBaseURL: {{ . | quote }} {{- end }} @@ -719,6 +1082,9 @@ grpcproxy: RATE_LIMIT_ENABLED: "true" RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777" {{- end }} + {{- /* No HA PodDisruptionBudget while grpc-proxy runs a single replica + (deferred until Envoy). A minAvailable:1 PDB on a singleton would + block node drains. */}} {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} @@ -736,15 +1102,28 @@ rateLimiter: nodeSelector: {{ .Values.global.nodeSelectors.controlplane.key }}: {{ .Values.global.nodeSelectors.controlplane.value }} {{- end }} - replicaCount: {{ .Values.rateLimiter.replicaCount }} + replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}{{ else }}{{ .Values.rateLimiter.replicaCount }}{{ end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "ratelimiter" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "ratelimiter" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} {{- if .Values.global.observability.tracing.enabled }} env: OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" {{- end }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "rateLimiter" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} ess: fullnameOverride: ess-api @@ -837,6 +1216,9 @@ sis: adminIssuerProxy: fullnameOverride: admin-token-issuer-proxy + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -853,6 +1235,12 @@ adminIssuerProxy: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "admin-issuer-proxy" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "admin-issuer-proxy" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} gateway: enabled: true namespace: {{ .Values.ingress.gatewayApi.gateways.shared.namespace }} @@ -860,10 +1248,17 @@ adminIssuerProxy: name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }} hostname: "api-keys.{{ .Values.global.domain }}" path: "/v1/admin/keys" + {{- if $haEnabled }} + {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "adminIssuerProxy" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} stateMetrics: enabled: false @@ -975,7 +1370,7 @@ reval: llmApiGateway: enabled: {{ dig "addons" "llm" "enabled" false .Values }} fullnameOverride: llm-api-gateway - replicaCount: {{ dig "addons" "llm" "gateway" "replicaCount" 3 .Values }} + replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}{{ else }}{{ dig "addons" "llm" "gateway" "replicaCount" 3 .Values }}{{ end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -995,6 +1390,12 @@ llmApiGateway: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "llm-api-gateway" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "llm-api-gateway" "class" "controlplane") }} + {{- . | nindent 2 }} + {{- end }} {{- if dig "addons" "llm" "gateway" "auth" "grpcInsecure" false .Values }} config: nvcfGrpcInsecure: true @@ -1009,10 +1410,17 @@ llmApiGateway: {{- if .Values.global.observability.tracing.enabled }} endpoint: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" {{- end }} + {{- if $haEnabled }} + {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }} + podDisruptionBudget: + {{- . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "llmApiGateway" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- $pylonGrpcDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonGrpcDialAddress" "" .Values | default "" | toString | trim }} {{- $pylonReverseTunnelDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonReverseTunnelDialAddress" "" .Values | default "" | toString | trim }} diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh new file mode 100755 index 0000000000..e60ccdfbe2 --- /dev/null +++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +# Test that highAvailability values thread from environment files through +# global.yaml.gotmpl into chart values for stateless / quorum releases. +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="ha-value-wiring-test" +environment_file="$test_stack_dir/environments/$environment_name.yaml" +secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "ha-value-wiring: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +printf '{}\n' >"$secrets_file" + +render_chart_values() { + local release="$1" + local output_file="$2" + local helmfile_file="$3" + shift 3 + + # global.yaml.gotmpl evaluates adminIssuerProxy gateway refs for every release. + HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$helmfile_file" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --selector "name=$release" \ + "$@" \ + write-values \ + --output-file-template "$output_file" +} + +write_env() { + cat >"$environment_file" +} + +deps="$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" +core="$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" + +echo "== highAvailability mode none: chart defaults / base values unchanged ==" +write_env <<'EOF' +highAvailability: + mode: none +EOF + +render_chart_values api "$work_dir/api-off.yaml" "$core" || fail "render api (ha none)" +# HA must not inject replicaCount into the api values when disabled. +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "replicaCount:"; then + fail "api: HA replicaCount leaked while highAvailability.mode=none" +fi +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "podAntiAffinity:"; then + fail "api: HA affinity leaked while highAvailability.mode=none" +fi +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "topologySpreadConstraints:"; then + fail "api: topology spread leaked while highAvailability.mode=none" +fi +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "podDisruptionBudget:"; then + fail "api: PDB leaked while highAvailability.mode=none" +fi + +render_chart_values ratelimiter "$work_dir/ratelimiter-off.yaml" "$core" --state-values-set rateLimiter.enabled=true || + fail "render ratelimiter (ha none)" +if awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-off.yaml" | grep -q "podAntiAffinity:"; then + fail "ratelimiter: HA affinity leaked while highAvailability.mode=none" +fi + +# Tier-2 (#989): anti-affinity must not leak into the quorum charts when off. +render_chart_values cassandra "$work_dir/cassandra-off.yaml" "$deps" || fail "render cassandra (ha none)" +if grep -q "podAntiAffinity:" "$work_dir/cassandra-off.yaml"; then + fail "cassandra: Tier-2 anti-affinity leaked while highAvailability.mode=none" +fi + +# JetStream RF (#989): the RF env must not leak into the stream creators when off. +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "NVCF_NATS_REPLICAS:"; then + fail "api: JetStream RF env leaked while highAvailability.mode=none" +fi +render_chart_values invocation-service "$work_dir/invocation-off.yaml" "$core" || fail "render invocation (ha none)" +if grep -q "NATS_PROPERTIES__REPLICAS:" "$work_dir/invocation-off.yaml"; then + fail "invocation: JetStream RF env leaked while highAvailability.mode=none" +fi + +echo "== highAvailability preferred: stateless / quorum sizing ==" +write_env <<'EOF' +highAvailability: + mode: preferred +EOF + +render_chart_values api "$work_dir/api-on.yaml" "$core" || fail "render api (preferred)" +grep -E "replicaCount:[[:space:]]*2" "$work_dir/api-on.yaml" >/dev/null || + fail "api: expected replicaCount 2 when highAvailability.mode=preferred" +grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-on.yaml" || + fail "api: expected preferred anti-affinity when highAvailability.mode=preferred" +grep -q "topologySpreadConstraints:" "$work_dir/api-on.yaml" || + fail "api: expected topologySpreadConstraints when highAvailability.mode=preferred" +grep -q "topology.kubernetes.io/zone" "$work_dir/api-on.yaml" || + fail "api: expected zone topologyKey when highAvailability.mode=preferred" +grep -q "whenUnsatisfiable: ScheduleAnyway" "$work_dir/api-on.yaml" || + fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=preferred" +awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-on.yaml" | grep -q "podDisruptionBudget:" || + fail "api: expected podDisruptionBudget when highAvailability.mode=preferred" +grep -q 'NVCF_NATS_REPLICAS: "3"' "$work_dir/api-on.yaml" || + fail "api: expected JetStream RF NVCF_NATS_REPLICAS=3 when highAvailability.mode=preferred" + +render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || fail "render invocation (preferred)" +grep -q 'NATS_PROPERTIES__REPLICAS: "3"' "$work_dir/invocation-on.yaml" || + fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=3 when highAvailability.mode=preferred" + +render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (preferred)" +grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null || + fail "cassandra: expected replicaCount 3 when highAvailability.mode=preferred" +grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-on.yaml" | grep -q "enabled: true" || + fail "cassandra: expected HA PDB enabled" +grep -q "podAntiAffinity:" "$work_dir/cassandra-on.yaml" || + fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=preferred" +grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-on.yaml" || + fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=preferred" +# Zone topology spread is part of the same mode convention as hostname +# anti-affinity (soft under preferred) — it is no longer a separate opt-in +# toggle. NOTE: the write-values file holds every release's values, so scope +# to the cassandra: block. +awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "topology.kubernetes.io/zone" || + fail "cassandra: expected Tier-2 zone spread when highAvailability.mode=preferred" +awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "whenUnsatisfiable: ScheduleAnyway" || + fail "cassandra: expected soft (ScheduleAnyway) Tier-2 spread when highAvailability.mode=preferred" + +render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (preferred)" +grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null || + fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=preferred" +grep -q "podAntiAffinity:" "$work_dir/openbao-on.yaml" || + fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=preferred" +grep -A3 "disruptionBudget:" "$work_dir/openbao-on.yaml" | grep -q "maxUnavailable: 1" || + fail "openbao: expected server.ha.disruptionBudget maxUnavailable=1 when highAvailability.mode=preferred" +awk '/^openbao:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/openbao-on.yaml" | grep -q "topology.kubernetes.io/zone" || + fail "openbao: expected Tier-2 zone spread when highAvailability.mode=preferred" + +render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (preferred)" +grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null || + fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=preferred" +grep -q "podAntiAffinity:" "$work_dir/nats-on.yaml" || + fail "nats: expected Tier-2 anti-affinity when highAvailability.mode=preferred" +awk '/^nats:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/nats-on.yaml" | grep -q "topology.kubernetes.io/zone" || + fail "nats: expected Tier-2 zone spread when highAvailability.mode=preferred" + +# Hot-path helpers (#988): rateLimiter + nats-auth-callout to 2 replicas. +render_chart_values ratelimiter "$work_dir/ratelimiter-on.yaml" "$core" --state-values-set rateLimiter.enabled=true || + fail "render ratelimiter (preferred)" +awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null || + fail "ratelimiter: expected replicaCount 2 when highAvailability.mode=preferred" +awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" || + fail "ratelimiter: expected preferred anti-affinity when highAvailability.mode=preferred" +awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "topology.kubernetes.io/zone" || + fail "ratelimiter: expected zone topology spread when highAvailability.mode=preferred" + +render_chart_values nats-auth-callout-service "$work_dir/natsauth-on.yaml" "$core" || + fail "render nats-auth-callout (preferred)" +awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null || + fail "nats-auth-callout: expected replicaCount 2 when highAvailability.mode=preferred" +awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -q "podDisruptionBudget:" || + fail "nats-auth-callout: expected podDisruptionBudget when highAvailability.mode=preferred (was previously never wired)" + +# llm-api-gateway (#987): stateless anti-affinity when the LLM addon is on. +render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-values-set addons.llm.enabled=true || + fail "render llm-api-gateway (preferred)" +awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" || + fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=preferred" + +# invocation-service + grpc-proxy: multi-replica scaling is deferred until +# Envoy support lands (worker-callback host binding; see #987/#989 review). +# Even under HA they must stay single-replica and get no PDB (a minAvailable:1 +# PDB on a singleton blocks node drains). Anti-affinity/zone-spread may still +# render but are no-ops at one replica. +render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || + fail "render invocation-service (preferred)" +if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then + fail "invocation-service: must stay single-replica under HA (deferred until Envoy)" +fi +# The chart's own PDB knob may render (enabled: false); the HA PDB (enabled: +# true / minAvailable on a singleton) must NOT. +if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.yaml" | grep -A3 "podDisruptionBudget:" | grep -q "enabled: true"; then + fail "invocation-service: HA PDB must not be enabled while single-replica (deferred until Envoy)" +fi + +render_chart_values grpc-proxy "$work_dir/grpcproxy-on.yaml" "$core" || + fail "render grpc-proxy (preferred)" +if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then + fail "grpc-proxy: must stay single-replica under HA (deferred until Envoy)" +fi +if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml" | grep -A3 "podDisruptionBudget:" | grep -q "enabled: true"; then + fail "grpc-proxy: HA PDB must not be enabled while single-replica (deferred until Envoy)" +fi + +echo "== global.affinity / global.topologySpreadConstraints fallback (class -> all -> convention) ==" + +# class-specific global.affinity override wins over the generated convention. +write_env <<'EOF' +highAvailability: + mode: preferred +global: + affinity: + cassandra: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + custom: override + topologyKey: kubernetes.io/hostname +EOF +render_chart_values cassandra "$work_dir/cassandra-global-class.yaml" "$deps" || fail "render cassandra (global.affinity.cassandra override)" +awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-class.yaml" | grep -q "custom: override" || + fail "cassandra: expected global.affinity.cassandra override to win over the generated convention" +# The generated anti-affinity's own selector ("operator: In" against +# app.kubernetes.io/instance) must not also render — only the override +# content. app.kubernetes.io/instance alone is not distinctive enough to +# assert on: the (unrelated, mode-derived) topologySpreadConstraints block +# legitimately uses it too. +if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-class.yaml" | grep -A2 "podAntiAffinity:" | grep -q "operator: In"; then + fail "cassandra: generated anti-affinity must not also render alongside a class override" +fi + +# global.affinity.all applies when no class-specific override exists. +write_env <<'EOF' +highAvailability: + mode: preferred +global: + affinity: + all: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + custom: shared + topologyKey: kubernetes.io/hostname +EOF +render_chart_values cassandra "$work_dir/cassandra-global-all.yaml" "$deps" || fail "render cassandra (global.affinity.all)" +awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-all.yaml" | grep -q "custom: shared" || + fail "cassandra: expected global.affinity.all to apply when no class-specific override exists" + +# An explicit {} for a class suppresses the generated anti-affinity entirely — +# presence is the signal, not truthiness. +write_env <<'EOF' +highAvailability: + mode: preferred +global: + affinity: + cassandra: {} + topologySpreadConstraints: + cassandra: [] +EOF +render_chart_values cassandra "$work_dir/cassandra-global-suppressed.yaml" "$deps" || fail "render cassandra (global.affinity.cassandra: {})" +if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-suppressed.yaml" | grep -q "podAntiAffinity:"; then + fail "cassandra: expected explicit global.affinity.cassandra: {} to suppress generated anti-affinity" +fi +if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-suppressed.yaml" | grep -q "topology.kubernetes.io/zone"; then + fail "cassandra: expected explicit global.topologySpreadConstraints.cassandra: [] to suppress generated zone spread" +fi + +echo "== highAvailability enforced: required anti-affinity, hard zone spread ==" +write_env <<'EOF' +highAvailability: + mode: enforced +EOF + +render_chart_values api "$work_dir/api-enforced.yaml" "$core" || fail "render api (enforced)" +grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-enforced.yaml" || + fail "api: expected required anti-affinity when highAvailability.mode=enforced" +grep -q "whenUnsatisfiable: DoNotSchedule" "$work_dir/api-enforced.yaml" || + fail "api: expected DoNotSchedule topology spread when highAvailability.mode=enforced" + +render_chart_values cassandra "$work_dir/cassandra-enforced.yaml" "$deps" || fail "render cassandra (enforced)" +grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-enforced.yaml" || + fail "cassandra: expected required Tier-2 anti-affinity when highAvailability.mode=enforced" +awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-enforced.yaml" | grep -q "whenUnsatisfiable: DoNotSchedule" || + fail "cassandra: expected hard (DoNotSchedule) Tier-2 spread when highAvailability.mode=enforced" + +render_chart_values ratelimiter "$work_dir/ratelimiter-enforced.yaml" "$core" --state-values-set rateLimiter.enabled=true || + fail "render ratelimiter (enforced)" +awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-enforced.yaml" | grep -q "requiredDuringSchedulingIgnoredDuringExecution:" || + fail "ratelimiter: expected required anti-affinity when highAvailability.mode=enforced" + +echo "== highAvailability invalid mode fails render ==" +write_env <<'EOF' +highAvailability: + mode: best-effort +EOF +if render_chart_values api "$work_dir/api-bad.yaml" "$core" 2>"$work_dir/api-bad.err"; then + fail "api: invalid highAvailability.mode should fail helmfile render" +fi +grep -q "highAvailability.mode" "$work_dir/api-bad.err" || + fail "api: expected fail message to mention highAvailability.mode" + +echo "ha-value-wiring: ok" diff --git a/docs/user/high-availability.md b/docs/user/high-availability.md new file mode 100644 index 0000000000..e0bffaaf30 --- /dev/null +++ b/docs/user/high-availability.md @@ -0,0 +1,391 @@ +# High Availability + +This guide explains how to run the self-hosted NVCF control plane in a +high-availability (HA) topology so that it survives the loss of a single node +or availability zone (AZ). It covers the cluster prerequisites you must provide, +how to turn HA on, what each service does under HA, and how to validate the +result. + +HA is opt-in. Single-node installs (local, CI, small proofs of concept) should +leave it off and are unaffected by anything in this guide. + +## Overview + +HA is controlled by a single switch in your Helmfile environment values, +`highAvailability.mode`. There is no per-component HA configuration beneath +it: sizing and placement are derived uniformly from the mode for every +in-scope release, in `deploy/stacks/self-managed/global.yaml.gotmpl`. + +| `highAvailability.mode` | Behavior | +| --- | --- | +| `none` (default) | Keep existing single-replica chart/env values. Nothing in this guide applies. Use for local, CI, and single-node installs. | +| `preferred` | HA sizing (multiple replicas, quorum services at 3). Node/AZ spread is a **preference**: if a second node or AZ has no capacity, pods still schedule (co-located) rather than staying `Pending`. | +| `enforced` | Same HA sizing, but node/AZ spread is **required**: a replica that cannot land on a distinct node/AZ stays `Pending` instead of packing onto an occupied one. | + +`preferred` is the recommended starting point: it gives you the full HA +topology while degrading gracefully on a constrained cluster. Move to +`enforced` once you have confirmed your node pools have capacity in every AZ +and you want hard placement guarantees. + +`preferred` guarantees continuity after a single **pod** failure but provides +only best-effort node/zone separation — under capacity pressure, replicas can +still end up co-located. `enforced` guarantees continuity after a single pod +**or node** failure when the prerequisites below are met. Neither mode alone +claims arbitrary availability-zone or site-loss tolerance; see +[Zone spread for the quorum services](#zone-spread-for-the-quorum-services) +and [Recovery objectives](#recovery-objectives-and-failure-behavior) for what +it takes to survive an AZ loss specifically. + +## Cluster prerequisites + +HA depends on infrastructure the **operator** provides. The stack cannot create +nodes or AZs for you — it only schedules against what you label. + +### 1. Node count + +Both HA modes require **at least 3 schedulable nodes** in the pool(s) that host +control-plane and quorum workloads for the *intended* spread to actually +schedule — the quorum services (Cassandra, NATS, OpenBao) run 3 replicas. +Under `preferred`, fewer nodes degrade to co-located (but still Ready) pods +rather than blocking the install; under `enforced`, insufficient nodes leave +replicas `Pending`. + +### 2. Availability-zone labels + +To spread replicas across failure domains, the scheduler uses the standard +Kubernetes well-known label on your nodes: + +``` +topology.kubernetes.io/zone= +``` + +You (the operator) must ensure this label is present on every node. Managed +Kubernetes services (EKS, AKS, GKE) apply it automatically. On bare-metal or +custom clusters, set it yourself, for example: + +```bash +kubectl label node topology.kubernetes.io/zone=az-1 +``` + +If the label is absent, zone topology spread has nothing to spread across. In +`preferred` this silently degrades to node-level spread only; in +`enforced` zone-constrained pods can stay `Pending`. Aim for capacity in +**at least two AZs** (three is better for the quorum services — see +[Zone spread for the quorum services](#zone-spread-for-the-quorum-services)). + +### 3. Dedicated node pools (recommended) + +For predictable placement and isolation, give the stateful quorum services their +own node pools and label them with `nvcf.nvidia.com/workload`. Configure the +selectors under `global.nodeSelectors` and **set `enabled: true`** — the +selectors ship disabled (`global.nodeSelectors.enabled: false`) and are a no-op +until you turn them on: + +```yaml +global: + nodeSelectors: + enabled: true # required; selectors are ignored when false + controlplane: + key: nvcf.nvidia.com/workload + value: control-plane + cassandra: + key: nvcf.nvidia.com/workload + value: cassandra + vault: + key: nvcf.nvidia.com/workload + value: vault +``` + +| Pool | Selector value | Hosts | +| --- | --- | --- | +| `controlplane` | `control-plane` | All Tier-1 Deployments (api, invocation, grpcproxy, adminIssuerProxy, rateLimiter, natsAuthCalloutService, llmApiGateway, nats) | +| `cassandra` | `cassandra` | Cassandra StatefulSet | +| `vault` | `vault` | OpenBao StatefulSet | + +These same three classes (`controlplane`/`cassandra`/`vault`) are also the +classes used by `global.affinity` and `global.topologySpreadConstraints` +below, so node-pool selection and scheduling-policy tuning stay consistent. + +Each dedicated pool must span the availability zones — that is, have **capacity +in every AZ you want to spread across** (both zones in a 2-AZ cluster, all three +in a 3-AZ cluster). A 3-node Cassandra pool concentrated in one AZ cannot spread +across zones no matter what the stack requests, and with `enabled: false` the +pods fall back to default scheduling regardless of your labels. If you run a +single shared pool instead, set `global.nodeSelectors.enabled: true` with +`global.nodeSelectors.all` and size it to hold every replica on distinct +nodes/AZs. + +## Enabling HA + +Set the mode in your environment file (for example +`deploy/stacks/self-managed/environments/.yaml`): + +```yaml +highAvailability: + mode: preferred +``` + +That single line activates all of the defaults documented below. Then apply +the stack as usual: + +```bash +helmfile -e apply +``` + +An invalid mode fails the render fast with a clear error, so a typo cannot +silently disable HA. + +## What HA changes, by tier + +### Replica-safe Deployments + +Active-active Deployments with no leader election: `api`, `rateLimiter`, +`natsAuthCalloutService`, `adminIssuerProxy`, and `llmApiGateway` (when the +LLM addon is enabled) all follow the same convention. + +> **Note — `invocation-service` and `grpc-proxy` are deferred.** These two are +> stateless too, but their multi-replica scaling is intentionally **held at a +> single replica for now**, pending Envoy support in the self-hosted stack. +> Worker callbacks are host-bound to the specific pod that accepted the request +> (per-pod pod-IP / DNS addressing), which is safe in a single cluster; the +> Envoy dependency is for the cross-cluster case. Until then they keep hostname +> anti-affinity and zone spread (no-ops at one replica) and get **no HA PDB** +> (a `minAvailable: 1` PDB on a singleton would block node drains). See the +> #987/#989 review. + +Under HA each of these gets: + +- **2 replicas.** +- **Hostname pod anti-affinity** so the two replicas never share a node. +- **Zone topology spread** (`topology.kubernetes.io/zone`, `maxSkew: 1`) so they + land in different AZs when zones are labelled. +- **A PodDisruptionBudget** (`minAvailable: 1`) so voluntary disruptions + (drains, upgrades) never take the last replica. +- **A surge rolling-update strategy** (`maxSurge: 1`, `maxUnavailable: 0`) so a + new pod is Ready before an old one is removed (api only, at present). + +Affinity and topology spread are shared scheduling policy, so they can be +tuned once for every release of a class instead of per-component. See +[Tuning scheduling policy](#tuning-scheduling-policy-globalaffinity--globaltopologyspreadconstraints) +below. + +### Quorum services (data durability) + +`Cassandra`, `NATS`, and `OpenBao` run as **3-replica quorum StatefulSets** with: + +- **Hostname pod anti-affinity** so the 3 peers land on 3 distinct nodes. + Following the mode, this is preferred (soft) under `preferred` and required + (hard) under `enforced` — the same convention as replica-safe Deployments. + OpenBao's upstream chart ships a hard anti-affinity that the stack disables + for single-node installs and re-enables (soft/hard by mode) under HA. +- **Zone topology spread** across `topology.kubernetes.io/zone`, same + soft/hard-by-mode convention. See the next section for what it takes for + this to actually protect against an AZ loss. +- **PodDisruptionBudgets** sized to tolerate exactly one voluntary disruption + (`minAvailable: 2` of 3, or OpenBao's equivalent `maxUnavailable: 1`). + +#### Zone spread for the quorum services + +Zone topology spread for the quorum peers is part of the same mode +convention as everything else — it is **not** a separate toggle. That said, +whether it actually protects you from an AZ loss depends on infrastructure +you must provide: + +- **StorageClass `volumeBindingMode: WaitForFirstConsumer`.** Each quorum pod + has a zonal PersistentVolume, and a zonal disk can only attach to a node in + its own AZ. With `WaitForFirstConsumer`, the scheduler places the pod first + (honoring the spread constraint) and the PV is then created in that pod's + zone. With `Immediate` binding the PV's zone is chosen up front and the pod + is pinned to it, which fights the spread constraint and can leave pods + `Pending`. The stack cannot set this for you — it is a property of the + StorageClass you supply. +- **Capacity in at least 3 AZs.** A 3-member quorum only survives an AZ loss + if no single AZ holds a majority. With only 2 AZs one zone inevitably holds + 2 of 3 members, and losing that zone breaks quorum. + +If you do not have 3-AZ capacity or a `WaitForFirstConsumer` StorageClass, +zone spread for the quorum tier will not reliably schedule or will not +protect you from an AZ loss even though it renders. Switching from `enforced` +to `preferred` alone does **not** fix this — `preferred`'s soft +(`ScheduleAnyway`) constraint will let pods co-locate in that case rather than +failing loudly, which hides the gap rather than closing it. To disable +generated zone spread for the quorum tier explicitly instead of relying on a +mode change, use the `global.topologySpreadConstraints` escape hatch: + +```yaml +global: + topologySpreadConstraints: + cassandra: [] + vault: [] +``` + +(NATS shares the `controlplane` class with the stateless tier; scope an +override to just NATS via a component-shaped value instead if you need to +disable spread for NATS specifically without affecting the rest of +`controlplane`.) + +**Cassandra needs one more thing: rack = AZ.** Spreading the *pods* across zones +does not by itself make the *data* zone-diverse. `NetworkTopologyStrategy` +replicates by **rack**, and Cassandra's rack is assigned by the image entrypoint, +not by the pod's Kubernetes zone. Unless each pod's Cassandra rack is set to its +AZ, RF=3 can still place all three data replicas in one rack. Map rack to AZ on +the Cassandra nodes to get true cross-AZ data placement; NATS and OpenBao (Raft) +replicate per member and need only the pod spread. + +Once a quorum pod's PV is created in a zone it is pinned there for the life of +that StatefulSet ordinal — steady-state placement stays spread, but a pod whose +AZ is lost cannot reschedule elsewhere until the AZ returns (its two peers carry +quorum in the meantime). + +Beyond placement, HA also raises the data-durability settings: + +#### NATS JetStream replica factor + +Streams default to a single replica. Under HA the stack sets the JetStream +replica factor (RF) to **3** (`highAvailability.nats.jetstream.replicaFactor`) +on the two services that create streams — `nvcf-api` (via `NVCF_NATS_REPLICAS`) +and `invocation-service` (via `NATS_PROPERTIES__REPLICAS`). JetStream streams +use Raft quorum: RF=3 tolerates the loss of one replica, matching the +3-member NATS cluster. **RF=2 is not sufficient** — a 2-member Raft group +loses quorum the moment either replica is unavailable, so it provides no +resilience benefit over RF=1. + +#### Cassandra replication and consistency + +The Cassandra keyspaces are created with `NetworkTopologyStrategy` and a +replication factor of 3 under HA, and the control-plane services read/write +at `LOCAL_QUORUM`. This is the correct configuration for both single-DC and +multi-AZ deployments: + +- **Single datacenter:** RF=3 with `LOCAL_QUORUM` tolerates the loss of one + replica for reads and writes. +- **Multi-AZ:** because replicas are placed with `NetworkTopologyStrategy`, + labelling nodes by rack/AZ makes Cassandra distribute the 3 replicas across + AZs automatically; `LOCAL_QUORUM` then keeps the cluster available through the + loss of a single AZ. + +No stack change is required to select the strategy — it is +`NetworkTopologyStrategy` in all cases. To get true cross-AZ placement, ensure +the Cassandra nodes carry AZ labels (see the prerequisites above). + +## Tuning scheduling policy (`global.affinity` / `global.topologySpreadConstraints`) + +Affinity and zone topology spread are shared scheduling policy, so you can +tune them once per workload class instead of overriding every component +individually. Both resolve in the same order: + +``` +class-specific global value -> global..all -> the highAvailability.mode-derived convention +``` + +The available classes match `global.nodeSelectors`: `controlplane`, +`cassandra`, `vault`. These values are optional — absence means "use the +mode-derived convention" — and an explicit override replaces the generated +policy for that class entirely, it does not merge with it: + +```yaml +highAvailability: + mode: preferred + +global: + affinity: + cassandra: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: cassandra + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + all: [] # disable generated zone spread everywhere except explicit class overrides +``` + +An explicit `{}` (for `affinity`) or `[]` (for `topologySpreadConstraints`) +suppresses the generated policy for that class — this is different from +omitting the key, which falls back to `all` and then to the convention. + +Component-shaped values (for example `api.affinity`, `cassandra.affinity`) in +your environment file remain the final, single-release escape hatch and take +precedence over both `global.affinity` and the generated convention. + +## Validation + +After applying HA, confirm replicas are spread as expected. + +Check that quorum peers landed on distinct nodes and AZs: + +```bash +# Nodes and their AZ labels +kubectl get nodes -L topology.kubernetes.io/zone + +# Cassandra / NATS / OpenBao pods with their nodes +kubectl -n cassandra-system get pods -o wide +kubectl -n nats-system get pods -o wide +kubectl -n vault-system get pods -o wide +``` + +Confirm the replica-safe Deployments scaled and spread: + +```bash +kubectl -n nvcf get deploy nvcf-api admin-token-issuer-proxy -o wide +kubectl -n nvcf get pods -o wide -l app.kubernetes.io/instance=nvcf-api +# invocation-service and grpc-proxy stay at 1 replica for now (deferred until Envoy) +kubectl -n nvcf get deploy invocation-service grpc-proxy -o wide +``` + +Confirm the JetStream RF took effect (streams report `Replicas: 3`): + +```bash +kubectl -n nats-system exec -it nats-0 -- nats stream ls +kubectl -n nats-system exec -it nats-0 -- nats stream info +``` + +Confirm Cassandra keyspace replication: + +```bash +kubectl -n cassandra-system exec -it cassandra-0 -- \ + cqlsh -e "SELECT keyspace_name, replication FROM system_schema.keyspaces;" +``` + +If any pod is stuck `Pending` under `enforced`, it usually means a node pool +lacks capacity in a second node/AZ. Add capacity, or drop to `preferred` to +let it schedule while you rebalance — but see +[Zone spread for the quorum services](#zone-spread-for-the-quorum-services) +for why that alone does not restore the AZ-loss guarantee if the underlying +capacity gap remains. + +## Recovery objectives and failure behavior + +With HA enabled and capacity in at least two AZs: + +- **Single node loss:** Multi-replica replica-safe Deployments keep + serving from their surviving replica; the scheduler recreates the lost pod on + another node (and the PDB prevents drains from removing the last one). + `invocation-service` and `grpc-proxy` (single replica until Envoy) are briefly + unavailable while the scheduler restarts the pod on another node. Quorum + services (Cassandra RF=3/`LOCAL_QUORUM`, NATS RF=3, OpenBao 3-node Raft) + retain quorum with 2 of 3 members and continue serving reads and writes. +- **Single AZ loss:** Only if zone spread actually took effect — see + [Zone spread for the quorum services](#zone-spread-for-the-quorum-services) + for its prerequisites (3-AZ capacity, `WaitForFirstConsumer` storage). With + those met, the control plane stays available on the surviving AZ(s) and + recovery time is dominated by pod reschedule/restart time rather than any + manual failover. Without them, an AZ loss can take a majority of a quorum + service's members and pause writes even with HA enabled. +- **Two simultaneous quorum-member losses:** A 3-member quorum service loses + quorum and pauses writes until a member returns. This is why three AZs (or at + least three nodes across two AZs, with the third member able to reschedule) is + the durable target. + +HA reduces recovery to automatic rescheduling within surviving failure domains; +it does not replace backups. Continue to back up Cassandra and OpenBao per the +[Control Plane Operations](./control-plane-operations.md) runbooks. + +## Related + +- [Control Plane Operations](./control-plane-operations.md) — service reference, + key rotation, and upgrade runbooks. +- [Infrastructure Sizing](./infrastructure-sizing.md) — node pool sizing + guidance. +- [Helmfile Installation](./helmfile-installation.md) — how environment values + and `global.yaml.gotmpl` are applied. diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml index e05048c70d..12f8b93cfc 100644 --- a/fern/versions/dev.yml +++ b/fern/versions/dev.yml @@ -55,6 +55,8 @@ navigation: path: ../../docs/user/optional-enhancements.md - page: Pod Disruption Budgets path: ../../docs/user/pod-disruption-budgets.md + - page: High Availability + path: ../../docs/user/high-availability.md - page: LLM Function Enablement path: ../../docs/user/llm-function-enablement.md - page: LLM Request Router Load Balancing