From 44055773bed1b3506f39b7493af7f99479ea5fea Mon Sep 17 00:00:00 2001
From: shobham <shobham@nvidia.com>
Date: Wed, 9 Sep 2026 18:19:34 +0530
Subject: [PATCH 01/11] feat(self-managed): add control-plane HA value layer
 (Tier-1) (#986 #987 #988)

highAvailability.mode enum (none/ha-preferred/ha-enforced) mapped through global.yaml.gotmpl to chart values for the stateless and hot-path tiers: replicaCount, hostname pod anti-affinity, zone topology spread, PDBs (including a new nvcf-api PDB template), and a surge rolling-update strategy. Default mode: none leaves single-node installs unchanged. Closes #986, #987, #988.
---
 .../templates/poddisruptionbudget.yaml        |  39 +++
 .../helm/cloud-functions/nvcf-api/values.yaml |  10 +
 deploy/stacks/self-managed/Makefile           |   1 +
 .../self-managed/environments/base.yaml       |  78 ++++++
 deploy/stacks/self-managed/global.yaml.gotmpl | 232 +++++++++++++++++-
 .../self-managed/tests/ha-value-wiring.sh     | 163 ++++++++++++
 6 files changed, 520 insertions(+), 3 deletions(-)
 create mode 100644 deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml
 create mode 100755 deploy/stacks/self-managed/tests/ha-value-wiring.sh

diff --git a/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml b/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml
new file mode 100644
index 0000000000..37878fa917
--- /dev/null
+++ b/deploy/helm/cloud-functions/nvcf-api/templates/poddisruptionbudget.yaml
@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.api.podDisruptionBudget.enabled }}
+{{- $pdbMinAvail := .Values.api.podDisruptionBudget.minAvailable | toString }}
+{{- $pdbMaxUnavail := .Values.api.podDisruptionBudget.maxUnavailable | toString }}
+{{- if and (ne $pdbMinAvail "") (ne $pdbMaxUnavail "") }}
+{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both" }}
+{{- end }}
+{{- if and (eq $pdbMinAvail "") (eq $pdbMaxUnavail "") }}
+{{- fail "podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" }}
+{{- end }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "nvcf-api.fullname" . }}
+  namespace: {{ include "nvcf-api.namespace" . }}
+  labels: {{- include "nvcf-api.labels" . | nindent 4 }}
+spec:
+  {{- if ne $pdbMinAvail "" }}
+  minAvailable: {{ .Values.api.podDisruptionBudget.minAvailable }}
+  {{- else }}
+  maxUnavailable: {{ .Values.api.podDisruptionBudget.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels: {{- include "nvcf-api.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/deploy/helm/cloud-functions/nvcf-api/values.yaml b/deploy/helm/cloud-functions/nvcf-api/values.yaml
index b10d4043f7..b98265cc8b 100644
--- a/deploy/helm/cloud-functions/nvcf-api/values.yaml
+++ b/deploy/helm/cloud-functions/nvcf-api/values.yaml
@@ -26,6 +26,16 @@ api:
   # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
   replicaCount: 1
 
+  # PodDisruptionBudget for the API. Disabled by default; the self-managed
+  # Helmfile enables it (minAvailable 1) via highAvailability.stateless when HA
+  # is on. See deploy/stacks/self-managed/global.yaml.gotmpl.
+  podDisruptionBudget:
+    enabled: false
+    # minAvailable and maxUnavailable are mutually exclusive; set exactly one.
+    # Accepts an integer or a percentage string (e.g. 1 or "50%").
+    minAvailable: ""
+    maxUnavailable: ""
+
   # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
   image:
     registry: "" # must be supplied
diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile
index ee727878f0..639aad87d5 100644
--- a/deploy/stacks/self-managed/Makefile
+++ b/deploy/stacks/self-managed/Makefile
@@ -27,6 +27,7 @@ test:
 	@tests/nats-placement-tags.sh
 	@tests/nats-tls-wiring.sh
 	@tests/notary-image-repository.sh
+	@tests/ha-value-wiring.sh
 
 test-published-charts:
 	@: "$${NVCF_PUBLISHED_CHART_REGISTRY:?NVCF_PUBLISHED_CHART_REGISTRY is required}"
diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 4dcf4282cf..690578b89d 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -714,3 +714,81 @@ grpcproxy:
   # an external compute-worker endpoint.
   natsServiceURL: ""
   workerConnectBaseURL: ""
+
+# =============================================================================
+# Control-plane high availability
+# =============================================================================
+# Resilience settings are exposed as a single highAvailability: block in
+# self-managed Helmfile environment values.
+# deploy/stacks/self-managed/global.yaml.gotmpl maps highAvailability.* onto
+# chart values (replicaCount, affinity, PDB, update strategy).
+#
+# deploy/stacks/self-managed/
+# ├── environments/<env>.yaml     # highAvailability: configuration
+# └── global.yaml.gotmpl          # value mapping to charts
+#
+# mode: none         (default) — keep existing env/chart replica and PDB values
+#                                (local / CI / BDD).
+#       ha-preferred           — HA sizing below; hostname anti-affinity is
+#                                preferred (still schedule if a second node/AZ
+#                                has no capacity). Requires ≥3 schedulable nodes.
+#       ha-enforced            — same HA sizing; hostname anti-affinity is
+#                                required (second replica stays Pending rather
+#                                than packing). Requires ≥3 schedulable nodes.
+#
+# In-scope charts MUST expose the required value hooks. Leave mode: none
+# for single-node installs.
+# =============================================================================
+highAvailability:
+  mode: none
+
+  # Stateless control-plane Deployments: nvcf-api, invocation-service,
+  # grpc-proxy, admin-token-issuer-proxy, and llm-api-gateway (when the LLM
+  # addon is enabled). Active-active; no application leader election.
+  stateless:
+    replicaCount: 2
+    podAntiAffinity:
+      enabled: true
+    # Spread replicas across availability zones. Nodes MUST be labelled
+    # topology.kubernetes.io/zone=<az>. whenUnsatisfiable follows mode:
+    # ha-preferred → ScheduleAnyway, ha-enforced → DoNotSchedule.
+    topologySpread:
+      enabled: true
+      maxSkew: 1
+    podDisruptionBudget:
+      enabled: true
+      minAvailable: 1
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+
+  # Hot-path helper Deployments raised to 2 replicas under HA so a single pod
+  # loss does not immediately hurt request/auth traffic: rateLimiter,
+  # nats-auth-callout. Reuse stateless.podAntiAffinity (hostname spread) and
+  # stateless.topologySpread (zone spread).
+  hotPath:
+    replicaCount: 2
+    podDisruptionBudget:
+      enabled: true
+      minAvailable: 1
+
+  nats:
+    replicas: 3
+    podDisruptionBudget:
+      enabled: true
+      merge:
+        spec:
+          minAvailable: 2
+
+  openbao:
+    ha:
+      enabled: true
+      replicas: 3
+    injector:
+      replicas: 2
+
+  cassandra:
+    replicaCount: 3
+    podDisruptionBudget:
+      enabled: true
+      minAvailable: 2
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 3eb2cad732..7f9a48cc7f 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -24,6 +24,99 @@ tolerations:
 {{- end -}}
 {{- end -}}
 
+{{/*
+highAvailability helpers.
+
+global.yaml.gotmpl maps highAvailability.* onto chart values (replicaCount,
+affinity, PDB, update strategy).
+
+highAvailability.mode:
+  none          — keep existing env/chart defaults (local / CI / BDD).
+  ha-preferred  — HA sizing; preferred hostname anti-affinity.
+  ha-enforced   — HA sizing; required hostname anti-affinity.
+
+Requires ≥3 schedulable nodes unless mode is none. Nested keys under
+stateless/hotPath/nats/openbao/cassandra override those defaults.
+
+stateless — nvcf-api, invocation, grpc-proxy, admin-token-issuer-proxy, and
+  llm-api-gateway (LLM addon): replicaCount, hostname anti-affinity, zone
+  topology spread, PDB.
+hotPath   — rateLimiter, nats-auth-callout: 2 replicas + anti-affinity +
+  zone topology spread + PDB.
+
+In-scope charts MUST expose the required value hooks.
+*/}}
+{{- define "nvcf.ha.statelessReplicaCount" -}}
+{{- dig "highAvailability" "stateless" "replicaCount" 2 . -}}
+{{- end -}}
+
+{{/* Replica count for hot-path helper Deployments (rateLimiter,
+     nats-auth-callout) under HA. Defaults to 2. */}}
+{{- define "nvcf.ha.hotPathReplicaCount" -}}
+{{- dig "highAvailability" "hotPath" "replicaCount" 2 . -}}
+{{- end -}}
+
+{{/* Soft/hard pod anti-affinity on hostname for a Helm release instance name.
+     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
+{{- define "nvcf.ha.statelessAffinity" -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- $paa := dig "highAvailability" "stateless" "podAntiAffinity" dict .Values -}}
+{{- if dig "enabled" true $paa -}}
+affinity:
+  podAntiAffinity:
+{{- if eq $haMode "ha-enforced" }}
+    requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchExpressions:
+            - key: app.kubernetes.io/instance
+              operator: In
+              values:
+                - {{ .instance | quote }}
+        topologyKey: kubernetes.io/hostname
+{{- else }}
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+              - key: app.kubernetes.io/instance
+                operator: In
+                values:
+                  - {{ .instance | quote }}
+          topologyKey: kubernetes.io/hostname
+{{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Zone-level topology spread for a Helm release instance name. Spreads
+     replicas across topology.kubernetes.io/zone. whenUnsatisfiable follows the
+     mode: ha-preferred → ScheduleAnyway (best effort), ha-enforced →
+     DoNotSchedule (hard). Nodes MUST carry topology.kubernetes.io/zone.
+     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
+{{- define "nvcf.ha.statelessTopologySpread" -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- $ts := dig "highAvailability" "stateless" "topologySpread" dict .Values -}}
+{{- if dig "enabled" true $ts -}}
+topologySpreadConstraints:
+  - maxSkew: {{ dig "maxSkew" 1 $ts }}
+    topologyKey: topology.kubernetes.io/zone
+    whenUnsatisfiable: {{ if eq $haMode "ha-enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/instance: {{ .instance | quote }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }}
+{{- if not (has $haMode (list "none" "ha-preferred" "ha-enforced")) }}
+{{- fail (printf "highAvailability.mode must be none, ha-preferred, or ha-enforced, got %q" $haMode) }}
+{{- end }}
+{{- $haEnabled := or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") }}
+
 cassandra:
   global:
     {{- if .Values.global.imagePullSecrets }}
@@ -36,13 +129,20 @@ cassandra:
     defaultStorageClass: {{ .Values.global.storageClass }}
     {{- end }}
 
-  replicaCount: {{ dig "cassandra" "replicaCount" 3 .Values }}
+  replicaCount: {{ if $haEnabled }}{{ dig "highAvailability" "cassandra" "replicaCount" 3 .Values }}{{ else }}{{ dig "cassandra" "replicaCount" 3 .Values }}{{ end }}
   resourcesPreset: {{ dig "cassandra" "resourcesPreset" "xlarge" .Values }}
 
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "cassandra" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "cassandra" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
   {{- with include "nvcf.nodeSelector" (dict "type" "cassandra" "selectors" .Values.global.nodeSelectors) }}
   {{- . | nindent 2 }}
@@ -155,7 +255,7 @@ openbao:
     {{- with include "nvcf.tolerations" (dict "type" "vault" "tolerations" .Values.global.tolerations) }}
     {{- . | nindent 4 }}
     {{- end }}
-    replicas: {{ .Values.openbao.injector.replicas }}
+    replicas: {{ if $haEnabled }}{{ dig "highAvailability" "openbao" "injector" "replicas" 2 .Values }}{{ else }}{{ .Values.openbao.injector.replicas }}{{ end }}
     {{- $nvcfUiEnabled := dig "addons" "nvcfUi" "enabled" false .Values }}
     {{- with dig "openbao" "injector" "webhook" dict .Values }}
     webhook:
@@ -193,6 +293,10 @@ openbao:
       size: {{ .Values.global.storageSize | default "10Gi" }}
 
     ha:
+      {{- if $haEnabled }}
+      enabled: {{ dig "highAvailability" "openbao" "ha" "enabled" true .Values }}
+      replicas: {{ dig "highAvailability" "openbao" "ha" "replicas" 3 .Values }}
+      {{- end }}
       {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }}
       disruptionBudget:
         {{- toYaml . | nindent 8 }}
@@ -262,6 +366,11 @@ nats:
       {{- if kindIs "bool" $allowNonTls }}
       allow_non_tls: {{ $allowNonTls }}
       {{- end }}
+    {{- if $haEnabled }}
+    cluster:
+      enabled: true
+      replicas: {{ dig "highAvailability" "nats" "replicas" 3 .Values }}
+    {{- end }}
     {{- if .Values.global.storageClass }}
     jetstream:
       fileStore:
@@ -274,10 +383,17 @@ nats:
         {{- toYaml . | nindent 8 }}
     {{- end }}
 
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "nats" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "nats" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 apikeys:
   fullnameOverride: api-keys
@@ -308,6 +424,15 @@ apikeys:
   {{- end }}
 
 natsAuthCalloutService:
+  {{- if $haEnabled }}
+  replicaCount: {{ include "nvcf.ha.hotPathReplicaCount" .Values }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "nats-auth-callout-service") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "nats-auth-callout-service") }}
+  {{- . | nindent 2 }}
+  {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -483,6 +608,9 @@ natsAuthCalloutService:
 {{- $apiRemoteConfigData = mergeOverwrite $apiRemoteConfigData $stackApiRemoteConfigData }}
 api:
   fullnameOverride: nvcf-api
+  {{- if $haEnabled }}
+  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -501,6 +629,23 @@ api:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "api") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "api") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
+  {{- with dig "api" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- end }}
   {{- if $apiRemoteConfigData }}
   remoteConfig:
     configData:
@@ -607,6 +752,9 @@ invocation:
   {{- end }}
   {{- $invocationEnv = mergeOverwrite $invocationEnv (deepCopy $configuredInvocationEnv) }}
   fullnameOverride: invocation-service
+  {{- if $haEnabled }}
+  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -620,6 +768,12 @@ invocation:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "invocation-service") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "invocation-service") }}
+  {{- . | nindent 2 }}
+  {{- end }}
 
   metrics:
     enabled: {{ or (eq (dig "observability" "profile" "disabled" .Values) "control") (eq (dig "observability" "profile" "disabled" .Values) "all") }}
@@ -634,10 +788,17 @@ invocation:
     baggageAttributeAllowlist:
     {{- toYaml . | nindent 6 }}
   {{- end }}
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "invocation" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 nvctApi:
   fullnameOverride: nvct-api
@@ -685,6 +846,9 @@ nvctApi:
 
 grpcproxy:
   fullnameOverride: grpc-proxy
+  {{- if $haEnabled }}
+  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -698,6 +862,19 @@ grpcproxy:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- if $haEnabled }}
+  {{- $grpcAffinity := include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "grpc-proxy") }}
+  {{- $grpcTopologySpread := include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "grpc-proxy") }}
+  {{- if or $grpcAffinity $grpcTopologySpread }}
+  deployment:
+    {{- with $grpcAffinity }}
+    {{- . | nindent 4 }}
+    {{- end }}
+    {{- with $grpcTopologySpread }}
+    {{- . | nindent 4 }}
+    {{- end }}
+  {{- end }}
+  {{- end }}
   {{- with $grpcProxyWorkerConnectBaseURL }}
   workerConnectBaseURL: {{ . | quote }}
   {{- end }}
@@ -719,10 +896,17 @@ grpcproxy:
     RATE_LIMIT_ENABLED: "true"
     RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777"
     {{- end }}
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 rateLimiter:
   {{- if .Values.global.imagePullSecrets }}
@@ -736,15 +920,28 @@ rateLimiter:
   nodeSelector:
     {{ .Values.global.nodeSelectors.controlplane.key }}: {{ .Values.global.nodeSelectors.controlplane.value }}
   {{- end }}
-  replicaCount: {{ .Values.rateLimiter.replicaCount }}
+  replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.hotPathReplicaCount" .Values }}{{ else }}{{ .Values.rateLimiter.replicaCount }}{{ end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "ratelimiter") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "ratelimiter") }}
+  {{- . | nindent 2 }}
+  {{- end }}
   {{- if .Values.global.observability.tracing.enabled }}
   env:
     OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}"
   {{- end }}
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "hotPath" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "rateLimiter" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 ess:
   fullnameOverride: ess-api
@@ -837,6 +1034,9 @@ sis:
 
 adminIssuerProxy:
   fullnameOverride: admin-token-issuer-proxy
+  {{- if $haEnabled }}
+  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -853,6 +1053,12 @@ adminIssuerProxy:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "admin-token-issuer-proxy") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "admin-token-issuer-proxy") }}
+  {{- . | nindent 2 }}
+  {{- end }}
   gateway:
     enabled: true
     namespace: {{ .Values.ingress.gatewayApi.gateways.shared.namespace }}
@@ -860,10 +1066,17 @@ adminIssuerProxy:
       name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }}
     hostname: "api-keys.{{ .Values.global.domain }}"
     path: "/v1/admin/keys"
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "adminIssuerProxy" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 stateMetrics:
   enabled: false
@@ -995,6 +1208,12 @@ llmApiGateway:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "llm-api-gateway") }}
+  {{- . | nindent 2 }}
+  {{- end }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "llm-api-gateway") }}
+  {{- . | nindent 2 }}
+  {{- end }}
   {{- if dig "addons" "llm" "gateway" "auth" "grpcInsecure" false .Values }}
   config:
     nvcfGrpcInsecure: true
@@ -1009,10 +1228,17 @@ llmApiGateway:
       {{- if .Values.global.observability.tracing.enabled }}
       endpoint: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}"
       {{- end }}
+  {{- if $haEnabled }}
+  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  podDisruptionBudget:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- else }}
   {{- with dig "llmApiGateway" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- end }}
 
 {{- $pylonGrpcDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonGrpcDialAddress" "" .Values | default "" | toString | trim }}
 {{- $pylonReverseTunnelDialAddress := dig "addons" "llm" "requestRouter" "backendRouter" "pylonReverseTunnelDialAddress" "" .Values | default "" | toString | trim }}
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
new file mode 100755
index 0000000000..a03cafd58e
--- /dev/null
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+# Test that highAvailability values thread from environment files through
+# global.yaml.gotmpl into chart values for stateless / quorum releases.
+set -euo pipefail
+
+stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+work_dir="$(mktemp -d)"
+test_stack_dir="$work_dir/self-managed"
+environment_name="ha-value-wiring-test"
+environment_file="$test_stack_dir/environments/$environment_name.yaml"
+secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml"
+trap 'rm -rf "$work_dir"' EXIT
+
+fail() {
+  echo "ha-value-wiring: $*" >&2
+  exit 1
+}
+
+mkdir -p "$test_stack_dir"
+cp -R "$stack_dir"/. "$test_stack_dir"
+printf '{}\n' >"$secrets_file"
+
+render_chart_values() {
+  local release="$1"
+  local output_file="$2"
+  local helmfile_file="$3"
+  shift 3
+
+  # global.yaml.gotmpl evaluates adminIssuerProxy gateway refs for every release.
+  HELMFILE_ENV="$environment_name" \
+    HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
+    helmfile \
+      --file "$helmfile_file" \
+      --environment default \
+      --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \
+      --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \
+      --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \
+      --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \
+      --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \
+      --selector "name=$release" \
+      "$@" \
+      write-values \
+      --output-file-template "$output_file"
+}
+
+write_env() {
+  cat >"$environment_file"
+}
+
+deps="$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl"
+core="$test_stack_dir/helmfile.d/02-core.yaml.gotmpl"
+
+echo "== highAvailability mode none: chart defaults / base values unchanged =="
+write_env <<'EOF'
+highAvailability:
+  mode: none
+EOF
+
+render_chart_values api "$work_dir/api-off.yaml" "$core" || fail "render api (ha none)"
+# HA must not inject replicaCount into the api values when disabled.
+if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "replicaCount:"; then
+  fail "api: HA replicaCount leaked while highAvailability.mode=none"
+fi
+if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "podAntiAffinity:"; then
+  fail "api: HA affinity leaked while highAvailability.mode=none"
+fi
+if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "topologySpreadConstraints:"; then
+  fail "api: topology spread leaked while highAvailability.mode=none"
+fi
+if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "podDisruptionBudget:"; then
+  fail "api: PDB leaked while highAvailability.mode=none"
+fi
+
+render_chart_values ratelimiter "$work_dir/ratelimiter-off.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
+  fail "render ratelimiter (ha none)"
+if awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-off.yaml" | grep -q "podAntiAffinity:"; then
+  fail "ratelimiter: HA affinity leaked while highAvailability.mode=none"
+fi
+
+echo "== highAvailability ha-preferred: stateless / quorum sizing =="
+write_env <<'EOF'
+highAvailability:
+  mode: ha-preferred
+EOF
+
+render_chart_values api "$work_dir/api-on.yaml" "$core" || fail "render api (ha-preferred)"
+grep -E "replicaCount:[[:space:]]*2" "$work_dir/api-on.yaml" >/dev/null ||
+  fail "api: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-on.yaml" ||
+  fail "api: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+grep -q "topologySpreadConstraints:" "$work_dir/api-on.yaml" ||
+  fail "api: expected topologySpreadConstraints when highAvailability.mode=ha-preferred"
+grep -q "topology.kubernetes.io/zone" "$work_dir/api-on.yaml" ||
+  fail "api: expected zone topologyKey when highAvailability.mode=ha-preferred"
+grep -q "whenUnsatisfiable: ScheduleAnyway" "$work_dir/api-on.yaml" ||
+  fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=ha-preferred"
+awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-on.yaml" | grep -q "podDisruptionBudget:" ||
+  fail "api: expected podDisruptionBudget when highAvailability.mode=ha-preferred"
+
+render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (ha-preferred)"
+grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null ||
+  fail "cassandra: expected replicaCount 3 when highAvailability.mode=ha-preferred"
+grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-on.yaml" | grep -q "enabled: true" ||
+  fail "cassandra: expected HA PDB enabled"
+
+render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (ha-preferred)"
+grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
+  fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=ha-preferred"
+
+render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (ha-preferred)"
+grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
+  fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=ha-preferred"
+
+# Hot-path helpers (#988): rateLimiter + nats-auth-callout to 2 replicas.
+render_chart_values ratelimiter "$work_dir/ratelimiter-on.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
+  fail "render ratelimiter (ha-preferred)"
+awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null ||
+  fail "ratelimiter: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
+  fail "ratelimiter: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "ratelimiter: expected zone topology spread when highAvailability.mode=ha-preferred"
+
+render_chart_values nats-auth-callout-service "$work_dir/natsauth-on.yaml" "$core" ||
+  fail "render nats-auth-callout (ha-preferred)"
+awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null ||
+  fail "nats-auth-callout: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+
+# llm-api-gateway (#987): stateless anti-affinity when the LLM addon is on.
+render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-values-set addons.llm.enabled=true ||
+  fail "render llm-api-gateway (ha-preferred)"
+awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
+  fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+
+echo "== highAvailability ha-enforced: required anti-affinity =="
+write_env <<'EOF'
+highAvailability:
+  mode: ha-enforced
+EOF
+
+render_chart_values api "$work_dir/api-enforced.yaml" "$core" || fail "render api (ha-enforced)"
+grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-enforced.yaml" ||
+  fail "api: expected required anti-affinity when highAvailability.mode=ha-enforced"
+grep -q "whenUnsatisfiable: DoNotSchedule" "$work_dir/api-enforced.yaml" ||
+  fail "api: expected DoNotSchedule topology spread when highAvailability.mode=ha-enforced"
+
+render_chart_values ratelimiter "$work_dir/ratelimiter-enforced.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
+  fail "render ratelimiter (ha-enforced)"
+awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-enforced.yaml" | grep -q "requiredDuringSchedulingIgnoredDuringExecution:" ||
+  fail "ratelimiter: expected required anti-affinity when highAvailability.mode=ha-enforced"
+
+echo "== highAvailability invalid mode fails render =="
+write_env <<'EOF'
+highAvailability:
+  mode: best-effort
+EOF
+if render_chart_values api "$work_dir/api-bad.yaml" "$core" 2>"$work_dir/api-bad.err"; then
+  fail "api: invalid highAvailability.mode should fail helmfile render"
+fi
+grep -q "highAvailability.mode" "$work_dir/api-bad.err" ||
+  fail "api: expected fail message to mention highAvailability.mode"
+
+echo "ha-value-wiring: ok"

From 52c21fa68f05ce5ba67bff749dc31f093c7b9311 Mon Sep 17 00:00:00 2001
From: shobham <shobham@nvidia.com>
Date: Wed, 9 Sep 2026 18:19:34 +0530
Subject: [PATCH 02/11] feat(self-managed): Tier-2 quorum resilience + HA
 operator docs (#989)

Tier-2 quorum pod anti-affinity (Cassandra/NATS/OpenBao), soft/hard by mode. NATS JetStream RF=2 wired via env on the stream creators (nvcf-api, invocation). Cassandra already uses NetworkTopologyStrategy + LOCAL_QUORUM (documented). Adds docs/v0.6.1/high-availability.md operator guide. Part of epic #985.
---
 .../self-managed/environments/base.yaml       |  18 ++
 deploy/stacks/self-managed/global.yaml.gotmpl |  84 ++++++-
 .../self-managed/tests/ha-value-wiring.sh     |  33 +++
 docs/v0.6.1/high-availability.md              | 237 ++++++++++++++++++
 docs/v0.6.1/index.md                          |   2 +
 5 files changed, 373 insertions(+), 1 deletion(-)
 create mode 100644 docs/v0.6.1/high-availability.md

diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 690578b89d..02da46f73e 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -772,8 +772,26 @@ highAvailability:
       enabled: true
       minAvailable: 1
 
+  # Tier-2 quorum services (Cassandra, NATS, OpenBao). Under HA their peers get
+  # hostname pod anti-affinity (matching app.kubernetes.io/instance per release)
+  # so the 3 quorum members land on distinct nodes. whenUnsatisfiable follows
+  # mode: ha-preferred → preferred/soft (stays schedulable on small pools),
+  # ha-enforced → required/hard. The upstream OpenBao chart ships a hard
+  # anti-affinity by default that the stack disables for single-node installs;
+  # this re-enables it (soft/hard by mode) under HA. Set enabled: false to keep
+  # the chart defaults.
+  tier2:
+    podAntiAffinity:
+      enabled: true
+
   nats:
     replicas: 3
+    # JetStream stream replica factor (RF). Applied under HA to the stream
+    # creators — nvcf-api and invocation-service — so the worker/result streams
+    # they declare are replicated across the NATS cluster (survives a node
+    # loss). RF must be <= nats.replicas. Default 2 per the SDD.
+    jetstream:
+      replicaFactor: 2
     podDisruptionBudget:
       enabled: true
       merge:
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 7f9a48cc7f..b4cbaa22fe 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -43,6 +43,9 @@ stateless — nvcf-api, invocation, grpc-proxy, admin-token-issuer-proxy, and
   topology spread, PDB.
 hotPath   — rateLimiter, nats-auth-callout: 2 replicas + anti-affinity +
   zone topology spread + PDB.
+tier2     — Cassandra, NATS, OpenBao: quorum sizing (3) + PDB + hostname pod
+  anti-affinity so the 3 peers land on distinct nodes. NATS JetStream RF is
+  applied to the stream creators (nvcf-api, invocation) via their env.
 
 In-scope charts MUST expose the required value hooks.
 */}}
@@ -56,6 +59,14 @@ In-scope charts MUST expose the required value hooks.
 {{- dig "highAvailability" "hotPath" "replicaCount" 2 . -}}
 {{- end -}}
 
+{{/* JetStream stream replica factor (RF) under HA. Applied to the stream
+     creators — nvcf-api (Java, nvcf.nats.replicas) and invocation (Rust,
+     nats_properties.replicas) — via their env so the streams they declare are
+     replicated across the NATS cluster. Defaults to 2 per the SDD. */}}
+{{- define "nvcf.ha.natsStreamReplicas" -}}
+{{- dig "highAvailability" "nats" "jetstream" "replicaFactor" 2 . -}}
+{{- end -}}
+
 {{/* Soft/hard pod anti-affinity on hostname for a Helm release instance name.
      Context: dict "Values" $.Values "instance" "<release-instance>" */}}
 {{- define "nvcf.ha.statelessAffinity" -}}
@@ -111,6 +122,45 @@ topologySpreadConstraints:
 {{- end -}}
 {{- end -}}
 
+{{/* Hostname pod anti-affinity for a Tier-2 quorum release (Cassandra, NATS,
+     OpenBao) so the 3 quorum peers land on distinct nodes. Matches
+     app.kubernetes.io/instance = release name. ha-preferred → preferred (soft,
+     stays schedulable on small pools), ha-enforced → required (hard). Emits the
+     podAntiAffinity body only (no affinity: wrapper) so callers can nest it as a
+     map (Cassandra affinity, NATS podTemplate.merge.spec.affinity) or a string
+     (OpenBao server.affinity). Gated by highAvailability.tier2.podAntiAffinity.
+     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
+{{- define "nvcf.ha.tier2Affinity" -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- $paa := dig "highAvailability" "tier2" "podAntiAffinity" dict .Values -}}
+{{- if dig "enabled" true $paa -}}
+podAntiAffinity:
+{{- if eq $haMode "ha-enforced" }}
+  requiredDuringSchedulingIgnoredDuringExecution:
+    - labelSelector:
+        matchExpressions:
+          - key: app.kubernetes.io/instance
+            operator: In
+            values:
+              - {{ .instance | quote }}
+      topologyKey: kubernetes.io/hostname
+{{- else }}
+  preferredDuringSchedulingIgnoredDuringExecution:
+    - weight: 100
+      podAffinityTerm:
+        labelSelector:
+          matchExpressions:
+            - key: app.kubernetes.io/instance
+              operator: In
+              values:
+                - {{ .instance | quote }}
+        topologyKey: kubernetes.io/hostname
+{{- end }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }}
 {{- if not (has $haMode (list "none" "ha-preferred" "ha-enforced")) }}
 {{- fail (printf "highAvailability.mode must be none, ha-preferred, or ha-enforced, got %q" $haMode) }}
@@ -150,6 +200,12 @@ cassandra:
   {{- with include "nvcf.tolerations" (dict "type" "cassandra" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
+  {{- if $haEnabled }}
+  {{- with include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "cassandra") }}
+  affinity:
+    {{- . | nindent 4 }}
+  {{- end }}
+  {{- end }}
   persistence:
     size: {{ .Values.global.storageSize | default "10Gi" }}
 
@@ -276,6 +332,14 @@ openbao:
     podDisruptionBudget:
       minAvailable: {{ dig "openbao" "injector" "podDisruptionBudget" "minAvailable" 1 .Values }}
   server:
+    {{- if $haEnabled }}
+    {{- with include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "openbao-server") }}
+    # String form: the OpenBao chart tpl-renders server.affinity. The wrapper
+    # sets it to "" for single-node installs; HA re-enables hostname spread.
+    affinity: |
+      {{- . | nindent 6 }}
+    {{- end }}
+    {{- end }}
     image:
       registry: {{ .Values.global.image.registry }}
       repository: {{ .Values.global.image.repository }}/nvcf-openbao
@@ -339,7 +403,11 @@ nats:
       repository: {{ .Values.global.image.repository }}/alpine-k8s
   {{- $natsNs := include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) -}}
   {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) -}}
-  {{- if or $natsNs $natsTol }}
+  {{- $natsAffinity := "" -}}
+  {{- if $haEnabled -}}
+  {{- $natsAffinity = include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "nats") -}}
+  {{- end -}}
+  {{- if or $natsNs $natsTol $natsAffinity }}
   podTemplate:
     merge:
       spec:
@@ -349,6 +417,10 @@ nats:
         {{- with $natsTol }}
         {{- . | nindent 8 }}
         {{- end }}
+        {{- with $natsAffinity }}
+        affinity:
+          {{- . | nindent 10 }}
+        {{- end }}
   {{- end }}
 
   # One merged config: block -- a second config: key under nats: would be
@@ -727,6 +799,11 @@ api:
   {{- if .Values.global.observability.tracing.enabled }}
   {{- $_ := set $apiEnv "MANAGEMENT_OTLP_TRACING_ENDPOINT" (printf "%s://%s:%v/v1/traces" .Values.global.observability.tracing.collectorProtocol .Values.global.observability.tracing.collectorEndpoint .Values.global.observability.tracing.collectorPort) }}
   {{- end }}
+  {{- if $haEnabled }}
+  {{- /* JetStream RF for the worker/result streams this service declares
+         (nvcf.nats.replicas → NVCF_NATS_REPLICAS via Spring relaxed binding). */}}
+  {{- $_ := set $apiEnv "NVCF_NATS_REPLICAS" (include "nvcf.ha.natsStreamReplicas" .Values) }}
+  {{- end }}
   {{- $apiEnv = mergeOverwrite $apiEnv $renderedApiEnv }}
   env:
     {{- toYaml $apiEnv | nindent 4 }}
@@ -750,6 +827,11 @@ invocation:
   {{- with $invocationWorkerBaseURL }}
   {{- $_ := set $invocationEnv "WORKER_STREAM_PROPERTIES__SELF_ADDRESS" . }}
   {{- end }}
+  {{- if $haEnabled }}
+  {{- /* JetStream RF for the streams this service declares
+         (nats_properties.replicas → NATS_PROPERTIES__REPLICAS). */}}
+  {{- $_ := set $invocationEnv "NATS_PROPERTIES__REPLICAS" (include "nvcf.ha.natsStreamReplicas" .Values) }}
+  {{- end }}
   {{- $invocationEnv = mergeOverwrite $invocationEnv (deepCopy $configuredInvocationEnv) }}
   fullnameOverride: invocation-service
   {{- if $haEnabled }}
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index a03cafd58e..59ad0e642a 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -77,6 +77,21 @@ if awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-off
   fail "ratelimiter: HA affinity leaked while highAvailability.mode=none"
 fi
 
+# Tier-2 (#989): anti-affinity must not leak into the quorum charts when off.
+render_chart_values cassandra "$work_dir/cassandra-off.yaml" "$deps" || fail "render cassandra (ha none)"
+if grep -q "podAntiAffinity:" "$work_dir/cassandra-off.yaml"; then
+  fail "cassandra: Tier-2 anti-affinity leaked while highAvailability.mode=none"
+fi
+
+# JetStream RF (#989): the RF env must not leak into the stream creators when off.
+if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "NVCF_NATS_REPLICAS:"; then
+  fail "api: JetStream RF env leaked while highAvailability.mode=none"
+fi
+render_chart_values invocation-service "$work_dir/invocation-off.yaml" "$core" || fail "render invocation (ha none)"
+if grep -q "NATS_PROPERTIES__REPLICAS:" "$work_dir/invocation-off.yaml"; then
+  fail "invocation: JetStream RF env leaked while highAvailability.mode=none"
+fi
+
 echo "== highAvailability ha-preferred: stateless / quorum sizing =="
 write_env <<'EOF'
 highAvailability:
@@ -96,20 +111,34 @@ grep -q "whenUnsatisfiable: ScheduleAnyway" "$work_dir/api-on.yaml" ||
   fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=ha-preferred"
 awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-on.yaml" | grep -q "podDisruptionBudget:" ||
   fail "api: expected podDisruptionBudget when highAvailability.mode=ha-preferred"
+grep -q 'NVCF_NATS_REPLICAS: "2"' "$work_dir/api-on.yaml" ||
+  fail "api: expected JetStream RF NVCF_NATS_REPLICAS=2 when highAvailability.mode=ha-preferred"
+
+render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || fail "render invocation (ha-preferred)"
+grep -q 'NATS_PROPERTIES__REPLICAS: "2"' "$work_dir/invocation-on.yaml" ||
+  fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=2 when highAvailability.mode=ha-preferred"
 
 render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (ha-preferred)"
 grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null ||
   fail "cassandra: expected replicaCount 3 when highAvailability.mode=ha-preferred"
 grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-on.yaml" | grep -q "enabled: true" ||
   fail "cassandra: expected HA PDB enabled"
+grep -q "podAntiAffinity:" "$work_dir/cassandra-on.yaml" ||
+  fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-on.yaml" ||
+  fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
 
 render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (ha-preferred)"
 grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
   fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=ha-preferred"
+grep -q "podAntiAffinity:" "$work_dir/openbao-on.yaml" ||
+  fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
 
 render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (ha-preferred)"
 grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
   fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=ha-preferred"
+grep -q "podAntiAffinity:" "$work_dir/nats-on.yaml" ||
+  fail "nats: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
 
 # Hot-path helpers (#988): rateLimiter + nats-auth-callout to 2 replicas.
 render_chart_values ratelimiter "$work_dir/ratelimiter-on.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
@@ -144,6 +173,10 @@ grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-enforce
 grep -q "whenUnsatisfiable: DoNotSchedule" "$work_dir/api-enforced.yaml" ||
   fail "api: expected DoNotSchedule topology spread when highAvailability.mode=ha-enforced"
 
+render_chart_values cassandra "$work_dir/cassandra-enforced.yaml" "$deps" || fail "render cassandra (ha-enforced)"
+grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-enforced.yaml" ||
+  fail "cassandra: expected required Tier-2 anti-affinity when highAvailability.mode=ha-enforced"
+
 render_chart_values ratelimiter "$work_dir/ratelimiter-enforced.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
   fail "render ratelimiter (ha-enforced)"
 awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-enforced.yaml" | grep -q "requiredDuringSchedulingIgnoredDuringExecution:" ||
diff --git a/docs/v0.6.1/high-availability.md b/docs/v0.6.1/high-availability.md
new file mode 100644
index 0000000000..fbe96ffd50
--- /dev/null
+++ b/docs/v0.6.1/high-availability.md
@@ -0,0 +1,237 @@
+# High Availability
+
+This guide explains how to run the self-hosted NVCF control plane in a
+high-availability (HA) topology so that it survives the loss of a single node
+or availability zone (AZ). It covers the cluster prerequisites you must provide,
+how to turn HA on, what each service does under HA, and how to validate the
+result.
+
+HA is opt-in. Single-node installs (local, CI, small proofs of concept) should
+leave it off and are unaffected by anything in this guide.
+
+## Overview
+
+HA is controlled by a single switch in your Helmfile environment values,
+`highAvailability.mode`. The self-managed stack maps that mode onto per-chart
+values (replica counts, pod anti-affinity, zone topology spread, pod disruption
+budgets, and data-durability settings) in
+`deploy/stacks/self-managed/global.yaml.gotmpl`.
+
+| `highAvailability.mode` | Behavior |
+| --- | --- |
+| `none` (default) | Keep existing single-replica chart/env values. Nothing in this guide applies. Use for local, CI, and single-node installs. |
+| `ha-preferred` | HA sizing (multiple replicas, quorum services at 3). Node/AZ spread is a **preference**: if a second node or AZ has no capacity, pods still schedule (co-located) rather than staying `Pending`. |
+| `ha-enforced` | Same HA sizing, but node/AZ spread is **required**: a replica that cannot land on a distinct node/AZ stays `Pending` instead of packing onto an occupied one. |
+
+`ha-preferred` is the recommended starting point: it gives you the full HA
+topology while degrading gracefully on a constrained cluster. Move to
+`ha-enforced` once you have confirmed your node pools have capacity in every AZ
+and you want hard placement guarantees.
+
+## Cluster prerequisites
+
+HA depends on infrastructure the **operator** provides. The stack cannot create
+nodes or AZs for you — it only schedules against what you label.
+
+### 1. Node count
+
+Both HA modes require **at least 3 schedulable nodes** in the pool(s) that host
+control-plane and quorum workloads. The quorum services (Cassandra, NATS,
+OpenBao) run 3 replicas that must land on 3 distinct nodes.
+
+### 2. Availability-zone labels
+
+To spread replicas across failure domains, the scheduler uses the standard
+Kubernetes well-known label on your nodes:
+
+```
+topology.kubernetes.io/zone=<az>
+```
+
+You (the operator) must ensure this label is present on every node. Managed
+Kubernetes services (EKS, AKS, GKE) apply it automatically. On bare-metal or
+custom clusters, set it yourself, for example:
+
+```bash
+kubectl label node <node-name> topology.kubernetes.io/zone=az-1
+```
+
+If the label is absent, zone topology spread has nothing to spread across. In
+`ha-preferred` this silently degrades to node-level spread only; in
+`ha-enforced` zone-constrained pods can stay `Pending`. Aim for capacity in
+**at least two AZs** (three is better for the quorum services).
+
+### 3. Dedicated node pools (recommended)
+
+For predictable placement and isolation, give the stateful quorum services their
+own node pools and label them with `nvcf.nvidia.com/workload`. Configure the
+selectors under `global.nodeSelectors` in your environment file:
+
+| Pool | Selector value | Hosts |
+| --- | --- | --- |
+| `controlplane` | `control-plane` | Stateless + hot-path Deployments |
+| `cassandra` | `cassandra` | Cassandra StatefulSet |
+| `vault` | `vault` | OpenBao StatefulSet |
+
+Each dedicated pool must itself have **capacity in each AZ** — a 3-node
+Cassandra pool concentrated in one AZ cannot spread across zones no matter what
+the stack requests. If you run a single shared pool, use `global.nodeSelectors.all`
+instead and size it to hold every replica on distinct nodes/AZs.
+
+## Enabling HA
+
+Set the mode in your environment file (for example
+`deploy/stacks/self-managed/environments/<env>.yaml`):
+
+```yaml
+highAvailability:
+  mode: ha-preferred
+```
+
+That single line activates all of the defaults documented below. Every knob
+under `highAvailability` has a sensible default; override only what you need.
+Then apply the stack as usual:
+
+```bash
+helmfile -e <env> apply
+```
+
+An invalid mode fails the render fast with a clear error, so a typo cannot
+silently disable HA.
+
+## What HA changes, by tier
+
+### Stateless control-plane services
+
+Active-active Deployments with no leader election: `nvcf-api`,
+`invocation-service`, `grpc-proxy`, `admin-token-issuer-proxy`, and
+`llm-api-gateway` (when the LLM addon is enabled).
+
+Under HA each of these gets:
+
+- **2 replicas** (`highAvailability.stateless.replicaCount`).
+- **Hostname pod anti-affinity** so the two replicas never share a node.
+- **Zone topology spread** (`topology.kubernetes.io/zone`, `maxSkew: 1`) so they
+  land in different AZs when zones are labelled.
+- **A PodDisruptionBudget** (`minAvailable: 1`) so voluntary disruptions
+  (drains, upgrades) never take the last replica.
+- **A surge rolling-update strategy** (`maxSurge: 1`, `maxUnavailable: 0`) so a
+  new pod is Ready before an old one is removed.
+
+### Hot-path helper services
+
+`rateLimiter` and `nats-auth-callout` sit on the request/auth hot path, so under
+HA they are raised to **2 replicas** (`highAvailability.hotPath.replicaCount`)
+with their own PDB (`minAvailable: 1`). They reuse the stateless hostname
+anti-affinity and zone spread.
+
+### Tier-2 quorum services (data durability)
+
+`Cassandra`, `NATS`, and `OpenBao` run as **3-replica quorum StatefulSets** with:
+
+- **Hostname pod anti-affinity** so the 3 peers land on 3 distinct nodes
+  (`highAvailability.tier2.podAntiAffinity`). Following the mode, this is
+  preferred (soft) under `ha-preferred` and required (hard) under `ha-enforced`.
+  OpenBao's upstream chart ships a hard anti-affinity that the stack disables for
+  single-node installs and re-enables (soft/hard by mode) under HA.
+- **PodDisruptionBudgets** sized to preserve quorum (`minAvailable: 2`).
+
+Beyond placement, HA also raises the data-durability settings:
+
+#### NATS JetStream replica factor
+
+Streams default to a single replica. Under HA the stack sets the JetStream
+replica factor (RF) to **2** (`highAvailability.nats.jetstream.replicaFactor`)
+on the two services that create streams — `nvcf-api` (via `NVCF_NATS_REPLICAS`)
+and `invocation-service` (via `NATS_PROPERTIES__REPLICAS`). With RF=2 the
+worker/result streams are replicated across the NATS cluster and survive the
+loss of the node hosting the leader. RF must be `<= highAvailability.nats.replicas`.
+
+#### Cassandra replication and consistency
+
+The Cassandra keyspaces are created with `NetworkTopologyStrategy` and a
+replication factor equal to `highAvailability.cassandra.replicaCount` (3 under
+HA), and the control-plane services read/write at `LOCAL_QUORUM`. This is the
+correct configuration for both single-DC and multi-AZ deployments:
+
+- **Single datacenter:** RF=3 with `LOCAL_QUORUM` tolerates the loss of one
+  replica for reads and writes.
+- **Multi-AZ:** because replicas are placed with `NetworkTopologyStrategy`,
+  labelling nodes by rack/AZ makes Cassandra distribute the 3 replicas across
+  AZs automatically; `LOCAL_QUORUM` then keeps the cluster available through the
+  loss of a single AZ.
+
+No stack change is required to select the strategy — it is
+`NetworkTopologyStrategy` in all cases. To get true cross-AZ placement, ensure
+the Cassandra nodes carry AZ labels (see the prerequisites above).
+
+## Validation
+
+After applying HA, confirm replicas are spread as expected.
+
+Check that quorum peers landed on distinct nodes and AZs:
+
+```bash
+# Nodes and their AZ labels
+kubectl get nodes -L topology.kubernetes.io/zone
+
+# Cassandra / NATS / OpenBao pods with their nodes
+kubectl -n cassandra-system get pods -o wide
+kubectl -n nats-system get pods -o wide
+kubectl -n vault-system get pods -o wide
+```
+
+Confirm the stateless Deployments scaled and spread:
+
+```bash
+kubectl -n nvcf get deploy nvcf-api invocation-service -o wide
+kubectl -n nvcf get pods -o wide -l app.kubernetes.io/instance=nvcf-api
+```
+
+Confirm the JetStream RF took effect (streams report `Replicas: 2`):
+
+```bash
+kubectl -n nats-system exec -it nats-0 -- nats stream ls
+kubectl -n nats-system exec -it nats-0 -- nats stream info <stream-name>
+```
+
+Confirm Cassandra keyspace replication:
+
+```bash
+kubectl -n cassandra-system exec -it cassandra-0 -- \
+  cqlsh -e "SELECT keyspace_name, replication FROM system_schema.keyspaces;"
+```
+
+If any pod is stuck `Pending` under `ha-enforced`, it usually means a node pool
+lacks capacity in a second node/AZ. Add capacity, or drop to `ha-preferred` to
+let it schedule while you rebalance.
+
+## Recovery objectives and failure behavior
+
+With HA enabled and capacity in at least two AZs:
+
+- **Single node loss:** Stateless and hot-path Deployments keep serving from
+  their surviving replica; the scheduler recreates the lost pod on another node
+  (and the PDB prevents drains from removing the last one). Quorum services
+  (Cassandra RF=3/`LOCAL_QUORUM`, NATS RF=2, OpenBao 3-node Raft) retain quorum
+  with 2 of 3 members and continue serving reads and writes.
+- **Single AZ loss:** With replicas spread across AZs, the control plane stays
+  available on the surviving AZ(s). Recovery time is dominated by pod
+  reschedule/restart time on the healthy AZ rather than any manual failover.
+- **Two simultaneous quorum-member losses:** A 3-member quorum service loses
+  quorum and pauses writes until a member returns. This is why three AZs (or at
+  least three nodes across two AZs, with the third member able to reschedule) is
+  the durable target.
+
+HA reduces recovery to automatic rescheduling within surviving failure domains;
+it does not replace backups. Continue to back up Cassandra and OpenBao per the
+[Control Plane Operations](./control-plane-operations.md) runbooks.
+
+## Related
+
+- [Control Plane Operations](./control-plane-operations.md) — service reference,
+  key rotation, and upgrade runbooks.
+- [Infrastructure Sizing](./infrastructure-sizing.md) — node pool sizing
+  guidance.
+- [Helmfile Installation](./helmfile-installation.md) — how environment values
+  and `global.yaml.gotmpl` are applied.
diff --git a/docs/v0.6.1/index.md b/docs/v0.6.1/index.md
index 814711d7a3..cb2333c5c9 100644
--- a/docs/v0.6.1/index.md
+++ b/docs/v0.6.1/index.md
@@ -10,6 +10,8 @@ This guide provides information for deploying and operating NVCF in self-managed
   : Connect GPU clusters to the NVCF control plane.
 - [Configuration](./optional-enhancements.md)
   : Configure gateway routing, registries, and optional enhancements.
+- [High Availability](./high-availability.md)
+  : Run the control plane across nodes and availability zones to survive node/AZ loss.
 - [Using Cloud Functions](./api.md)
   : Create and invoke functions using the NVCF API and CLI.
 - [Managed (Legacy)](../ngc-managed/cluster-management/ngc-managed.md)

From 5db36684007f5fca846be226fa89cbd5b938baab Mon Sep 17 00:00:00 2001
From: shobham <shobham@nvidia.com>
Date: Wed, 9 Sep 2026 18:19:34 +0530
Subject: [PATCH 03/11] feat(self-managed): opt-in Tier-2 zone topology spread
 under HA (#989)

Spread the 3 Cassandra/NATS/OpenBao quorum peers across topology.kubernetes.io/zone via nvcf.ha.tier2TopologySpread, gated behind highAvailability.tier2.topologySpread (default off; soft by default, strict: true for >=3-AZ hard spread). Documents the WaitForFirstConsumer StorageClass, >=3-AZ, dedicated node-pool enablement, and Cassandra rack=AZ requirements. Part of epic #985.
---
 .../cassandra/helm/templates/statefulset.yaml |  3 +
 deploy/helm/cassandra/helm/values.yaml        |  5 ++
 .../self-managed/environments/base.yaml       | 19 +++++
 deploy/stacks/self-managed/global.yaml.gotmpl | 47 ++++++++++-
 .../self-managed/tests/ha-value-wiring.sh     | 44 +++++++++++
 docs/v0.6.1/high-availability.md              | 79 +++++++++++++++++--
 6 files changed, 191 insertions(+), 6 deletions(-)

diff --git a/deploy/helm/cassandra/helm/templates/statefulset.yaml b/deploy/helm/cassandra/helm/templates/statefulset.yaml
index f451842d73..975bad4258 100644
--- a/deploy/helm/cassandra/helm/templates/statefulset.yaml
+++ b/deploy/helm/cassandra/helm/templates/statefulset.yaml
@@ -41,6 +41,9 @@ spec:
       {{- with .Values.cassandra.affinity }}
       affinity: {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.cassandra.topologySpreadConstraints }}
+      topologySpreadConstraints: {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.cassandra.tolerations }}
       tolerations: {{- toYaml . | nindent 8 }}
       {{- end }}
diff --git a/deploy/helm/cassandra/helm/values.yaml b/deploy/helm/cassandra/helm/values.yaml
index 35e1a26d4a..4c331947e2 100644
--- a/deploy/helm/cassandra/helm/values.yaml
+++ b/deploy/helm/cassandra/helm/values.yaml
@@ -156,6 +156,11 @@ cassandra:
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Pod topology spread constraints. Empty by default; the self-managed Helmfile
+  # sets these (zone spread across topology.kubernetes.io/zone) via
+  # highAvailability.tier2.topologySpread when HA is on. Requires a
+  # WaitForFirstConsumer StorageClass so each zonal PV binds in the pod's zone.
+  topologySpreadConstraints: []
 
   hooks:
     initializeCluster:
diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 02da46f73e..a79a72dbe1 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -783,6 +783,25 @@ highAvailability:
   tier2:
     podAntiAffinity:
       enabled: true
+    # Zone topology spread for the 3 quorum peers across
+    # topology.kubernetes.io/zone (Cassandra, NATS, OpenBao). OPT-IN (default
+    # off) because it depends on operator-provided infrastructure:
+    #   * a StorageClass with volumeBindingMode: WaitForFirstConsumer, so each
+    #     pod's zonal PersistentVolume is created in the zone the scheduler
+    #     picks. With Immediate binding the PV zone is fixed first and pods can
+    #     land Pending when the spread constraint disagrees.
+    #   * capacity in >= 3 AZs. A 3-member quorum only survives an AZ loss if no
+    #     single AZ holds a majority; with 2 AZs one zone holds 2 of 3, so
+    #     losing it breaks quorum.
+    # whenUnsatisfiable defaults to ScheduleAnyway (soft) so a short AZ never
+    # leaves a peer Pending. Set strict: true ONLY on >= 3-AZ clusters to make
+    # it DoNotSchedule (hard). NOTE: for Cassandra, data-level AZ diversity also
+    # requires rack = AZ on the nodes (image/entrypoint side), not just pod
+    # spread; see docs/v0.6.1/high-availability.md.
+    topologySpread:
+      enabled: false
+      maxSkew: 1
+      strict: false
 
   nats:
     replicas: 3
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index b4cbaa22fe..d9a5a1c60b 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -161,6 +161,35 @@ podAntiAffinity:
 {{- end -}}
 {{- end -}}
 
+{{/* Zone topology spread for a Tier-2 quorum release (Cassandra, NATS,
+     OpenBao) so the 3 quorum peers land in distinct availability zones. Matches
+     app.kubernetes.io/instance = release name on topology.kubernetes.io/zone.
+     OPT-IN: gated by highAvailability.tier2.topologySpread.enabled (default
+     false) because correct behaviour depends on operator infrastructure — a
+     StorageClass with volumeBindingMode: WaitForFirstConsumer (so each zonal PV
+     is created in the zone the scheduler picks) and capacity in >= 3 AZs.
+     whenUnsatisfiable defaults to ScheduleAnyway (soft) so a short AZ never
+     leaves a peer Pending; set topologySpread.strict: true (>= 3-AZ clusters
+     only) for DoNotSchedule (hard). Emits only the constraint list items (no
+     topologySpreadConstraints: key) so callers can nest it as a list (Cassandra
+     value, NATS podTemplate.merge.spec) or a string (OpenBao
+     server.topologySpreadConstraints). Context: dict "Values" $.Values
+     "instance" "<release-instance>" */}}
+{{- define "nvcf.ha.tier2TopologySpread" -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- $ts := dig "highAvailability" "tier2" "topologySpread" dict .Values -}}
+{{- if dig "enabled" false $ts -}}
+- maxSkew: {{ dig "maxSkew" 1 $ts }}
+  topologyKey: topology.kubernetes.io/zone
+  whenUnsatisfiable: {{ if dig "strict" false $ts }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
+  labelSelector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .instance | quote }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }}
 {{- if not (has $haMode (list "none" "ha-preferred" "ha-enforced")) }}
 {{- fail (printf "highAvailability.mode must be none, ha-preferred, or ha-enforced, got %q" $haMode) }}
@@ -205,6 +234,10 @@ cassandra:
   affinity:
     {{- . | nindent 4 }}
   {{- end }}
+  {{- with include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "cassandra") }}
+  topologySpreadConstraints:
+    {{- . | nindent 4 }}
+  {{- end }}
   {{- end }}
   persistence:
     size: {{ .Values.global.storageSize | default "10Gi" }}
@@ -339,6 +372,12 @@ openbao:
     affinity: |
       {{- . | nindent 6 }}
     {{- end }}
+    {{- with include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "openbao-server") }}
+    # String form: the OpenBao chart tpl-renders server.topologySpreadConstraints
+    # under a topologySpreadConstraints: key. Opt-in via tier2.topologySpread.
+    topologySpreadConstraints: |
+      {{- . | nindent 6 }}
+    {{- end }}
     {{- end }}
     image:
       registry: {{ .Values.global.image.registry }}
@@ -404,10 +443,12 @@ nats:
   {{- $natsNs := include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) -}}
   {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) -}}
   {{- $natsAffinity := "" -}}
+  {{- $natsTopoSpread := "" -}}
   {{- if $haEnabled -}}
   {{- $natsAffinity = include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "nats") -}}
+  {{- $natsTopoSpread = include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "nats") -}}
   {{- end -}}
-  {{- if or $natsNs $natsTol $natsAffinity }}
+  {{- if or $natsNs $natsTol $natsAffinity $natsTopoSpread }}
   podTemplate:
     merge:
       spec:
@@ -421,6 +462,10 @@ nats:
         affinity:
           {{- . | nindent 10 }}
         {{- end }}
+        {{- with $natsTopoSpread }}
+        topologySpreadConstraints:
+          {{- . | nindent 10 }}
+        {{- end }}
   {{- end }}
 
   # One merged config: block -- a second config: key under nats: would be
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index 59ad0e642a..d5d0fe98dd 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -127,6 +127,13 @@ grep -q "podAntiAffinity:" "$work_dir/cassandra-on.yaml" ||
   fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
 grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-on.yaml" ||
   fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+# Zone topology spread is opt-in: no zone constraint in the cassandra block
+# unless tier2.topologySpread.enabled (the chart ships an empty default list).
+# NOTE: the write-values file holds every release's values, so scope to the
+# cassandra: block — stateless releases carry their own (expected) zone spread.
+if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "topology.kubernetes.io/zone"; then
+  fail "cassandra: Tier-2 zone spread leaked without tier2.topologySpread.enabled"
+fi
 
 render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (ha-preferred)"
 grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
@@ -161,6 +168,43 @@ render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-va
 awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
   fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
 
+echo "== highAvailability tier-2 zone topology spread (opt-in) =="
+write_env <<'EOF'
+highAvailability:
+  mode: ha-preferred
+  tier2:
+    topologySpread:
+      enabled: true
+EOF
+
+# Scope assertions to each release's own block (the write-values file holds all).
+render_chart_values cassandra "$work_dir/cassandra-spread.yaml" "$deps" || fail "render cassandra (zone spread)"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "cassandra: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread.yaml" | grep -q "whenUnsatisfiable: ScheduleAnyway" ||
+  fail "cassandra: expected soft (ScheduleAnyway) Tier-2 spread by default"
+
+render_chart_values openbao-server "$work_dir/openbao-spread.yaml" "$deps" || fail "render openbao (zone spread)"
+awk '/^openbao:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/openbao-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "openbao: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
+
+render_chart_values nats "$work_dir/nats-spread.yaml" "$deps" || fail "render nats (zone spread)"
+awk '/^nats:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/nats-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "nats: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
+
+# strict: true -> hard (DoNotSchedule), for >= 3-AZ clusters.
+write_env <<'EOF'
+highAvailability:
+  mode: ha-enforced
+  tier2:
+    topologySpread:
+      enabled: true
+      strict: true
+EOF
+render_chart_values cassandra "$work_dir/cassandra-spread-strict.yaml" "$deps" || fail "render cassandra (strict spread)"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread-strict.yaml" | grep -q "whenUnsatisfiable: DoNotSchedule" ||
+  fail "cassandra: expected hard (DoNotSchedule) Tier-2 spread when strict=true"
+
 echo "== highAvailability ha-enforced: required anti-affinity =="
 write_env <<'EOF'
 highAvailability:
diff --git a/docs/v0.6.1/high-availability.md b/docs/v0.6.1/high-availability.md
index fbe96ffd50..be4c57c16e 100644
--- a/docs/v0.6.1/high-availability.md
+++ b/docs/v0.6.1/high-availability.md
@@ -65,7 +65,24 @@ If the label is absent, zone topology spread has nothing to spread across. In
 
 For predictable placement and isolation, give the stateful quorum services their
 own node pools and label them with `nvcf.nvidia.com/workload`. Configure the
-selectors under `global.nodeSelectors` in your environment file:
+selectors under `global.nodeSelectors` and **set `enabled: true`** — the
+selectors ship disabled (`global.nodeSelectors.enabled: false`) and are a no-op
+until you turn them on:
+
+```yaml
+global:
+  nodeSelectors:
+    enabled: true                 # required; selectors are ignored when false
+    controlplane:
+      key: nvcf.nvidia.com/workload
+      value: control-plane
+    cassandra:
+      key: nvcf.nvidia.com/workload
+      value: cassandra
+    vault:
+      key: nvcf.nvidia.com/workload
+      value: vault
+```
 
 | Pool | Selector value | Hosts |
 | --- | --- | --- |
@@ -73,10 +90,14 @@ selectors under `global.nodeSelectors` in your environment file:
 | `cassandra` | `cassandra` | Cassandra StatefulSet |
 | `vault` | `vault` | OpenBao StatefulSet |
 
-Each dedicated pool must itself have **capacity in each AZ** — a 3-node
-Cassandra pool concentrated in one AZ cannot spread across zones no matter what
-the stack requests. If you run a single shared pool, use `global.nodeSelectors.all`
-instead and size it to hold every replica on distinct nodes/AZs.
+Each dedicated pool must span the availability zones — that is, have **capacity
+in every AZ you want to spread across** (both zones in a 2-AZ cluster, all three
+in a 3-AZ cluster). A 3-node Cassandra pool concentrated in one AZ cannot spread
+across zones no matter what the stack requests, and with `enabled: false` the
+pods fall back to default scheduling regardless of your labels. If you run a
+single shared pool instead, set `global.nodeSelectors.enabled: true` with
+`global.nodeSelectors.all` and size it to hold every replica on distinct
+nodes/AZs.
 
 ## Enabling HA
 
@@ -136,6 +157,54 @@ anti-affinity and zone spread.
   single-node installs and re-enables (soft/hard by mode) under HA.
 - **PodDisruptionBudgets** sized to preserve quorum (`minAvailable: 2`).
 
+#### Zone spread for the quorum pods (opt-in)
+
+By default the quorum peers are only guaranteed distinct **nodes**, not distinct
+**zones** — three nodes can all be in one AZ, so a single-AZ loss could still
+break quorum. To spread the three peers across `topology.kubernetes.io/zone`,
+enable:
+
+```yaml
+highAvailability:
+  tier2:
+    topologySpread:
+      enabled: true      # off by default
+      maxSkew: 1
+      strict: false      # ScheduleAnyway (soft). Set true only on >= 3 AZs.
+```
+
+This is **opt-in** because, unlike the stateless tier, it depends on
+infrastructure you must provide:
+
+- **StorageClass `volumeBindingMode: WaitForFirstConsumer`.** Each quorum pod has
+  a zonal PersistentVolume, and a zonal disk can only attach to a node in its own
+  AZ. With `WaitForFirstConsumer`, the scheduler places the pod first (honoring
+  the spread constraint) and the PV is then created in that pod's zone. With
+  `Immediate` binding the PV's zone is chosen up front and the pod is pinned to
+  it, which fights the spread constraint and can leave pods `Pending`. The stack
+  cannot set this for you — it is a property of the StorageClass you supply.
+- **Capacity in at least 3 AZs.** A 3-member quorum only survives an AZ loss if
+  no single AZ holds a majority. With only 2 AZs one zone inevitably holds 2 of
+  3 members, and losing that zone breaks quorum. On a 2-AZ cluster leave zone
+  spread off (or keep `strict: false`).
+
+`whenUnsatisfiable` defaults to `ScheduleAnyway` (soft) so a temporarily short
+AZ never leaves a peer `Pending`. Only set `strict: true` — which makes it
+`DoNotSchedule` (hard) — on clusters you know have capacity in ≥3 AZs.
+
+**Cassandra needs one more thing: rack = AZ.** Spreading the *pods* across zones
+does not by itself make the *data* zone-diverse. `NetworkTopologyStrategy`
+replicates by **rack**, and Cassandra's rack is assigned by the image entrypoint,
+not by the pod's Kubernetes zone. Unless each pod's Cassandra rack is set to its
+AZ, RF=3 can still place all three data replicas in one rack. Map rack to AZ on
+the Cassandra nodes to get true cross-AZ data placement; NATS and OpenBao (Raft)
+replicate per member and need only the pod spread.
+
+Once a quorum pod's PV is created in a zone it is pinned there for the life of
+that StatefulSet ordinal — steady-state placement stays spread, but a pod whose
+AZ is lost cannot reschedule elsewhere until the AZ returns (its two peers carry
+quorum in the meantime).
+
 Beyond placement, HA also raises the data-durability settings:
 
 #### NATS JetStream replica factor

From 6b83e5dcb2ea3766ea7c496011e688bc1f873bd6 Mon Sep 17 00:00:00 2001
From: shobham <shobham@nvidia.com>
Date: Wed, 16 Sep 2026 12:09:48 +0530
Subject: [PATCH 04/11] feat(self-managed): defer invocation-service +
 grpc-proxy HA scaling until Envoy

Keep invocation-service and grpc-proxy at a single replica under
highAvailability, pending Envoy support in the self-hosted stack. Worker
callbacks are per-pod host-bound; multi-replica scaling is deferred per the
#987/#989 review until Envoy is available.

- global.yaml.gotmpl: pin both to replicaCount 1 under HA; drop the HA
  PodDisruptionBudget for them (a minAvailable:1 PDB on a singleton blocks
  drains); keep anti-affinity/zone-spread (no-ops at 1 replica).
- base.yaml: document the deferral in the highAvailability.stateless comment.
- ha-value-wiring.sh: assert both stay single-replica with no HA PDB under
  ha-preferred.
---
 .../self-managed/environments/base.yaml       | 10 +++--
 deploy/stacks/self-managed/global.yaml.gotmpl | 41 ++++++++++---------
 .../self-managed/tests/ha-value-wiring.sh     | 25 +++++++++++
 3 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 690578b89d..a916047f1b 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -742,9 +742,13 @@ grpcproxy:
 highAvailability:
   mode: none
 
-  # Stateless control-plane Deployments: nvcf-api, invocation-service,
-  # grpc-proxy, admin-token-issuer-proxy, and llm-api-gateway (when the LLM
-  # addon is enabled). Active-active; no application leader election.
+  # Stateless control-plane Deployments: nvcf-api, admin-token-issuer-proxy,
+  # and llm-api-gateway (when the LLM addon is enabled). Active-active; no
+  # application leader election.
+  # NOTE: invocation-service and grpc-proxy are intentionally kept at a single
+  # replica for now. Their multi-replica scaling is deferred until Envoy
+  # support lands in the self-hosted stack (worker-callback host binding; see
+  # #987/#989 review). replicaCount below therefore does not apply to them.
   stateless:
     replicaCount: 2
     podAntiAffinity:
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 7f9a48cc7f..961eda18a5 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -38,9 +38,12 @@ highAvailability.mode:
 Requires ≥3 schedulable nodes unless mode is none. Nested keys under
 stateless/hotPath/nats/openbao/cassandra override those defaults.
 
-stateless — nvcf-api, invocation, grpc-proxy, admin-token-issuer-proxy, and
-  llm-api-gateway (LLM addon): replicaCount, hostname anti-affinity, zone
-  topology spread, PDB.
+stateless — nvcf-api, admin-token-issuer-proxy, and llm-api-gateway (LLM
+  addon): replicaCount, hostname anti-affinity, zone topology spread, PDB.
+  NOTE: invocation-service and grpc-proxy remain single-replica for now
+  (anti-affinity/zone-spread still applied but no-ops at 1 replica, no PDB).
+  Their multi-replica scaling is deferred until Envoy support lands in the
+  self-hosted stack (worker-callback host binding; see #987/#989 review).
 hotPath   — rateLimiter, nats-auth-callout: 2 replicas + anti-affinity +
   zone topology spread + PDB.
 
@@ -753,7 +756,11 @@ invocation:
   {{- $invocationEnv = mergeOverwrite $invocationEnv (deepCopy $configuredInvocationEnv) }}
   fullnameOverride: invocation-service
   {{- if $haEnabled }}
-  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- /* HA replica scaling for invocation-service is deferred until Envoy
+         support lands in the self-hosted stack (worker-callback host binding,
+         see #987/#989 review). Pin to a single replica for now; the
+         anti-affinity / zone-spread below are no-ops at one replica. */}}
+  replicaCount: 1
   {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
@@ -788,17 +795,13 @@ invocation:
     baggageAttributeAllowlist:
     {{- toYaml . | nindent 6 }}
   {{- end }}
-  {{- if $haEnabled }}
-  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
-  podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- else }}
+  {{- /* No HA PodDisruptionBudget while invocation-service runs a single
+         replica (deferred until Envoy). A minAvailable:1 PDB on a singleton
+         would block node drains. */}}
   {{- with dig "invocation" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
-  {{- end }}
 
 nvctApi:
   fullnameOverride: nvct-api
@@ -847,7 +850,11 @@ nvctApi:
 grpcproxy:
   fullnameOverride: grpc-proxy
   {{- if $haEnabled }}
-  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  {{- /* HA replica scaling for grpc-proxy is deferred until Envoy support
+         lands in the self-hosted stack (worker-callback host binding, see
+         #987/#989 review). Pin to a single replica for now; the anti-affinity
+         / zone-spread below are no-ops at one replica. */}}
+  replicaCount: 1
   {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
@@ -896,17 +903,13 @@ grpcproxy:
     RATE_LIMIT_ENABLED: "true"
     RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777"
     {{- end }}
-  {{- if $haEnabled }}
-  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
-  podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- else }}
+  {{- /* No HA PodDisruptionBudget while grpc-proxy runs a single replica
+         (deferred until Envoy). A minAvailable:1 PDB on a singleton would
+         block node drains. */}}
   {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }}
   podDisruptionBudget:
     {{- toYaml . | nindent 4 }}
   {{- end }}
-  {{- end }}
 
 rateLimiter:
   {{- if .Values.global.imagePullSecrets }}
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index a03cafd58e..926282cdb5 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -132,6 +132,31 @@ render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-va
 awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
   fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
 
+# invocation-service + grpc-proxy: multi-replica scaling is deferred until
+# Envoy support lands (worker-callback host binding; see #987/#989 review).
+# Even under HA they must stay single-replica and get no PDB (a minAvailable:1
+# PDB on a singleton blocks node drains). Anti-affinity/zone-spread may still
+# render but are no-ops at one replica.
+render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" ||
+  fail "render invocation-service (ha-preferred)"
+if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then
+  fail "invocation-service: must stay single-replica under HA (deferred until Envoy)"
+fi
+# The chart's own PDB knob may render (enabled: false); the HA PDB (enabled:
+# true / minAvailable on a singleton) must NOT.
+if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.yaml" | grep -A3 "podDisruptionBudget:" | grep -q "enabled: true"; then
+  fail "invocation-service: HA PDB must not be enabled while single-replica (deferred until Envoy)"
+fi
+
+render_chart_values grpc-proxy "$work_dir/grpcproxy-on.yaml" "$core" ||
+  fail "render grpc-proxy (ha-preferred)"
+if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then
+  fail "grpc-proxy: must stay single-replica under HA (deferred until Envoy)"
+fi
+if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml" | grep -A3 "podDisruptionBudget:" | grep -q "enabled: true"; then
+  fail "grpc-proxy: HA PDB must not be enabled while single-replica (deferred until Envoy)"
+fi
+
 echo "== highAvailability ha-enforced: required anti-affinity =="
 write_env <<'EOF'
 highAvailability:

From 11819dbf5d4cf03d33bba8dae418418b8cf6340c Mon Sep 17 00:00:00 2001
From: shobham <shobham@nvidia.com>
Date: Wed, 16 Sep 2026 12:20:48 +0530
Subject: [PATCH 05/11] docs(self-managed): note invocation-service +
 grpc-proxy HA deferral until Envoy

---
 docs/v0.6.1/high-availability.md | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/docs/v0.6.1/high-availability.md b/docs/v0.6.1/high-availability.md
index fbe96ffd50..24b4be9339 100644
--- a/docs/v0.6.1/high-availability.md
+++ b/docs/v0.6.1/high-availability.md
@@ -104,8 +104,18 @@ silently disable HA.
 ### Stateless control-plane services
 
 Active-active Deployments with no leader election: `nvcf-api`,
-`invocation-service`, `grpc-proxy`, `admin-token-issuer-proxy`, and
-`llm-api-gateway` (when the LLM addon is enabled).
+`admin-token-issuer-proxy`, and `llm-api-gateway` (when the LLM addon is
+enabled).
+
+> **Note — `invocation-service` and `grpc-proxy` are deferred.** These two are
+> stateless too, but their multi-replica scaling is intentionally **held at a
+> single replica for now**, pending Envoy support in the self-hosted stack.
+> Worker callbacks are host-bound to the specific pod that accepted the request
+> (per-pod pod-IP / DNS addressing), which is safe in a single cluster; the
+> Envoy dependency is for the cross-cluster case. Until then they keep hostname
+> anti-affinity and zone spread (no-ops at one replica) and get **no HA PDB**
+> (a `minAvailable: 1` PDB on a singleton would block node drains). See the
+> #987/#989 review.
 
 Under HA each of these gets:
 
@@ -184,8 +194,10 @@ kubectl -n vault-system get pods -o wide
 Confirm the stateless Deployments scaled and spread:
 
 ```bash
-kubectl -n nvcf get deploy nvcf-api invocation-service -o wide
+kubectl -n nvcf get deploy nvcf-api admin-token-issuer-proxy -o wide
 kubectl -n nvcf get pods -o wide -l app.kubernetes.io/instance=nvcf-api
+# invocation-service and grpc-proxy stay at 1 replica for now (deferred until Envoy)
+kubectl -n nvcf get deploy invocation-service grpc-proxy -o wide
 ```
 
 Confirm the JetStream RF took effect (streams report `Replicas: 2`):
@@ -210,9 +222,12 @@ let it schedule while you rebalance.
 
 With HA enabled and capacity in at least two AZs:
 
-- **Single node loss:** Stateless and hot-path Deployments keep serving from
-  their surviving replica; the scheduler recreates the lost pod on another node
-  (and the PDB prevents drains from removing the last one). Quorum services
+- **Single node loss:** Multi-replica stateless and hot-path Deployments keep
+  serving from their surviving replica; the scheduler recreates the lost pod on
+  another node (and the PDB prevents drains from removing the last one).
+  `invocation-service` and `grpc-proxy` (single replica until Envoy) are briefly
+  unavailable while the scheduler restarts the pod on another node. Quorum
+  services
   (Cassandra RF=3/`LOCAL_QUORUM`, NATS RF=2, OpenBao 3-node Raft) retain quorum
   with 2 of 3 members and continue serving reads and writes.
 - **Single AZ loss:** With replicas spread across AZs, the control plane stays

From 44c95962d7850c2b9ae2aadc2950144c14313e91 Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 15:54:06 +0530
Subject: [PATCH 06/11] refactor(self-managed): rename highAvailability.mode
 enum to none|preferred|enforced

Drops the "ha-" prefix on the mode values per the HA values design
review (docs: NVCF Self-Hosted Control-Plane HA Values Rework). The
enum was already a single highAvailability.mode field with no separate
enabled toggle; this is a pure rename, no behavior change.
---
 .../self-managed/environments/base.yaml       | 10 +--
 deploy/stacks/self-managed/global.yaml.gotmpl | 30 +++----
 .../self-managed/tests/ha-value-wiring.sh     | 86 +++++++++----------
 3 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index 3f04907076..c98bedc46c 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -729,10 +729,10 @@ grpcproxy:
 #
 # mode: none         (default) — keep existing env/chart replica and PDB values
 #                                (local / CI / BDD).
-#       ha-preferred           — HA sizing below; hostname anti-affinity is
+#       preferred           — HA sizing below; hostname anti-affinity is
 #                                preferred (still schedule if a second node/AZ
 #                                has no capacity). Requires ≥3 schedulable nodes.
-#       ha-enforced            — same HA sizing; hostname anti-affinity is
+#       enforced            — same HA sizing; hostname anti-affinity is
 #                                required (second replica stays Pending rather
 #                                than packing). Requires ≥3 schedulable nodes.
 #
@@ -755,7 +755,7 @@ highAvailability:
       enabled: true
     # Spread replicas across availability zones. Nodes MUST be labelled
     # topology.kubernetes.io/zone=<az>. whenUnsatisfiable follows mode:
-    # ha-preferred → ScheduleAnyway, ha-enforced → DoNotSchedule.
+    # preferred → ScheduleAnyway, enforced → DoNotSchedule.
     topologySpread:
       enabled: true
       maxSkew: 1
@@ -779,8 +779,8 @@ highAvailability:
   # Tier-2 quorum services (Cassandra, NATS, OpenBao). Under HA their peers get
   # hostname pod anti-affinity (matching app.kubernetes.io/instance per release)
   # so the 3 quorum members land on distinct nodes. whenUnsatisfiable follows
-  # mode: ha-preferred → preferred/soft (stays schedulable on small pools),
-  # ha-enforced → required/hard. The upstream OpenBao chart ships a hard
+  # mode: preferred → preferred/soft (stays schedulable on small pools),
+  # enforced → required/hard. The upstream OpenBao chart ships a hard
   # anti-affinity by default that the stack disables for single-node installs;
   # this re-enables it (soft/hard by mode) under HA. Set enabled: false to keep
   # the chart defaults.
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 08f79c76b3..ed9d741ef1 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -32,8 +32,8 @@ affinity, PDB, update strategy).
 
 highAvailability.mode:
   none          — keep existing env/chart defaults (local / CI / BDD).
-  ha-preferred  — HA sizing; preferred hostname anti-affinity.
-  ha-enforced   — HA sizing; required hostname anti-affinity.
+  preferred  — HA sizing; preferred hostname anti-affinity.
+  enforced   — HA sizing; required hostname anti-affinity.
 
 Requires ≥3 schedulable nodes unless mode is none. Nested keys under
 stateless/hotPath/nats/openbao/cassandra override those defaults.
@@ -74,12 +74,12 @@ In-scope charts MUST expose the required value hooks.
      Context: dict "Values" $.Values "instance" "<release-instance>" */}}
 {{- define "nvcf.ha.statelessAffinity" -}}
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
 {{- $paa := dig "highAvailability" "stateless" "podAntiAffinity" dict .Values -}}
 {{- if dig "enabled" true $paa -}}
 affinity:
   podAntiAffinity:
-{{- if eq $haMode "ha-enforced" }}
+{{- if eq $haMode "enforced" }}
     requiredDuringSchedulingIgnoredDuringExecution:
       - labelSelector:
           matchExpressions:
@@ -106,18 +106,18 @@ affinity:
 
 {{/* Zone-level topology spread for a Helm release instance name. Spreads
      replicas across topology.kubernetes.io/zone. whenUnsatisfiable follows the
-     mode: ha-preferred → ScheduleAnyway (best effort), ha-enforced →
+     mode: preferred → ScheduleAnyway (best effort), enforced →
      DoNotSchedule (hard). Nodes MUST carry topology.kubernetes.io/zone.
      Context: dict "Values" $.Values "instance" "<release-instance>" */}}
 {{- define "nvcf.ha.statelessTopologySpread" -}}
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
 {{- $ts := dig "highAvailability" "stateless" "topologySpread" dict .Values -}}
 {{- if dig "enabled" true $ts -}}
 topologySpreadConstraints:
   - maxSkew: {{ dig "maxSkew" 1 $ts }}
     topologyKey: topology.kubernetes.io/zone
-    whenUnsatisfiable: {{ if eq $haMode "ha-enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
+    whenUnsatisfiable: {{ if eq $haMode "enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
     labelSelector:
       matchLabels:
         app.kubernetes.io/instance: {{ .instance | quote }}
@@ -127,19 +127,19 @@ topologySpreadConstraints:
 
 {{/* Hostname pod anti-affinity for a Tier-2 quorum release (Cassandra, NATS,
      OpenBao) so the 3 quorum peers land on distinct nodes. Matches
-     app.kubernetes.io/instance = release name. ha-preferred → preferred (soft,
-     stays schedulable on small pools), ha-enforced → required (hard). Emits the
+     app.kubernetes.io/instance = release name. preferred → preferred (soft,
+     stays schedulable on small pools), enforced → required (hard). Emits the
      podAntiAffinity body only (no affinity: wrapper) so callers can nest it as a
      map (Cassandra affinity, NATS podTemplate.merge.spec.affinity) or a string
      (OpenBao server.affinity). Gated by highAvailability.tier2.podAntiAffinity.
      Context: dict "Values" $.Values "instance" "<release-instance>" */}}
 {{- define "nvcf.ha.tier2Affinity" -}}
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
 {{- $paa := dig "highAvailability" "tier2" "podAntiAffinity" dict .Values -}}
 {{- if dig "enabled" true $paa -}}
 podAntiAffinity:
-{{- if eq $haMode "ha-enforced" }}
+{{- if eq $haMode "enforced" }}
   requiredDuringSchedulingIgnoredDuringExecution:
     - labelSelector:
         matchExpressions:
@@ -180,7 +180,7 @@ podAntiAffinity:
      "instance" "<release-instance>" */}}
 {{- define "nvcf.ha.tier2TopologySpread" -}}
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") -}}
+{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
 {{- $ts := dig "highAvailability" "tier2" "topologySpread" dict .Values -}}
 {{- if dig "enabled" false $ts -}}
 - maxSkew: {{ dig "maxSkew" 1 $ts }}
@@ -194,10 +194,10 @@ podAntiAffinity:
 {{- end -}}
 
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }}
-{{- if not (has $haMode (list "none" "ha-preferred" "ha-enforced")) }}
-{{- fail (printf "highAvailability.mode must be none, ha-preferred, or ha-enforced, got %q" $haMode) }}
+{{- if not (has $haMode (list "none" "preferred" "enforced")) }}
+{{- fail (printf "highAvailability.mode must be none, preferred, or enforced, got %q" $haMode) }}
 {{- end }}
-{{- $haEnabled := or (eq $haMode "ha-preferred") (eq $haMode "ha-enforced") }}
+{{- $haEnabled := or (eq $haMode "preferred") (eq $haMode "enforced") }}
 
 cassandra:
   global:
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index 59130e83ca..2a7a3065f8 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -92,41 +92,41 @@ if grep -q "NATS_PROPERTIES__REPLICAS:" "$work_dir/invocation-off.yaml"; then
   fail "invocation: JetStream RF env leaked while highAvailability.mode=none"
 fi
 
-echo "== highAvailability ha-preferred: stateless / quorum sizing =="
+echo "== highAvailability preferred: stateless / quorum sizing =="
 write_env <<'EOF'
 highAvailability:
-  mode: ha-preferred
+  mode: preferred
 EOF
 
-render_chart_values api "$work_dir/api-on.yaml" "$core" || fail "render api (ha-preferred)"
+render_chart_values api "$work_dir/api-on.yaml" "$core" || fail "render api (preferred)"
 grep -E "replicaCount:[[:space:]]*2" "$work_dir/api-on.yaml" >/dev/null ||
-  fail "api: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+  fail "api: expected replicaCount 2 when highAvailability.mode=preferred"
 grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-on.yaml" ||
-  fail "api: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+  fail "api: expected preferred anti-affinity when highAvailability.mode=preferred"
 grep -q "topologySpreadConstraints:" "$work_dir/api-on.yaml" ||
-  fail "api: expected topologySpreadConstraints when highAvailability.mode=ha-preferred"
+  fail "api: expected topologySpreadConstraints when highAvailability.mode=preferred"
 grep -q "topology.kubernetes.io/zone" "$work_dir/api-on.yaml" ||
-  fail "api: expected zone topologyKey when highAvailability.mode=ha-preferred"
+  fail "api: expected zone topologyKey when highAvailability.mode=preferred"
 grep -q "whenUnsatisfiable: ScheduleAnyway" "$work_dir/api-on.yaml" ||
-  fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=ha-preferred"
+  fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=preferred"
 awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-on.yaml" | grep -q "podDisruptionBudget:" ||
-  fail "api: expected podDisruptionBudget when highAvailability.mode=ha-preferred"
+  fail "api: expected podDisruptionBudget when highAvailability.mode=preferred"
 grep -q 'NVCF_NATS_REPLICAS: "2"' "$work_dir/api-on.yaml" ||
-  fail "api: expected JetStream RF NVCF_NATS_REPLICAS=2 when highAvailability.mode=ha-preferred"
+  fail "api: expected JetStream RF NVCF_NATS_REPLICAS=2 when highAvailability.mode=preferred"
 
-render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || fail "render invocation (ha-preferred)"
+render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || fail "render invocation (preferred)"
 grep -q 'NATS_PROPERTIES__REPLICAS: "2"' "$work_dir/invocation-on.yaml" ||
-  fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=2 when highAvailability.mode=ha-preferred"
+  fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=2 when highAvailability.mode=preferred"
 
-render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (ha-preferred)"
+render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (preferred)"
 grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null ||
-  fail "cassandra: expected replicaCount 3 when highAvailability.mode=ha-preferred"
+  fail "cassandra: expected replicaCount 3 when highAvailability.mode=preferred"
 grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-on.yaml" | grep -q "enabled: true" ||
   fail "cassandra: expected HA PDB enabled"
 grep -q "podAntiAffinity:" "$work_dir/cassandra-on.yaml" ||
-  fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+  fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
 grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-on.yaml" ||
-  fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+  fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=preferred"
 # Zone topology spread is opt-in: no zone constraint in the cassandra block
 # unless tier2.topologySpread.enabled (the chart ships an empty default list).
 # NOTE: the write-values file holds every release's values, so scope to the
@@ -135,38 +135,38 @@ if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml
   fail "cassandra: Tier-2 zone spread leaked without tier2.topologySpread.enabled"
 fi
 
-render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (ha-preferred)"
+render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (preferred)"
 grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
-  fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=ha-preferred"
+  fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=preferred"
 grep -q "podAntiAffinity:" "$work_dir/openbao-on.yaml" ||
-  fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+  fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
 
-render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (ha-preferred)"
+render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (preferred)"
 grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
-  fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=ha-preferred"
+  fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=preferred"
 grep -q "podAntiAffinity:" "$work_dir/nats-on.yaml" ||
-  fail "nats: expected Tier-2 anti-affinity when highAvailability.mode=ha-preferred"
+  fail "nats: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
 
 # Hot-path helpers (#988): rateLimiter + nats-auth-callout to 2 replicas.
 render_chart_values ratelimiter "$work_dir/ratelimiter-on.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
-  fail "render ratelimiter (ha-preferred)"
+  fail "render ratelimiter (preferred)"
 awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null ||
-  fail "ratelimiter: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+  fail "ratelimiter: expected replicaCount 2 when highAvailability.mode=preferred"
 awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
-  fail "ratelimiter: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+  fail "ratelimiter: expected preferred anti-affinity when highAvailability.mode=preferred"
 awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-on.yaml" | grep -q "topology.kubernetes.io/zone" ||
-  fail "ratelimiter: expected zone topology spread when highAvailability.mode=ha-preferred"
+  fail "ratelimiter: expected zone topology spread when highAvailability.mode=preferred"
 
 render_chart_values nats-auth-callout-service "$work_dir/natsauth-on.yaml" "$core" ||
-  fail "render nats-auth-callout (ha-preferred)"
+  fail "render nats-auth-callout (preferred)"
 awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null ||
-  fail "nats-auth-callout: expected replicaCount 2 when highAvailability.mode=ha-preferred"
+  fail "nats-auth-callout: expected replicaCount 2 when highAvailability.mode=preferred"
 
 # llm-api-gateway (#987): stateless anti-affinity when the LLM addon is on.
 render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-values-set addons.llm.enabled=true ||
-  fail "render llm-api-gateway (ha-preferred)"
+  fail "render llm-api-gateway (preferred)"
 awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" | grep -q "preferredDuringSchedulingIgnoredDuringExecution:" ||
-  fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=ha-preferred"
+  fail "llm-api-gateway: expected preferred anti-affinity when highAvailability.mode=preferred"
 
 # invocation-service + grpc-proxy: multi-replica scaling is deferred until
 # Envoy support lands (worker-callback host binding; see #987/#989 review).
@@ -174,7 +174,7 @@ awk '/^llmApiGateway:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/llmgw-on.yaml" |
 # PDB on a singleton blocks node drains). Anti-affinity/zone-spread may still
 # render but are no-ops at one replica.
 render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" ||
-  fail "render invocation-service (ha-preferred)"
+  fail "render invocation-service (preferred)"
 if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then
   fail "invocation-service: must stay single-replica under HA (deferred until Envoy)"
 fi
@@ -185,7 +185,7 @@ if awk '/^invocation:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/invocation-on.ya
 fi
 
 render_chart_values grpc-proxy "$work_dir/grpcproxy-on.yaml" "$core" ||
-  fail "render grpc-proxy (ha-preferred)"
+  fail "render grpc-proxy (preferred)"
 if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml" | grep -qE "replicaCount:[[:space:]]*[2-9]"; then
   fail "grpc-proxy: must stay single-replica under HA (deferred until Envoy)"
 fi
@@ -196,7 +196,7 @@ fi
 echo "== highAvailability tier-2 zone topology spread (opt-in) =="
 write_env <<'EOF'
 highAvailability:
-  mode: ha-preferred
+  mode: preferred
   tier2:
     topologySpread:
       enabled: true
@@ -220,7 +220,7 @@ awk '/^nats:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/nats-spread.yaml" | grep
 # strict: true -> hard (DoNotSchedule), for >= 3-AZ clusters.
 write_env <<'EOF'
 highAvailability:
-  mode: ha-enforced
+  mode: enforced
   tier2:
     topologySpread:
       enabled: true
@@ -230,26 +230,26 @@ render_chart_values cassandra "$work_dir/cassandra-spread-strict.yaml" "$deps" |
 awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread-strict.yaml" | grep -q "whenUnsatisfiable: DoNotSchedule" ||
   fail "cassandra: expected hard (DoNotSchedule) Tier-2 spread when strict=true"
 
-echo "== highAvailability ha-enforced: required anti-affinity =="
+echo "== highAvailability enforced: required anti-affinity =="
 write_env <<'EOF'
 highAvailability:
-  mode: ha-enforced
+  mode: enforced
 EOF
 
-render_chart_values api "$work_dir/api-enforced.yaml" "$core" || fail "render api (ha-enforced)"
+render_chart_values api "$work_dir/api-enforced.yaml" "$core" || fail "render api (enforced)"
 grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/api-enforced.yaml" ||
-  fail "api: expected required anti-affinity when highAvailability.mode=ha-enforced"
+  fail "api: expected required anti-affinity when highAvailability.mode=enforced"
 grep -q "whenUnsatisfiable: DoNotSchedule" "$work_dir/api-enforced.yaml" ||
-  fail "api: expected DoNotSchedule topology spread when highAvailability.mode=ha-enforced"
+  fail "api: expected DoNotSchedule topology spread when highAvailability.mode=enforced"
 
-render_chart_values cassandra "$work_dir/cassandra-enforced.yaml" "$deps" || fail "render cassandra (ha-enforced)"
+render_chart_values cassandra "$work_dir/cassandra-enforced.yaml" "$deps" || fail "render cassandra (enforced)"
 grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-enforced.yaml" ||
-  fail "cassandra: expected required Tier-2 anti-affinity when highAvailability.mode=ha-enforced"
+  fail "cassandra: expected required Tier-2 anti-affinity when highAvailability.mode=enforced"
 
 render_chart_values ratelimiter "$work_dir/ratelimiter-enforced.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
-  fail "render ratelimiter (ha-enforced)"
+  fail "render ratelimiter (enforced)"
 awk '/^rateLimiter:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/ratelimiter-enforced.yaml" | grep -q "requiredDuringSchedulingIgnoredDuringExecution:" ||
-  fail "ratelimiter: expected required anti-affinity when highAvailability.mode=ha-enforced"
+  fail "ratelimiter: expected required anti-affinity when highAvailability.mode=enforced"
 
 echo "== highAvailability invalid mode fails render =="
 write_env <<'EOF'

From c41b0d505e5434f898668e63d307e938ab05d8b4 Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 15:55:30 +0530
Subject: [PATCH 07/11] fix(self-managed): bump NATS JetStream RF default from
 2 to 3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

JetStream streams use Raft quorum. RF=2 provides no failure tolerance
(loses quorum the moment either replica is unavailable) and is no
better than RF=1 for HA purposes. RF=3 tolerates one replica loss,
matching the 3-member NATS cluster. Flagged as a 🟠 Major finding on
PR #1679 (both a code default and a false "survives replica loss"
documentation claim.
EOF
)
---
 deploy/stacks/self-managed/environments/base.yaml   | 6 ++++--
 deploy/stacks/self-managed/global.yaml.gotmpl       | 7 +++++--
 deploy/stacks/self-managed/tests/ha-value-wiring.sh | 8 ++++----
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index c98bedc46c..b7a0efda5d 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -812,9 +812,11 @@ highAvailability:
     # JetStream stream replica factor (RF). Applied under HA to the stream
     # creators — nvcf-api and invocation-service — so the worker/result streams
     # they declare are replicated across the NATS cluster (survives a node
-    # loss). RF must be <= nats.replicas. Default 2 per the SDD.
+    # loss). RF must be <= nats.replicas. Default 3: JetStream streams use
+    # Raft quorum, so RF=2 loses quorum on a single replica failure and
+    # provides no HA benefit over RF=1.
     jetstream:
-      replicaFactor: 2
+      replicaFactor: 3
     podDisruptionBudget:
       enabled: true
       merge:
diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index ed9d741ef1..6e9d9c2570 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -65,9 +65,12 @@ In-scope charts MUST expose the required value hooks.
 {{/* JetStream stream replica factor (RF) under HA. Applied to the stream
      creators — nvcf-api (Java, nvcf.nats.replicas) and invocation (Rust,
      nats_properties.replicas) — via their env so the streams they declare are
-     replicated across the NATS cluster. Defaults to 2 per the SDD. */}}
+     replicated across the NATS cluster. Defaults to 3: JetStream streams use
+     Raft quorum, and RF=2 provides no failure tolerance (loses quorum on a
+     single replica loss) — see the NATS JetStream replication guidance and
+     the HA values design review. */}}
 {{- define "nvcf.ha.natsStreamReplicas" -}}
-{{- dig "highAvailability" "nats" "jetstream" "replicaFactor" 2 . -}}
+{{- dig "highAvailability" "nats" "jetstream" "replicaFactor" 3 . -}}
 {{- end -}}
 
 {{/* Soft/hard pod anti-affinity on hostname for a Helm release instance name.
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index 2a7a3065f8..b39c00064e 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -111,12 +111,12 @@ grep -q "whenUnsatisfiable: ScheduleAnyway" "$work_dir/api-on.yaml" ||
   fail "api: expected ScheduleAnyway topology spread when highAvailability.mode=preferred"
 awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-on.yaml" | grep -q "podDisruptionBudget:" ||
   fail "api: expected podDisruptionBudget when highAvailability.mode=preferred"
-grep -q 'NVCF_NATS_REPLICAS: "2"' "$work_dir/api-on.yaml" ||
-  fail "api: expected JetStream RF NVCF_NATS_REPLICAS=2 when highAvailability.mode=preferred"
+grep -q 'NVCF_NATS_REPLICAS: "3"' "$work_dir/api-on.yaml" ||
+  fail "api: expected JetStream RF NVCF_NATS_REPLICAS=3 when highAvailability.mode=preferred"
 
 render_chart_values invocation-service "$work_dir/invocation-on.yaml" "$core" || fail "render invocation (preferred)"
-grep -q 'NATS_PROPERTIES__REPLICAS: "2"' "$work_dir/invocation-on.yaml" ||
-  fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=2 when highAvailability.mode=preferred"
+grep -q 'NATS_PROPERTIES__REPLICAS: "3"' "$work_dir/invocation-on.yaml" ||
+  fail "invocation: expected JetStream RF NATS_PROPERTIES__REPLICAS=3 when highAvailability.mode=preferred"
 
 render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (preferred)"
 grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null ||

From a589011197c9bb0ec8d0baf969085a3b1072eb5c Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 15:57:49 +0530
Subject: [PATCH 08/11] fix(self-managed): default OpenBao disruption budget
 under HA; fix admin-proxy selector

- OpenBao: HA mode rendered 3 Raft replicas but never set a
  disruptionBudget unless the operator explicitly configured
  openbao.server.ha.disruptionBudget, leaving the quorum with no
  voluntary-disruption protection. Default to
  {enabled: true, maxUnavailable: 1} when HA is on, so a drain/upgrade
  can never take more than 1 of 3 peers down at once. Existing
  operator overrides still win (dig still checks the override key
  first).

- admin-issuer-proxy: the Tier-1 affinity/topology-spread selector
  helper was matching app.kubernetes.io/instance against the charts
  fullnameOverride (admin-token-issuer-proxy) instead of the Helmfile
  release name (admin-issuer-proxy, see helmfile.d/02-core.yaml.gotmpl).
  Helm sets app.kubernetes.io/instance from .Release.Name, so the
  generated anti-affinity/topology-spread selectors never matched any
  pod and were silent no-ops.
---
 deploy/stacks/self-managed/global.yaml.gotmpl      | 14 +++++++++++---
 .../stacks/self-managed/tests/ha-value-wiring.sh   |  2 ++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 6e9d9c2570..769aedd460 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -406,7 +406,15 @@ openbao:
       enabled: {{ dig "highAvailability" "openbao" "ha" "enabled" true .Values }}
       replicas: {{ dig "highAvailability" "openbao" "ha" "replicas" 3 .Values }}
       {{- end }}
-      {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }}
+      {{- $openbaoDisruptionBudget := dict }}
+      {{- if $haEnabled }}
+      {{/* Default: allow at most 1 of the 3 Raft peers to be voluntarily
+           disrupted at a time, so a rolling drain/upgrade can never take the
+           quorum below 2 of 3. Without this, HA renders 3 replicas but the
+           Raft cluster has no disruption protection. */}}
+      {{- $openbaoDisruptionBudget = dict "enabled" true "maxUnavailable" 1 }}
+      {{- end }}
+      {{- with dig "openbao" "server" "ha" "disruptionBudget" $openbaoDisruptionBudget .Values }}
       disruptionBudget:
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -1186,10 +1194,10 @@ adminIssuerProxy:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "admin-token-issuer-proxy") }}
+  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "admin-issuer-proxy") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "admin-token-issuer-proxy") }}
+  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "admin-issuer-proxy") }}
   {{- . | nindent 2 }}
   {{- end }}
   gateway:
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index b39c00064e..e3e3c0560d 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -140,6 +140,8 @@ grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:s
   fail "openbao: expected server.ha.replicas 3 when highAvailability.mode=preferred"
 grep -q "podAntiAffinity:" "$work_dir/openbao-on.yaml" ||
   fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
+grep -A3 "disruptionBudget:" "$work_dir/openbao-on.yaml" | grep -q "maxUnavailable: 1" ||
+  fail "openbao: expected server.ha.disruptionBudget maxUnavailable=1 when highAvailability.mode=preferred"
 
 render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (preferred)"
 grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||

From 0ab50c51d58f57bc6cbec279715c898c4971549f Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 16:29:22 +0530
Subject: [PATCH 09/11] refactor(self-managed): unify HA
 affinity/topology-spread, remove hotPath/tier2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Per the HA values design review: there is no longer a per-component
highAvailability subtree (stateless/hotPath/tier2). Sizing and
placement are derived uniformly from highAvailability.mode for every
in-scope release, via two new generic helpers:

- nvcf.ha.podAntiAffinity / nvcf.ha.affinity
- nvcf.ha.topologySpreadItems / nvcf.ha.topologySpreadConstraints

Both resolve global.affinity.<class> -> global.affinity.all -> the
mode-derived convention (same class -> all -> convention fallback as
the existing global.nodeSelectors/global.tolerations). An explicit {}
(affinity) or [] (topology spread) for a class suppresses the
generated policy for that class; presence is the signal, not
truthiness. Workload classes reuse the existing nodeSelector taxonomy
(controlplane/cassandra/vault) rather than inventing a second one.

Behavior changes from this unification:

- Tier-2 (Cassandra/NATS/OpenBao) zone topology spread is no longer a
  separate opt-in toggle (highAvailability.tier2.topologySpread.enabled,
  default off) — it now follows the same mode convention as every other
  workload: soft (ScheduleAnyway) under preferred, hard (DoNotSchedule)
  under enforced. Operators without >=3-AZ capacity use an explicit
  global.topologySpreadConstraints override ({} or a custom list) to
  opt back out, same as any other escape hatch.
- rateLimiter and natsAuthCalloutService (formerly "hotPath") now use
  the exact same replicaCount/affinity/topology-spread/PDB derivation
  as api/invocation/grpcproxy/adminIssuerProxy/llmApiGateway
  ("replica-safe Deployments") — no separate hotPath concept.
- Fixes two review-flagged gaps while unifying: natsAuthCalloutService
  never had a PodDisruptionBudget wired at all (now gets the same
  minAvailable:1 as every other replica-safe Deployment), and
  llmApiGateway's replicaCount ignored highAvailability.mode entirely
  (always read addons.llm.gateway.replicaCount, default 3; now follows
  the same mode-derived convention as the other Tier-1 releases).
- Quorum replica-count escape hatches (highAvailability.cassandra.*,
  highAvailability.nats.*, highAvailability.openbao.*) are removed per
  the design review's explicit removal list; the fixed sizes (3 for
  quorum members, 2 for the OpenBao injector) are now inlined in the
  template. The plain component-shaped values (cassandra.replicaCount,
  etc.) remain as the mode:none-only escape hatch, unchanged.
- highAvailability.nats.jetstream.replicaFactor is intentionally KEPT
  as a configurable value (not fully removed per the design review),
  since it was already fixed to a correct default (3) in a prior
  commit and removing operator configurability entirely is a larger,
  separable behavior change.

Test coverage added: global.affinity/global.topologySpreadConstraints
class override, "all" fallback, and explicit {}/[] suppression: the
existing "Tier-2 zone spread is opt-in" test is replaced with
assertions that it's on by default (soft under preferred, hard under
enforced) for Cassandra/NATS/OpenBao, matching the new behavior.
---
 deploy/stacks/self-managed/global.yaml.gotmpl | 381 ++++++++++--------
 .../self-managed/tests/ha-value-wiring.sh     | 107 +++--
 2 files changed, 283 insertions(+), 205 deletions(-)

diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl
index 769aedd460..5094b4aad2 100644
--- a/deploy/stacks/self-managed/global.yaml.gotmpl
+++ b/deploy/stacks/self-managed/global.yaml.gotmpl
@@ -27,39 +27,84 @@ tolerations:
 {{/*
 highAvailability helpers.
 
-global.yaml.gotmpl maps highAvailability.* onto chart values (replicaCount,
-affinity, PDB, update strategy).
+global.yaml.gotmpl maps highAvailability.mode onto chart values (replicaCount,
+affinity, topology spread, PDB, update strategy). There is no per-component HA
+subtree (stateless/hotPath/tier2/nats/openbao/cassandra) beneath
+highAvailability: sizing and placement are derived uniformly from mode for
+every in-scope release, per the HA values design review.
 
 highAvailability.mode:
-  none          — keep existing env/chart defaults (local / CI / BDD).
-  preferred  — HA sizing; preferred hostname anti-affinity.
-  enforced   — HA sizing; required hostname anti-affinity.
-
-Requires ≥3 schedulable nodes unless mode is none. Nested keys under
-stateless/hotPath/nats/openbao/cassandra override those defaults.
-
-stateless — nvcf-api, admin-token-issuer-proxy, and llm-api-gateway (LLM
-  addon): replicaCount, hostname anti-affinity, zone topology spread, PDB.
+  none       — keep existing env/chart defaults (local / CI / BDD).
+  preferred  — HA sizing; soft (preferred/ScheduleAnyway) hostname
+               anti-affinity and zone topology spread.
+  enforced   — HA sizing; hard (required/DoNotSchedule) hostname
+               anti-affinity and zone topology spread.
+
+Requires >= 3 schedulable nodes unless mode is none.
+
+Shared scheduling tuning: global.affinity and global.topologySpreadConstraints
+resolve class-specific override -> "all" -> the mode-derived convention above,
+the same fallback pattern as the existing global.nodeSelectors /
+global.tolerations. An explicit {} (affinity) or [] (topology spread) for a
+class — or for "all" — suppresses the generated policy for that class;
+presence is the signal, so this is NOT equivalent to omitting the key.
+Component-shaped values in the selected environment remain the final
+per-release escape hatch and are applied by each release block below, not by
+these helpers.
+
+Replica-safe Deployments (api, invocation, grpcproxy, adminIssuerProxy,
+rateLimiter, natsAuthCalloutService, llmApiGateway): 2 replicas under HA (see
+NOTE below for invocation/grpcproxy), hostname anti-affinity, zone topology
+spread, PDB minAvailable 1.
   NOTE: invocation-service and grpc-proxy remain single-replica for now
   (anti-affinity/zone-spread still applied but no-ops at 1 replica, no PDB).
   Their multi-replica scaling is deferred until Envoy support lands in the
   self-hosted stack (worker-callback host binding; see #987/#989 review).
-hotPath   — rateLimiter, nats-auth-callout: 2 replicas + anti-affinity +
-  zone topology spread + PDB.
-tier2     — Cassandra, NATS, OpenBao: quorum sizing (3) + PDB + hostname pod
-  anti-affinity so the 3 peers land on distinct nodes. NATS JetStream RF is
-  applied to the stream creators (nvcf-api, invocation) via their env.
+
+Quorum services (Cassandra, NATS, OpenBao): 3 replicas, hostname
+anti-affinity and zone topology spread so peers land on distinct
+nodes/zones, PDB sized to tolerate exactly one voluntary disruption. NATS
+JetStream RF is applied to the stream creators (nvcf-api, invocation) via
+their env.
 
 In-scope charts MUST expose the required value hooks.
 */}}
-{{- define "nvcf.ha.statelessReplicaCount" -}}
-{{- dig "highAvailability" "stateless" "replicaCount" 2 . -}}
+{{- define "nvcf.ha.replicaSafeReplicaCount" -}}
+{{- if ne (dig "highAvailability" "mode" "none" . | toString) "none" -}}2{{- end -}}
 {{- end -}}
 
-{{/* Replica count for hot-path helper Deployments (rateLimiter,
-     nats-auth-callout) under HA. Defaults to 2. */}}
-{{- define "nvcf.ha.hotPathReplicaCount" -}}
-{{- dig "highAvailability" "hotPath" "replicaCount" 2 . -}}
+{{/* Internal: minAvailable-shaped PodDisruptionBudget body, empty under mode
+     none. Context: dict "haMode" "<mode>" "minAvailable" <int> */}}
+{{- define "nvcf.ha._podDisruptionBudget" -}}
+{{- if ne .haMode "none" -}}
+enabled: true
+minAvailable: {{ .minAvailable }}
+{{- end -}}
+{{- end -}}
+
+{{/* PodDisruptionBudget for a replica-safe Tier-1 Deployment (2 replicas
+     under HA): minAvailable 1, so a voluntary disruption can never remove
+     both. Context: .Values (called as nvcf.ha.replicaSafeReplicaCount is) */}}
+{{- define "nvcf.ha.replicaSafePodDisruptionBudget" -}}
+{{- include "nvcf.ha._podDisruptionBudget" (dict "haMode" (dig "highAvailability" "mode" "none" . | toString) "minAvailable" 1) -}}
+{{- end -}}
+
+{{/* PodDisruptionBudget for a 3-member quorum service (Cassandra): tolerates
+     exactly one voluntary disruption (minAvailable 2 of 3). Context: .Values */}}
+{{- define "nvcf.ha.quorumPodDisruptionBudget" -}}
+{{- include "nvcf.ha._podDisruptionBudget" (dict "haMode" (dig "highAvailability" "mode" "none" . | toString) "minAvailable" 2) -}}
+{{- end -}}
+
+{{/* PodDisruptionBudget for the NATS quorum, in the shape the NATS chart's
+     podDisruptionBudget.merge.spec values expect. Tolerates exactly one
+     voluntary disruption (minAvailable 2 of 3). Context: .Values */}}
+{{- define "nvcf.ha.natsQuorumPodDisruptionBudget" -}}
+{{- if ne (dig "highAvailability" "mode" "none" . | toString) "none" -}}
+enabled: true
+merge:
+  spec:
+    minAvailable: 2
+{{- end -}}
 {{- end -}}
 
 {{/* JetStream stream replica factor (RF) under HA. Applied to the stream
@@ -73,134 +118,126 @@ In-scope charts MUST expose the required value hooks.
 {{- dig "highAvailability" "nats" "jetstream" "replicaFactor" 3 . -}}
 {{- end -}}
 
-{{/* Soft/hard pod anti-affinity on hostname for a Helm release instance name.
-     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
-{{- define "nvcf.ha.statelessAffinity" -}}
-{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
-{{- $paa := dig "highAvailability" "stateless" "podAntiAffinity" dict .Values -}}
-{{- if dig "enabled" true $paa -}}
-affinity:
-  podAntiAffinity:
-{{- if eq $haMode "enforced" }}
-    requiredDuringSchedulingIgnoredDuringExecution:
-      - labelSelector:
-          matchExpressions:
-            - key: app.kubernetes.io/instance
-              operator: In
-              values:
-                - {{ .instance | quote }}
-        topologyKey: kubernetes.io/hostname
-{{- else }}
-    preferredDuringSchedulingIgnoredDuringExecution:
-      - weight: 100
-        podAffinityTerm:
-          labelSelector:
-            matchExpressions:
-              - key: app.kubernetes.io/instance
-                operator: In
-                values:
-                  - {{ .instance | quote }}
-          topologyKey: kubernetes.io/hostname
-{{- end }}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/* Zone-level topology spread for a Helm release instance name. Spreads
-     replicas across topology.kubernetes.io/zone. whenUnsatisfiable follows the
-     mode: preferred → ScheduleAnyway (best effort), enforced →
-     DoNotSchedule (hard). Nodes MUST carry topology.kubernetes.io/zone.
-     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
-{{- define "nvcf.ha.statelessTopologySpread" -}}
-{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
-{{- $ts := dig "highAvailability" "stateless" "topologySpread" dict .Values -}}
-{{- if dig "enabled" true $ts -}}
-topologySpreadConstraints:
-  - maxSkew: {{ dig "maxSkew" 1 $ts }}
-    topologyKey: topology.kubernetes.io/zone
-    whenUnsatisfiable: {{ if eq $haMode "enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
-    labelSelector:
-      matchLabels:
-        app.kubernetes.io/instance: {{ .instance | quote }}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/* Hostname pod anti-affinity for a Tier-2 quorum release (Cassandra, NATS,
-     OpenBao) so the 3 quorum peers land on distinct nodes. Matches
-     app.kubernetes.io/instance = release name. preferred → preferred (soft,
-     stays schedulable on small pools), enforced → required (hard). Emits the
-     podAntiAffinity body only (no affinity: wrapper) so callers can nest it as a
-     map (Cassandra affinity, NATS podTemplate.merge.spec.affinity) or a string
-     (OpenBao server.affinity). Gated by highAvailability.tier2.podAntiAffinity.
-     Context: dict "Values" $.Values "instance" "<release-instance>" */}}
-{{- define "nvcf.ha.tier2Affinity" -}}
-{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
-{{- $paa := dig "highAvailability" "tier2" "podAntiAffinity" dict .Values -}}
-{{- if dig "enabled" true $paa -}}
-podAntiAffinity:
-{{- if eq $haMode "enforced" }}
-  requiredDuringSchedulingIgnoredDuringExecution:
-    - labelSelector:
+{{/* Mode-derived hostname pod anti-affinity body (no podAntiAffinity: key),
+     hard under enforced, soft under preferred, empty under none. Internal:
+     called only from nvcf.ha.podAntiAffinity. Context: dict "haMode" "<mode>"
+     "instance" "<release-instance>" */}}
+{{- define "nvcf.ha._generatedPodAntiAffinity" -}}
+{{- if eq .haMode "enforced" -}}
+requiredDuringSchedulingIgnoredDuringExecution:
+  - labelSelector:
+      matchExpressions:
+        - key: app.kubernetes.io/instance
+          operator: In
+          values:
+            - {{ .instance | quote }}
+    topologyKey: kubernetes.io/hostname
+{{- else if eq .haMode "preferred" -}}
+preferredDuringSchedulingIgnoredDuringExecution:
+  - weight: 100
+    podAffinityTerm:
+      labelSelector:
         matchExpressions:
           - key: app.kubernetes.io/instance
             operator: In
             values:
               - {{ .instance | quote }}
       topologyKey: kubernetes.io/hostname
-{{- else }}
-  preferredDuringSchedulingIgnoredDuringExecution:
-    - weight: 100
-      podAffinityTerm:
-        labelSelector:
-          matchExpressions:
-            - key: app.kubernetes.io/instance
-              operator: In
-              values:
-                - {{ .instance | quote }}
-        topologyKey: kubernetes.io/hostname
-{{- end }}
-{{- end -}}
 {{- end -}}
 {{- end -}}
 
-{{/* Zone topology spread for a Tier-2 quorum release (Cassandra, NATS,
-     OpenBao) so the 3 quorum peers land in distinct availability zones. Matches
-     app.kubernetes.io/instance = release name on topology.kubernetes.io/zone.
-     OPT-IN: gated by highAvailability.tier2.topologySpread.enabled (default
-     false) because correct behaviour depends on operator infrastructure — a
-     StorageClass with volumeBindingMode: WaitForFirstConsumer (so each zonal PV
-     is created in the zone the scheduler picks) and capacity in >= 3 AZs.
-     whenUnsatisfiable defaults to ScheduleAnyway (soft) so a short AZ never
-     leaves a peer Pending; set topologySpread.strict: true (>= 3-AZ clusters
-     only) for DoNotSchedule (hard). Emits only the constraint list items (no
-     topologySpreadConstraints: key) so callers can nest it as a list (Cassandra
-     value, NATS podTemplate.merge.spec) or a string (OpenBao
-     server.topologySpreadConstraints). Context: dict "Values" $.Values
+{{/* Mode-derived zone topology-spread constraint list items (no
+     topologySpreadConstraints: key), ScheduleAnyway under preferred,
+     DoNotSchedule under enforced, empty under none. Internal: called only
+     from nvcf.ha.topologySpreadItems. Context: dict "haMode" "<mode>"
      "instance" "<release-instance>" */}}
-{{- define "nvcf.ha.tier2TopologySpread" -}}
-{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
-{{- if or (eq $haMode "preferred") (eq $haMode "enforced") -}}
-{{- $ts := dig "highAvailability" "tier2" "topologySpread" dict .Values -}}
-{{- if dig "enabled" false $ts -}}
-- maxSkew: {{ dig "maxSkew" 1 $ts }}
+{{- define "nvcf.ha._generatedTopologySpread" -}}
+{{- if or (eq .haMode "preferred") (eq .haMode "enforced") -}}
+- maxSkew: 1
   topologyKey: topology.kubernetes.io/zone
-  whenUnsatisfiable: {{ if dig "strict" false $ts }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
+  whenUnsatisfiable: {{ if eq .haMode "enforced" }}DoNotSchedule{{ else }}ScheduleAnyway{{ end }}
   labelSelector:
     matchLabels:
       app.kubernetes.io/instance: {{ .instance | quote }}
 {{- end -}}
 {{- end -}}
+
+{{/* Resolves the content of a workload's affinity: block: global.affinity.<class>
+     -> global.affinity.all -> the mode-derived convention. Emits content ready
+     to nest directly under an affinity: key (map value), or to wrap as a
+     string (chart values that take affinity as templated YAML text, e.g.
+     OpenBao) — never the affinity: key itself, so callers control that
+     nesting. An operator override IS the full affinity content (may set
+     podAntiAffinity, podAffinity, and/or nodeAffinity) and is emitted as-is,
+     not merged with the generated convention. The generated convention emits
+     only podAntiAffinity, matching that same "content under affinity:"
+     contract. Context: dict "Values" $.Values "instance" "<release-instance>"
+     "class" "<workload-class>" */}}
+{{- define "nvcf.ha.podAntiAffinity" -}}
+{{- $byClass := dig "global" "affinity" dict .Values -}}
+{{- if hasKey $byClass .class -}}
+{{- with index $byClass .class -}}{{- toYaml . -}}{{- end -}}
+{{- else if hasKey $byClass "all" -}}
+{{- with index $byClass "all" -}}{{- toYaml . -}}{{- end -}}
+{{- else -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- with include "nvcf.ha._generatedPodAntiAffinity" (dict "haMode" $haMode "instance" .instance) }}
+podAntiAffinity:
+{{- . | nindent 2 }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Resolves topologySpreadConstraints for a workload: same fallback as
+     nvcf.ha.podAntiAffinity via global.topologySpreadConstraints. Emits the
+     constraint list items only (no topologySpreadConstraints: key), for the
+     same nesting/string reasons as nvcf.ha.podAntiAffinity. Context: dict
+     "Values" $.Values "instance" "<release-instance>" "class" "<workload-class>" */}}
+{{- define "nvcf.ha.topologySpreadItems" -}}
+{{- $byClass := dig "global" "topologySpreadConstraints" dict .Values -}}
+{{- if hasKey $byClass .class -}}
+{{- with index $byClass .class -}}{{- toYaml . -}}{{- end -}}
+{{- else if hasKey $byClass "all" -}}
+{{- with index $byClass "all" -}}{{- toYaml . -}}{{- end -}}
+{{- else -}}
+{{- $haMode := dig "highAvailability" "mode" "none" .Values | toString -}}
+{{- include "nvcf.ha._generatedTopologySpread" (dict "haMode" $haMode "instance" .instance) -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Full affinity: block wrapper around nvcf.ha.podAntiAffinity, for release
+     values that take affinity as a plain map with no other keys nvcf.yaml
+     needs to set. Context: same as nvcf.ha.podAntiAffinity */}}
+{{- define "nvcf.ha.affinity" -}}
+{{- with include "nvcf.ha.podAntiAffinity" . -}}
+affinity:
+{{- . | nindent 2 }}
+{{- end -}}
+{{- end -}}
+
+{{/* Full topologySpreadConstraints: block wrapper around
+     nvcf.ha.topologySpreadItems. Context: same as nvcf.ha.topologySpreadItems */}}
+{{- define "nvcf.ha.topologySpreadConstraints" -}}
+{{- with include "nvcf.ha.topologySpreadItems" . -}}
+topologySpreadConstraints:
+{{- . | nindent 2 }}
+{{- end -}}
 {{- end -}}
 
 {{- $haMode := dig "highAvailability" "mode" "none" .Values | toString }}
+{{- if not (kindIs "string" $haMode) }}
+{{- fail "highAvailability.mode must be a string" }}
+{{- end }}
 {{- if not (has $haMode (list "none" "preferred" "enforced")) }}
 {{- fail (printf "highAvailability.mode must be none, preferred, or enforced, got %q" $haMode) }}
 {{- end }}
 {{- $haEnabled := or (eq $haMode "preferred") (eq $haMode "enforced") }}
+{{- range $key := list "affinity" "topologySpreadConstraints" }}
+{{- $byClass := dig "global" $key dict $.Values }}
+{{- if not (kindIs "map" $byClass) }}
+{{- fail (printf "global.%s must be a map keyed by workload class (or \"all\")" $key) }}
+{{- end }}
+{{- end }}
 
 cassandra:
   global:
@@ -214,13 +251,13 @@ cassandra:
     defaultStorageClass: {{ .Values.global.storageClass }}
     {{- end }}
 
-  replicaCount: {{ if $haEnabled }}{{ dig "highAvailability" "cassandra" "replicaCount" 3 .Values }}{{ else }}{{ dig "cassandra" "replicaCount" 3 .Values }}{{ end }}
+  replicaCount: {{ if $haEnabled }}3{{ else }}{{ dig "cassandra" "replicaCount" 3 .Values }}{{ end }}
   resourcesPreset: {{ dig "cassandra" "resourcesPreset" "xlarge" .Values }}
 
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "cassandra" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.quorumPodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "cassandra" "podDisruptionBudget" dict .Values }}
@@ -236,11 +273,11 @@ cassandra:
   {{- . | nindent 2 }}
   {{- end }}
   {{- if $haEnabled }}
-  {{- with include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "cassandra") }}
+  {{- with include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "cassandra" "class" "cassandra") }}
   affinity:
     {{- . | nindent 4 }}
   {{- end }}
-  {{- with include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "cassandra") }}
+  {{- with include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "cassandra" "class" "cassandra") }}
   topologySpreadConstraints:
     {{- . | nindent 4 }}
   {{- end }}
@@ -350,7 +387,7 @@ openbao:
     {{- with include "nvcf.tolerations" (dict "type" "vault" "tolerations" .Values.global.tolerations) }}
     {{- . | nindent 4 }}
     {{- end }}
-    replicas: {{ if $haEnabled }}{{ dig "highAvailability" "openbao" "injector" "replicas" 2 .Values }}{{ else }}{{ .Values.openbao.injector.replicas }}{{ end }}
+    replicas: {{ if $haEnabled }}2{{ else }}{{ .Values.openbao.injector.replicas }}{{ end }}
     {{- $nvcfUiEnabled := dig "addons" "nvcfUi" "enabled" false .Values }}
     {{- with dig "openbao" "injector" "webhook" dict .Values }}
     webhook:
@@ -372,15 +409,15 @@ openbao:
       minAvailable: {{ dig "openbao" "injector" "podDisruptionBudget" "minAvailable" 1 .Values }}
   server:
     {{- if $haEnabled }}
-    {{- with include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "openbao-server") }}
+    {{- with include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "openbao-server" "class" "vault") }}
     # String form: the OpenBao chart tpl-renders server.affinity. The wrapper
     # sets it to "" for single-node installs; HA re-enables hostname spread.
     affinity: |
       {{- . | nindent 6 }}
     {{- end }}
-    {{- with include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "openbao-server") }}
+    {{- with include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "openbao-server" "class" "vault") }}
     # String form: the OpenBao chart tpl-renders server.topologySpreadConstraints
-    # under a topologySpreadConstraints: key. Opt-in via tier2.topologySpread.
+    # under a topologySpreadConstraints: key.
     topologySpreadConstraints: |
       {{- . | nindent 6 }}
     {{- end }}
@@ -403,8 +440,8 @@ openbao:
 
     ha:
       {{- if $haEnabled }}
-      enabled: {{ dig "highAvailability" "openbao" "ha" "enabled" true .Values }}
-      replicas: {{ dig "highAvailability" "openbao" "ha" "replicas" 3 .Values }}
+      enabled: true
+      replicas: 3
       {{- end }}
       {{- $openbaoDisruptionBudget := dict }}
       {{- if $haEnabled }}
@@ -459,8 +496,8 @@ nats:
   {{- $natsAffinity := "" -}}
   {{- $natsTopoSpread := "" -}}
   {{- if $haEnabled -}}
-  {{- $natsAffinity = include "nvcf.ha.tier2Affinity" (dict "Values" .Values "instance" "nats") -}}
-  {{- $natsTopoSpread = include "nvcf.ha.tier2TopologySpread" (dict "Values" .Values "instance" "nats") -}}
+  {{- $natsAffinity = include "nvcf.ha.podAntiAffinity" (dict "Values" .Values "instance" "nats" "class" "controlplane") -}}
+  {{- $natsTopoSpread = include "nvcf.ha.topologySpreadItems" (dict "Values" .Values "instance" "nats" "class" "controlplane") -}}
   {{- end -}}
   {{- if or $natsNs $natsTol $natsAffinity $natsTopoSpread }}
   podTemplate:
@@ -500,7 +537,7 @@ nats:
     {{- if $haEnabled }}
     cluster:
       enabled: true
-      replicas: {{ dig "highAvailability" "nats" "replicas" 3 .Values }}
+      replicas: 3
     {{- end }}
     {{- if .Values.global.storageClass }}
     jetstream:
@@ -515,9 +552,9 @@ nats:
     {{- end }}
 
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "nats" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.natsQuorumPodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "nats" "podDisruptionBudget" dict .Values }}
@@ -556,12 +593,16 @@ apikeys:
 
 natsAuthCalloutService:
   {{- if $haEnabled }}
-  replicaCount: {{ include "nvcf.ha.hotPathReplicaCount" .Values }}
+  replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}
+  {{- end }}
+  {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }}
+  podDisruptionBudget:
+    {{- . | nindent 4 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "nats-auth-callout-service") }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "nats-auth-callout-service" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "nats-auth-callout-service") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "nats-auth-callout-service" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
   {{- if .Values.global.imagePullSecrets }}
@@ -740,7 +781,7 @@ natsAuthCalloutService:
 api:
   fullnameOverride: nvcf-api
   {{- if $haEnabled }}
-  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}
   {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
@@ -760,16 +801,16 @@ api:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "api") }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "api" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "api") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "api" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "api" "podDisruptionBudget" dict .Values }}
@@ -913,10 +954,10 @@ invocation:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "invocation-service") }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "invocation-service" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "invocation-service") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "invocation-service" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
 
@@ -1008,8 +1049,8 @@ grpcproxy:
   {{- . | nindent 2 }}
   {{- end }}
   {{- if $haEnabled }}
-  {{- $grpcAffinity := include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "grpc-proxy") }}
-  {{- $grpcTopologySpread := include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "grpc-proxy") }}
+  {{- $grpcAffinity := include "nvcf.ha.affinity" (dict "Values" .Values "instance" "grpc-proxy" "class" "controlplane") }}
+  {{- $grpcTopologySpread := include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "grpc-proxy" "class" "controlplane") }}
   {{- if or $grpcAffinity $grpcTopologySpread }}
   deployment:
     {{- with $grpcAffinity }}
@@ -1061,11 +1102,11 @@ rateLimiter:
   nodeSelector:
     {{ .Values.global.nodeSelectors.controlplane.key }}: {{ .Values.global.nodeSelectors.controlplane.value }}
   {{- end }}
-  replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.hotPathReplicaCount" .Values }}{{ else }}{{ .Values.rateLimiter.replicaCount }}{{ end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "ratelimiter") }}
+  replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}{{ else }}{{ .Values.rateLimiter.replicaCount }}{{ end }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "ratelimiter" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "ratelimiter") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "ratelimiter" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
   {{- if .Values.global.observability.tracing.enabled }}
@@ -1073,9 +1114,9 @@ rateLimiter:
     OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}"
   {{- end }}
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "hotPath" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "rateLimiter" "podDisruptionBudget" dict .Values }}
@@ -1176,7 +1217,7 @@ sis:
 adminIssuerProxy:
   fullnameOverride: admin-token-issuer-proxy
   {{- if $haEnabled }}
-  replicaCount: {{ include "nvcf.ha.statelessReplicaCount" .Values }}
+  replicaCount: {{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}
   {{- end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
@@ -1194,10 +1235,10 @@ adminIssuerProxy:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "admin-issuer-proxy") }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "admin-issuer-proxy" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "admin-issuer-proxy") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "admin-issuer-proxy" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
   gateway:
@@ -1208,9 +1249,9 @@ adminIssuerProxy:
     hostname: "api-keys.{{ .Values.global.domain }}"
     path: "/v1/admin/keys"
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "adminIssuerProxy" "podDisruptionBudget" dict .Values }}
@@ -1329,7 +1370,7 @@ reval:
 llmApiGateway:
   enabled: {{ dig "addons" "llm" "enabled" false .Values }}
   fullnameOverride: llm-api-gateway
-  replicaCount: {{ dig "addons" "llm" "gateway" "replicaCount" 3 .Values }}
+  replicaCount: {{ if $haEnabled }}{{ include "nvcf.ha.replicaSafeReplicaCount" .Values }}{{ else }}{{ dig "addons" "llm" "gateway" "replicaCount" 3 .Values }}{{ end }}
   {{- if .Values.global.imagePullSecrets }}
   imagePullSecrets:
     {{- toYaml .Values.global.imagePullSecrets | nindent 4 }}
@@ -1349,10 +1390,10 @@ llmApiGateway:
   {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessAffinity" (dict "Values" .Values "instance" "llm-api-gateway") }}
+  {{- with include "nvcf.ha.affinity" (dict "Values" .Values "instance" "llm-api-gateway" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
-  {{- with include "nvcf.ha.statelessTopologySpread" (dict "Values" .Values "instance" "llm-api-gateway") }}
+  {{- with include "nvcf.ha.topologySpreadConstraints" (dict "Values" .Values "instance" "llm-api-gateway" "class" "controlplane") }}
   {{- . | nindent 2 }}
   {{- end }}
   {{- if dig "addons" "llm" "gateway" "auth" "grpcInsecure" false .Values }}
@@ -1370,9 +1411,9 @@ llmApiGateway:
       endpoint: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}"
       {{- end }}
   {{- if $haEnabled }}
-  {{- with dig "highAvailability" "stateless" "podDisruptionBudget" dict .Values }}
+  {{- with include "nvcf.ha.replicaSafePodDisruptionBudget" .Values }}
   podDisruptionBudget:
-    {{- toYaml . | nindent 4 }}
+    {{- . | nindent 4 }}
   {{- end }}
   {{- else }}
   {{- with dig "llmApiGateway" "podDisruptionBudget" dict .Values }}
diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
index e3e3c0560d..e60ccdfbe2 100755
--- a/deploy/stacks/self-managed/tests/ha-value-wiring.sh
+++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh
@@ -127,13 +127,14 @@ grep -q "podAntiAffinity:" "$work_dir/cassandra-on.yaml" ||
   fail "cassandra: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
 grep -q "preferredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-on.yaml" ||
   fail "cassandra: expected preferred Tier-2 anti-affinity when highAvailability.mode=preferred"
-# Zone topology spread is opt-in: no zone constraint in the cassandra block
-# unless tier2.topologySpread.enabled (the chart ships an empty default list).
-# NOTE: the write-values file holds every release's values, so scope to the
-# cassandra: block — stateless releases carry their own (expected) zone spread.
-if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "topology.kubernetes.io/zone"; then
-  fail "cassandra: Tier-2 zone spread leaked without tier2.topologySpread.enabled"
-fi
+# Zone topology spread is part of the same mode convention as hostname
+# anti-affinity (soft under preferred) — it is no longer a separate opt-in
+# toggle. NOTE: the write-values file holds every release's values, so scope
+# to the cassandra: block.
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "cassandra: expected Tier-2 zone spread when highAvailability.mode=preferred"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-on.yaml" | grep -q "whenUnsatisfiable: ScheduleAnyway" ||
+  fail "cassandra: expected soft (ScheduleAnyway) Tier-2 spread when highAvailability.mode=preferred"
 
 render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (preferred)"
 grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
@@ -142,12 +143,16 @@ grep -q "podAntiAffinity:" "$work_dir/openbao-on.yaml" ||
   fail "openbao: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
 grep -A3 "disruptionBudget:" "$work_dir/openbao-on.yaml" | grep -q "maxUnavailable: 1" ||
   fail "openbao: expected server.ha.disruptionBudget maxUnavailable=1 when highAvailability.mode=preferred"
+awk '/^openbao:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/openbao-on.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "openbao: expected Tier-2 zone spread when highAvailability.mode=preferred"
 
 render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (preferred)"
 grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null ||
   fail "nats: expected config.cluster.replicas 3 when highAvailability.mode=preferred"
 grep -q "podAntiAffinity:" "$work_dir/nats-on.yaml" ||
   fail "nats: expected Tier-2 anti-affinity when highAvailability.mode=preferred"
+awk '/^nats:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/nats-on.yaml" | grep -q "topology.kubernetes.io/zone" ||
+  fail "nats: expected Tier-2 zone spread when highAvailability.mode=preferred"
 
 # Hot-path helpers (#988): rateLimiter + nats-auth-callout to 2 replicas.
 render_chart_values ratelimiter "$work_dir/ratelimiter-on.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
@@ -163,6 +168,8 @@ render_chart_values nats-auth-callout-service "$work_dir/natsauth-on.yaml" "$cor
   fail "render nats-auth-callout (preferred)"
 awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -E "replicaCount:[[:space:]]*2" >/dev/null ||
   fail "nats-auth-callout: expected replicaCount 2 when highAvailability.mode=preferred"
+awk '/^natsAuthCalloutService:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/natsauth-on.yaml" | grep -q "podDisruptionBudget:" ||
+  fail "nats-auth-callout: expected podDisruptionBudget when highAvailability.mode=preferred (was previously never wired)"
 
 # llm-api-gateway (#987): stateless anti-affinity when the LLM addon is on.
 render_chart_values llm-api-gateway "$work_dir/llmgw-on.yaml" "$core" --state-values-set addons.llm.enabled=true ||
@@ -195,44 +202,72 @@ if awk '/^grpcproxy:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/grpcproxy-on.yaml
   fail "grpc-proxy: HA PDB must not be enabled while single-replica (deferred until Envoy)"
 fi
 
-echo "== highAvailability tier-2 zone topology spread (opt-in) =="
+echo "== global.affinity / global.topologySpreadConstraints fallback (class -> all -> convention) =="
+
+# class-specific global.affinity override wins over the generated convention.
 write_env <<'EOF'
 highAvailability:
   mode: preferred
-  tier2:
-    topologySpread:
-      enabled: true
+global:
+  affinity:
+    cassandra:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                custom: override
+            topologyKey: kubernetes.io/hostname
 EOF
+render_chart_values cassandra "$work_dir/cassandra-global-class.yaml" "$deps" || fail "render cassandra (global.affinity.cassandra override)"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-class.yaml" | grep -q "custom: override" ||
+  fail "cassandra: expected global.affinity.cassandra override to win over the generated convention"
+# The generated anti-affinity's own selector ("operator: In" against
+# app.kubernetes.io/instance) must not also render — only the override
+# content. app.kubernetes.io/instance alone is not distinctive enough to
+# assert on: the (unrelated, mode-derived) topologySpreadConstraints block
+# legitimately uses it too.
+if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-class.yaml" | grep -A2 "podAntiAffinity:" | grep -q "operator: In"; then
+  fail "cassandra: generated anti-affinity must not also render alongside a class override"
+fi
 
-# Scope assertions to each release's own block (the write-values file holds all).
-render_chart_values cassandra "$work_dir/cassandra-spread.yaml" "$deps" || fail "render cassandra (zone spread)"
-awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
-  fail "cassandra: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
-awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread.yaml" | grep -q "whenUnsatisfiable: ScheduleAnyway" ||
-  fail "cassandra: expected soft (ScheduleAnyway) Tier-2 spread by default"
-
-render_chart_values openbao-server "$work_dir/openbao-spread.yaml" "$deps" || fail "render openbao (zone spread)"
-awk '/^openbao:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/openbao-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
-  fail "openbao: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
-
-render_chart_values nats "$work_dir/nats-spread.yaml" "$deps" || fail "render nats (zone spread)"
-awk '/^nats:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/nats-spread.yaml" | grep -q "topology.kubernetes.io/zone" ||
-  fail "nats: expected Tier-2 zone spread when tier2.topologySpread.enabled=true"
+# global.affinity.all applies when no class-specific override exists.
+write_env <<'EOF'
+highAvailability:
+  mode: preferred
+global:
+  affinity:
+    all:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                custom: shared
+            topologyKey: kubernetes.io/hostname
+EOF
+render_chart_values cassandra "$work_dir/cassandra-global-all.yaml" "$deps" || fail "render cassandra (global.affinity.all)"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-all.yaml" | grep -q "custom: shared" ||
+  fail "cassandra: expected global.affinity.all to apply when no class-specific override exists"
 
-# strict: true -> hard (DoNotSchedule), for >= 3-AZ clusters.
+# An explicit {} for a class suppresses the generated anti-affinity entirely —
+# presence is the signal, not truthiness.
 write_env <<'EOF'
 highAvailability:
-  mode: enforced
-  tier2:
-    topologySpread:
-      enabled: true
-      strict: true
+  mode: preferred
+global:
+  affinity:
+    cassandra: {}
+  topologySpreadConstraints:
+    cassandra: []
 EOF
-render_chart_values cassandra "$work_dir/cassandra-spread-strict.yaml" "$deps" || fail "render cassandra (strict spread)"
-awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-spread-strict.yaml" | grep -q "whenUnsatisfiable: DoNotSchedule" ||
-  fail "cassandra: expected hard (DoNotSchedule) Tier-2 spread when strict=true"
+render_chart_values cassandra "$work_dir/cassandra-global-suppressed.yaml" "$deps" || fail "render cassandra (global.affinity.cassandra: {})"
+if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-suppressed.yaml" | grep -q "podAntiAffinity:"; then
+  fail "cassandra: expected explicit global.affinity.cassandra: {} to suppress generated anti-affinity"
+fi
+if awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-global-suppressed.yaml" | grep -q "topology.kubernetes.io/zone"; then
+  fail "cassandra: expected explicit global.topologySpreadConstraints.cassandra: [] to suppress generated zone spread"
+fi
 
-echo "== highAvailability enforced: required anti-affinity =="
+echo "== highAvailability enforced: required anti-affinity, hard zone spread =="
 write_env <<'EOF'
 highAvailability:
   mode: enforced
@@ -247,6 +282,8 @@ grep -q "whenUnsatisfiable: DoNotSchedule" "$work_dir/api-enforced.yaml" ||
 render_chart_values cassandra "$work_dir/cassandra-enforced.yaml" "$deps" || fail "render cassandra (enforced)"
 grep -q "requiredDuringSchedulingIgnoredDuringExecution:" "$work_dir/cassandra-enforced.yaml" ||
   fail "cassandra: expected required Tier-2 anti-affinity when highAvailability.mode=enforced"
+awk '/^cassandra:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/cassandra-enforced.yaml" | grep -q "whenUnsatisfiable: DoNotSchedule" ||
+  fail "cassandra: expected hard (DoNotSchedule) Tier-2 spread when highAvailability.mode=enforced"
 
 render_chart_values ratelimiter "$work_dir/ratelimiter-enforced.yaml" "$core" --state-values-set rateLimiter.enabled=true ||
   fail "render ratelimiter (enforced)"

From 4bb51c2599aec7841cb87ba8710eb45d089c09b8 Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 16:29:36 +0530
Subject: [PATCH 10/11] refactor(self-managed): remove unused
 highAvailability.* subtrees from base.yaml
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Removes highAvailability.stateless, hotPath, tier2,
tier2.topologySpread(.strict), nats (replicas/podDisruptionBudget),
openbao (ha/injector), and cassandra (replicaCount/podDisruptionBudget)
from environments/base.yaml — global.yaml.gotmpl no longer reads any of
these keys after the previous commit's unification, and the design
review's explicit-removals list names every one of them. Only
highAvailability.mode and highAvailability.nats.jetstream.replicaFactor
remain (see the previous commit for why replicaFactor stays
configurable).

No behavior change: every value removed here was already dead
configuration relative to the current template.
---
 .../self-managed/environments/base.yaml       | 124 ++++--------------
 1 file changed, 25 insertions(+), 99 deletions(-)

diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml
index b7a0efda5d..0ce81cc0b3 100644
--- a/deploy/stacks/self-managed/environments/base.yaml
+++ b/deploy/stacks/self-managed/environments/base.yaml
@@ -727,111 +727,37 @@ grpcproxy:
 # ├── environments/<env>.yaml     # highAvailability: configuration
 # └── global.yaml.gotmpl          # value mapping to charts
 #
-# mode: none         (default) — keep existing env/chart replica and PDB values
-#                                (local / CI / BDD).
-#       preferred           — HA sizing below; hostname anti-affinity is
-#                                preferred (still schedule if a second node/AZ
-#                                has no capacity). Requires ≥3 schedulable nodes.
-#       enforced            — same HA sizing; hostname anti-affinity is
-#                                required (second replica stays Pending rather
-#                                than packing). Requires ≥3 schedulable nodes.
+# mode: none         (default) — keep existing env/chart replica, affinity,
+#                                and PDB values (local / CI / BDD).
+#       preferred           — HA sizing and placement below; hostname
+#                                anti-affinity and zone topology spread are
+#                                soft (still schedule if capacity is short).
+#                                Requires ≥3 schedulable nodes.
+#       enforced            — same HA sizing; hostname anti-affinity and zone
+#                                topology spread are hard (Pending rather than
+#                                packing / ignoring the constraint). Requires
+#                                ≥3 schedulable nodes in ≥3 zones for the
+#                                zone constraint to actually schedule.
+#
+# There is no per-component subtree beneath highAvailability: sizing and
+# placement are derived uniformly from mode for every in-scope release (see
+# global.yaml.gotmpl). Shared scheduling tuning is available under
+# global.affinity / global.topologySpreadConstraints (class -> "all" -> the
+# mode convention, same fallback pattern as global.nodeSelectors); an
+# explicit {} / [] there suppresses the generated policy for that class.
+# Component-shaped values elsewhere in this file remain the final per-release
+# escape hatch for replica counts and PDBs.
 #
 # In-scope charts MUST expose the required value hooks. Leave mode: none
 # for single-node installs.
 # =============================================================================
 highAvailability:
   mode: none
-
-  # Stateless control-plane Deployments: nvcf-api, admin-token-issuer-proxy,
-  # and llm-api-gateway (when the LLM addon is enabled). Active-active; no
-  # application leader election.
-  # NOTE: invocation-service and grpc-proxy are intentionally kept at a single
-  # replica for now. Their multi-replica scaling is deferred until Envoy
-  # support lands in the self-hosted stack (worker-callback host binding; see
-  # #987/#989 review). replicaCount below therefore does not apply to them.
-  stateless:
-    replicaCount: 2
-    podAntiAffinity:
-      enabled: true
-    # Spread replicas across availability zones. Nodes MUST be labelled
-    # topology.kubernetes.io/zone=<az>. whenUnsatisfiable follows mode:
-    # preferred → ScheduleAnyway, enforced → DoNotSchedule.
-    topologySpread:
-      enabled: true
-      maxSkew: 1
-    podDisruptionBudget:
-      enabled: true
-      minAvailable: 1
-    rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 0
-
-  # Hot-path helper Deployments raised to 2 replicas under HA so a single pod
-  # loss does not immediately hurt request/auth traffic: rateLimiter,
-  # nats-auth-callout. Reuse stateless.podAntiAffinity (hostname spread) and
-  # stateless.topologySpread (zone spread).
-  hotPath:
-    replicaCount: 2
-    podDisruptionBudget:
-      enabled: true
-      minAvailable: 1
-
-  # Tier-2 quorum services (Cassandra, NATS, OpenBao). Under HA their peers get
-  # hostname pod anti-affinity (matching app.kubernetes.io/instance per release)
-  # so the 3 quorum members land on distinct nodes. whenUnsatisfiable follows
-  # mode: preferred → preferred/soft (stays schedulable on small pools),
-  # enforced → required/hard. The upstream OpenBao chart ships a hard
-  # anti-affinity by default that the stack disables for single-node installs;
-  # this re-enables it (soft/hard by mode) under HA. Set enabled: false to keep
-  # the chart defaults.
-  tier2:
-    podAntiAffinity:
-      enabled: true
-    # Zone topology spread for the 3 quorum peers across
-    # topology.kubernetes.io/zone (Cassandra, NATS, OpenBao). OPT-IN (default
-    # off) because it depends on operator-provided infrastructure:
-    #   * a StorageClass with volumeBindingMode: WaitForFirstConsumer, so each
-    #     pod's zonal PersistentVolume is created in the zone the scheduler
-    #     picks. With Immediate binding the PV zone is fixed first and pods can
-    #     land Pending when the spread constraint disagrees.
-    #   * capacity in >= 3 AZs. A 3-member quorum only survives an AZ loss if no
-    #     single AZ holds a majority; with 2 AZs one zone holds 2 of 3, so
-    #     losing it breaks quorum.
-    # whenUnsatisfiable defaults to ScheduleAnyway (soft) so a short AZ never
-    # leaves a peer Pending. Set strict: true ONLY on >= 3-AZ clusters to make
-    # it DoNotSchedule (hard). NOTE: for Cassandra, data-level AZ diversity also
-    # requires rack = AZ on the nodes (image/entrypoint side), not just pod
-    # spread; see docs/v0.6.1/high-availability.md.
-    topologySpread:
-      enabled: false
-      maxSkew: 1
-      strict: false
-
   nats:
-    replicas: 3
-    # JetStream stream replica factor (RF). Applied under HA to the stream
-    # creators — nvcf-api and invocation-service — so the worker/result streams
-    # they declare are replicated across the NATS cluster (survives a node
-    # loss). RF must be <= nats.replicas. Default 3: JetStream streams use
-    # Raft quorum, so RF=2 loses quorum on a single replica failure and
-    # provides no HA benefit over RF=1.
     jetstream:
+      # JetStream stream replica factor (RF), applied under HA to the stream
+      # creators (nvcf-api, invocation-service) via env so the worker/result
+      # streams they declare are replicated across the NATS cluster. Default
+      # 3: JetStream streams use Raft quorum, so RF=2 loses quorum on a
+      # single replica failure and provides no HA benefit over RF=1.
       replicaFactor: 3
-    podDisruptionBudget:
-      enabled: true
-      merge:
-        spec:
-          minAvailable: 2
-
-  openbao:
-    ha:
-      enabled: true
-      replicas: 3
-    injector:
-      replicas: 2
-
-  cassandra:
-    replicaCount: 3
-    podDisruptionBudget:
-      enabled: true
-      minAvailable: 2

From 9163a4ab2a5c495ce6587dc8c306c7dbe1283a18 Mon Sep 17 00:00:00 2001
From: Anand Parthasarathi <aparthasarat@nvidia.com>
Date: Thu, 17 Sep 2026 16:32:37 +0530
Subject: [PATCH 11/11] docs(self-managed): move HA guide to docs/user/, fix
 guarantee overclaims
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Moves docs/v0.6.1/high-availability.md (frozen version-pinned tree) to
docs/user/high-availability.md (the live/current docs tree the site
navigates by default), adds it to fern/versions/dev.yml next to the
related Pod Disruption Budgets page, and drops its now-dead link from
the frozen docs/v0.6.1/index.md.

Content updates, mostly addressing review feedback on #1679/#1683:

- ha-preferred/ha-enforced -> preferred/enforced (matches the mode
  rename in an earlier commit).
- Replaces the "Stateless" / "Hot-path helper services" split with a
  single "Replica-safe Deployments" section, and adds a new "Tuning
  scheduling policy" section documenting global.affinity /
  global.topologySpreadConstraints (class -> all -> convention
  fallback, explicit {}/[] suppression) — the redesigned public API
  from the earlier commits, replacing the old highAvailability.stateless
  / .hotPath / .tier2 subtree documentation.
- Quorum zone spread is documented as part of the standard mode
  convention (on by default), not a separate opt-in
  (highAvailability.tier2.topologySpread.enabled no longer exists).
  The prerequisites (WaitForFirstConsumer StorageClass, >=3-AZ
  capacity) are reframed as "what it takes for the AZ-loss guarantee
  to actually hold", with an explicit warning that switching enforced
  -> preferred does not substitute for having the capacity — it just
  hides the gap behind a soft constraint.
- NATS JetStream RF corrected from 2 to 3 throughout (matches the
  earlier code-default fix); explains why RF=2 gives no HA benefit
  (Raft quorum loses tolerance the moment either replica is down).
- Adds an explicit guarantee summary distinguishing preferred (survives
  a pod failure, best-effort node separation) from enforced (survives a
  pod or node failure when prerequisites are met), and states plainly
  that neither mode alone claims AZ/site-loss tolerance.
- "Recovery objectives" section no longer states AZ-loss survival as a
  given under HA; it's conditioned on the zone-spread prerequisites
  actually being met.
---
 docs/{v0.6.1 => user}/high-availability.md | 244 +++++++++++++--------
 docs/v0.6.1/index.md                       |   2 -
 fern/versions/dev.yml                      |   2 +
 3 files changed, 159 insertions(+), 89 deletions(-)
 rename docs/{v0.6.1 => user}/high-availability.md (51%)

diff --git a/docs/v0.6.1/high-availability.md b/docs/user/high-availability.md
similarity index 51%
rename from docs/v0.6.1/high-availability.md
rename to docs/user/high-availability.md
index 5aca41cf68..e0bffaaf30 100644
--- a/docs/v0.6.1/high-availability.md
+++ b/docs/user/high-availability.md
@@ -12,22 +12,30 @@ leave it off and are unaffected by anything in this guide.
 ## Overview
 
 HA is controlled by a single switch in your Helmfile environment values,
-`highAvailability.mode`. The self-managed stack maps that mode onto per-chart
-values (replica counts, pod anti-affinity, zone topology spread, pod disruption
-budgets, and data-durability settings) in
-`deploy/stacks/self-managed/global.yaml.gotmpl`.
+`highAvailability.mode`. There is no per-component HA configuration beneath
+it: sizing and placement are derived uniformly from the mode for every
+in-scope release, in `deploy/stacks/self-managed/global.yaml.gotmpl`.
 
 | `highAvailability.mode` | Behavior |
 | --- | --- |
 | `none` (default) | Keep existing single-replica chart/env values. Nothing in this guide applies. Use for local, CI, and single-node installs. |
-| `ha-preferred` | HA sizing (multiple replicas, quorum services at 3). Node/AZ spread is a **preference**: if a second node or AZ has no capacity, pods still schedule (co-located) rather than staying `Pending`. |
-| `ha-enforced` | Same HA sizing, but node/AZ spread is **required**: a replica that cannot land on a distinct node/AZ stays `Pending` instead of packing onto an occupied one. |
+| `preferred` | HA sizing (multiple replicas, quorum services at 3). Node/AZ spread is a **preference**: if a second node or AZ has no capacity, pods still schedule (co-located) rather than staying `Pending`. |
+| `enforced` | Same HA sizing, but node/AZ spread is **required**: a replica that cannot land on a distinct node/AZ stays `Pending` instead of packing onto an occupied one. |
 
-`ha-preferred` is the recommended starting point: it gives you the full HA
+`preferred` is the recommended starting point: it gives you the full HA
 topology while degrading gracefully on a constrained cluster. Move to
-`ha-enforced` once you have confirmed your node pools have capacity in every AZ
+`enforced` once you have confirmed your node pools have capacity in every AZ
 and you want hard placement guarantees.
 
+`preferred` guarantees continuity after a single **pod** failure but provides
+only best-effort node/zone separation — under capacity pressure, replicas can
+still end up co-located. `enforced` guarantees continuity after a single pod
+**or node** failure when the prerequisites below are met. Neither mode alone
+claims arbitrary availability-zone or site-loss tolerance; see
+[Zone spread for the quorum services](#zone-spread-for-the-quorum-services)
+and [Recovery objectives](#recovery-objectives-and-failure-behavior) for what
+it takes to survive an AZ loss specifically.
+
 ## Cluster prerequisites
 
 HA depends on infrastructure the **operator** provides. The stack cannot create
@@ -36,8 +44,11 @@ nodes or AZs for you — it only schedules against what you label.
 ### 1. Node count
 
 Both HA modes require **at least 3 schedulable nodes** in the pool(s) that host
-control-plane and quorum workloads. The quorum services (Cassandra, NATS,
-OpenBao) run 3 replicas that must land on 3 distinct nodes.
+control-plane and quorum workloads for the *intended* spread to actually
+schedule — the quorum services (Cassandra, NATS, OpenBao) run 3 replicas.
+Under `preferred`, fewer nodes degrade to co-located (but still Ready) pods
+rather than blocking the install; under `enforced`, insufficient nodes leave
+replicas `Pending`.
 
 ### 2. Availability-zone labels
 
@@ -57,9 +68,10 @@ kubectl label node <node-name> topology.kubernetes.io/zone=az-1
 ```
 
 If the label is absent, zone topology spread has nothing to spread across. In
-`ha-preferred` this silently degrades to node-level spread only; in
-`ha-enforced` zone-constrained pods can stay `Pending`. Aim for capacity in
-**at least two AZs** (three is better for the quorum services).
+`preferred` this silently degrades to node-level spread only; in
+`enforced` zone-constrained pods can stay `Pending`. Aim for capacity in
+**at least two AZs** (three is better for the quorum services — see
+[Zone spread for the quorum services](#zone-spread-for-the-quorum-services)).
 
 ### 3. Dedicated node pools (recommended)
 
@@ -86,10 +98,14 @@ global:
 
 | Pool | Selector value | Hosts |
 | --- | --- | --- |
-| `controlplane` | `control-plane` | Stateless + hot-path Deployments |
+| `controlplane` | `control-plane` | All Tier-1 Deployments (api, invocation, grpcproxy, adminIssuerProxy, rateLimiter, natsAuthCalloutService, llmApiGateway, nats) |
 | `cassandra` | `cassandra` | Cassandra StatefulSet |
 | `vault` | `vault` | OpenBao StatefulSet |
 
+These same three classes (`controlplane`/`cassandra`/`vault`) are also the
+classes used by `global.affinity` and `global.topologySpreadConstraints`
+below, so node-pool selection and scheduling-policy tuning stay consistent.
+
 Each dedicated pool must span the availability zones — that is, have **capacity
 in every AZ you want to spread across** (both zones in a 2-AZ cluster, all three
 in a 3-AZ cluster). A 3-node Cassandra pool concentrated in one AZ cannot spread
@@ -106,12 +122,11 @@ Set the mode in your environment file (for example
 
 ```yaml
 highAvailability:
-  mode: ha-preferred
+  mode: preferred
 ```
 
-That single line activates all of the defaults documented below. Every knob
-under `highAvailability` has a sensible default; override only what you need.
-Then apply the stack as usual:
+That single line activates all of the defaults documented below. Then apply
+the stack as usual:
 
 ```bash
 helmfile -e <env> apply
@@ -122,11 +137,11 @@ silently disable HA.
 
 ## What HA changes, by tier
 
-### Stateless control-plane services
+### Replica-safe Deployments
 
-Active-active Deployments with no leader election: `nvcf-api`,
-`admin-token-issuer-proxy`, and `llm-api-gateway` (when the LLM addon is
-enabled).
+Active-active Deployments with no leader election: `api`, `rateLimiter`,
+`natsAuthCalloutService`, `adminIssuerProxy`, and `llmApiGateway` (when the
+LLM addon is enabled) all follow the same convention.
 
 > **Note — `invocation-service` and `grpc-proxy` are deferred.** These two are
 > stateless too, but their multi-replica scaling is intentionally **held at a
@@ -140,67 +155,74 @@ enabled).
 
 Under HA each of these gets:
 
-- **2 replicas** (`highAvailability.stateless.replicaCount`).
+- **2 replicas.**
 - **Hostname pod anti-affinity** so the two replicas never share a node.
 - **Zone topology spread** (`topology.kubernetes.io/zone`, `maxSkew: 1`) so they
   land in different AZs when zones are labelled.
 - **A PodDisruptionBudget** (`minAvailable: 1`) so voluntary disruptions
   (drains, upgrades) never take the last replica.
 - **A surge rolling-update strategy** (`maxSurge: 1`, `maxUnavailable: 0`) so a
-  new pod is Ready before an old one is removed.
-
-### Hot-path helper services
+  new pod is Ready before an old one is removed (api only, at present).
 
-`rateLimiter` and `nats-auth-callout` sit on the request/auth hot path, so under
-HA they are raised to **2 replicas** (`highAvailability.hotPath.replicaCount`)
-with their own PDB (`minAvailable: 1`). They reuse the stateless hostname
-anti-affinity and zone spread.
+Affinity and topology spread are shared scheduling policy, so they can be
+tuned once for every release of a class instead of per-component. See
+[Tuning scheduling policy](#tuning-scheduling-policy-globalaffinity--globaltopologyspreadconstraints)
+below.
 
-### Tier-2 quorum services (data durability)
+### Quorum services (data durability)
 
 `Cassandra`, `NATS`, and `OpenBao` run as **3-replica quorum StatefulSets** with:
 
-- **Hostname pod anti-affinity** so the 3 peers land on 3 distinct nodes
-  (`highAvailability.tier2.podAntiAffinity`). Following the mode, this is
-  preferred (soft) under `ha-preferred` and required (hard) under `ha-enforced`.
-  OpenBao's upstream chart ships a hard anti-affinity that the stack disables for
-  single-node installs and re-enables (soft/hard by mode) under HA.
-- **PodDisruptionBudgets** sized to preserve quorum (`minAvailable: 2`).
-
-#### Zone spread for the quorum pods (opt-in)
-
-By default the quorum peers are only guaranteed distinct **nodes**, not distinct
-**zones** — three nodes can all be in one AZ, so a single-AZ loss could still
-break quorum. To spread the three peers across `topology.kubernetes.io/zone`,
-enable:
+- **Hostname pod anti-affinity** so the 3 peers land on 3 distinct nodes.
+  Following the mode, this is preferred (soft) under `preferred` and required
+  (hard) under `enforced` — the same convention as replica-safe Deployments.
+  OpenBao's upstream chart ships a hard anti-affinity that the stack disables
+  for single-node installs and re-enables (soft/hard by mode) under HA.
+- **Zone topology spread** across `topology.kubernetes.io/zone`, same
+  soft/hard-by-mode convention. See the next section for what it takes for
+  this to actually protect against an AZ loss.
+- **PodDisruptionBudgets** sized to tolerate exactly one voluntary disruption
+  (`minAvailable: 2` of 3, or OpenBao's equivalent `maxUnavailable: 1`).
+
+#### Zone spread for the quorum services
+
+Zone topology spread for the quorum peers is part of the same mode
+convention as everything else — it is **not** a separate toggle. That said,
+whether it actually protects you from an AZ loss depends on infrastructure
+you must provide:
+
+- **StorageClass `volumeBindingMode: WaitForFirstConsumer`.** Each quorum pod
+  has a zonal PersistentVolume, and a zonal disk can only attach to a node in
+  its own AZ. With `WaitForFirstConsumer`, the scheduler places the pod first
+  (honoring the spread constraint) and the PV is then created in that pod's
+  zone. With `Immediate` binding the PV's zone is chosen up front and the pod
+  is pinned to it, which fights the spread constraint and can leave pods
+  `Pending`. The stack cannot set this for you — it is a property of the
+  StorageClass you supply.
+- **Capacity in at least 3 AZs.** A 3-member quorum only survives an AZ loss
+  if no single AZ holds a majority. With only 2 AZs one zone inevitably holds
+  2 of 3 members, and losing that zone breaks quorum.
+
+If you do not have 3-AZ capacity or a `WaitForFirstConsumer` StorageClass,
+zone spread for the quorum tier will not reliably schedule or will not
+protect you from an AZ loss even though it renders. Switching from `enforced`
+to `preferred` alone does **not** fix this — `preferred`'s soft
+(`ScheduleAnyway`) constraint will let pods co-locate in that case rather than
+failing loudly, which hides the gap rather than closing it. To disable
+generated zone spread for the quorum tier explicitly instead of relying on a
+mode change, use the `global.topologySpreadConstraints` escape hatch:
 
 ```yaml
-highAvailability:
-  tier2:
-    topologySpread:
-      enabled: true      # off by default
-      maxSkew: 1
-      strict: false      # ScheduleAnyway (soft). Set true only on >= 3 AZs.
+global:
+  topologySpreadConstraints:
+    cassandra: []
+    vault: []
 ```
 
-This is **opt-in** because, unlike the stateless tier, it depends on
-infrastructure you must provide:
-
-- **StorageClass `volumeBindingMode: WaitForFirstConsumer`.** Each quorum pod has
-  a zonal PersistentVolume, and a zonal disk can only attach to a node in its own
-  AZ. With `WaitForFirstConsumer`, the scheduler places the pod first (honoring
-  the spread constraint) and the PV is then created in that pod's zone. With
-  `Immediate` binding the PV's zone is chosen up front and the pod is pinned to
-  it, which fights the spread constraint and can leave pods `Pending`. The stack
-  cannot set this for you — it is a property of the StorageClass you supply.
-- **Capacity in at least 3 AZs.** A 3-member quorum only survives an AZ loss if
-  no single AZ holds a majority. With only 2 AZs one zone inevitably holds 2 of
-  3 members, and losing that zone breaks quorum. On a 2-AZ cluster leave zone
-  spread off (or keep `strict: false`).
-
-`whenUnsatisfiable` defaults to `ScheduleAnyway` (soft) so a temporarily short
-AZ never leaves a peer `Pending`. Only set `strict: true` — which makes it
-`DoNotSchedule` (hard) — on clusters you know have capacity in ≥3 AZs.
+(NATS shares the `controlplane` class with the stateless tier; scope an
+override to just NATS via a component-shaped value instead if you need to
+disable spread for NATS specifically without affecting the rest of
+`controlplane`.)
 
 **Cassandra needs one more thing: rack = AZ.** Spreading the *pods* across zones
 does not by itself make the *data* zone-diverse. `NetworkTopologyStrategy`
@@ -220,18 +242,20 @@ Beyond placement, HA also raises the data-durability settings:
 #### NATS JetStream replica factor
 
 Streams default to a single replica. Under HA the stack sets the JetStream
-replica factor (RF) to **2** (`highAvailability.nats.jetstream.replicaFactor`)
+replica factor (RF) to **3** (`highAvailability.nats.jetstream.replicaFactor`)
 on the two services that create streams — `nvcf-api` (via `NVCF_NATS_REPLICAS`)
-and `invocation-service` (via `NATS_PROPERTIES__REPLICAS`). With RF=2 the
-worker/result streams are replicated across the NATS cluster and survive the
-loss of the node hosting the leader. RF must be `<= highAvailability.nats.replicas`.
+and `invocation-service` (via `NATS_PROPERTIES__REPLICAS`). JetStream streams
+use Raft quorum: RF=3 tolerates the loss of one replica, matching the
+3-member NATS cluster. **RF=2 is not sufficient** — a 2-member Raft group
+loses quorum the moment either replica is unavailable, so it provides no
+resilience benefit over RF=1.
 
 #### Cassandra replication and consistency
 
 The Cassandra keyspaces are created with `NetworkTopologyStrategy` and a
-replication factor equal to `highAvailability.cassandra.replicaCount` (3 under
-HA), and the control-plane services read/write at `LOCAL_QUORUM`. This is the
-correct configuration for both single-DC and multi-AZ deployments:
+replication factor of 3 under HA, and the control-plane services read/write
+at `LOCAL_QUORUM`. This is the correct configuration for both single-DC and
+multi-AZ deployments:
 
 - **Single datacenter:** RF=3 with `LOCAL_QUORUM` tolerates the loss of one
   replica for reads and writes.
@@ -244,6 +268,46 @@ No stack change is required to select the strategy — it is
 `NetworkTopologyStrategy` in all cases. To get true cross-AZ placement, ensure
 the Cassandra nodes carry AZ labels (see the prerequisites above).
 
+## Tuning scheduling policy (`global.affinity` / `global.topologySpreadConstraints`)
+
+Affinity and zone topology spread are shared scheduling policy, so you can
+tune them once per workload class instead of overriding every component
+individually. Both resolve in the same order:
+
+```
+class-specific global value  ->  global.<key>.all  ->  the highAvailability.mode-derived convention
+```
+
+The available classes match `global.nodeSelectors`: `controlplane`,
+`cassandra`, `vault`. These values are optional — absence means "use the
+mode-derived convention" — and an explicit override replaces the generated
+policy for that class entirely, it does not merge with it:
+
+```yaml
+highAvailability:
+  mode: preferred
+
+global:
+  affinity:
+    cassandra:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                app.kubernetes.io/instance: cassandra
+            topologyKey: kubernetes.io/hostname
+  topologySpreadConstraints:
+    all: []   # disable generated zone spread everywhere except explicit class overrides
+```
+
+An explicit `{}` (for `affinity`) or `[]` (for `topologySpreadConstraints`)
+suppresses the generated policy for that class — this is different from
+omitting the key, which falls back to `all` and then to the convention.
+
+Component-shaped values (for example `api.affinity`, `cassandra.affinity`) in
+your environment file remain the final, single-release escape hatch and take
+precedence over both `global.affinity` and the generated convention.
+
 ## Validation
 
 After applying HA, confirm replicas are spread as expected.
@@ -260,7 +324,7 @@ kubectl -n nats-system get pods -o wide
 kubectl -n vault-system get pods -o wide
 ```
 
-Confirm the stateless Deployments scaled and spread:
+Confirm the replica-safe Deployments scaled and spread:
 
 ```bash
 kubectl -n nvcf get deploy nvcf-api admin-token-issuer-proxy -o wide
@@ -269,7 +333,7 @@ kubectl -n nvcf get pods -o wide -l app.kubernetes.io/instance=nvcf-api
 kubectl -n nvcf get deploy invocation-service grpc-proxy -o wide
 ```
 
-Confirm the JetStream RF took effect (streams report `Replicas: 2`):
+Confirm the JetStream RF took effect (streams report `Replicas: 3`):
 
 ```bash
 kubectl -n nats-system exec -it nats-0 -- nats stream ls
@@ -283,25 +347,31 @@ kubectl -n cassandra-system exec -it cassandra-0 -- \
   cqlsh -e "SELECT keyspace_name, replication FROM system_schema.keyspaces;"
 ```
 
-If any pod is stuck `Pending` under `ha-enforced`, it usually means a node pool
-lacks capacity in a second node/AZ. Add capacity, or drop to `ha-preferred` to
-let it schedule while you rebalance.
+If any pod is stuck `Pending` under `enforced`, it usually means a node pool
+lacks capacity in a second node/AZ. Add capacity, or drop to `preferred` to
+let it schedule while you rebalance — but see
+[Zone spread for the quorum services](#zone-spread-for-the-quorum-services)
+for why that alone does not restore the AZ-loss guarantee if the underlying
+capacity gap remains.
 
 ## Recovery objectives and failure behavior
 
 With HA enabled and capacity in at least two AZs:
 
-- **Single node loss:** Multi-replica stateless and hot-path Deployments keep
+- **Single node loss:** Multi-replica replica-safe Deployments keep
   serving from their surviving replica; the scheduler recreates the lost pod on
   another node (and the PDB prevents drains from removing the last one).
   `invocation-service` and `grpc-proxy` (single replica until Envoy) are briefly
   unavailable while the scheduler restarts the pod on another node. Quorum
-  services
-  (Cassandra RF=3/`LOCAL_QUORUM`, NATS RF=2, OpenBao 3-node Raft) retain quorum
-  with 2 of 3 members and continue serving reads and writes.
-- **Single AZ loss:** With replicas spread across AZs, the control plane stays
-  available on the surviving AZ(s). Recovery time is dominated by pod
-  reschedule/restart time on the healthy AZ rather than any manual failover.
+  services (Cassandra RF=3/`LOCAL_QUORUM`, NATS RF=3, OpenBao 3-node Raft)
+  retain quorum with 2 of 3 members and continue serving reads and writes.
+- **Single AZ loss:** Only if zone spread actually took effect — see
+  [Zone spread for the quorum services](#zone-spread-for-the-quorum-services)
+  for its prerequisites (3-AZ capacity, `WaitForFirstConsumer` storage). With
+  those met, the control plane stays available on the surviving AZ(s) and
+  recovery time is dominated by pod reschedule/restart time rather than any
+  manual failover. Without them, an AZ loss can take a majority of a quorum
+  service's members and pause writes even with HA enabled.
 - **Two simultaneous quorum-member losses:** A 3-member quorum service loses
   quorum and pauses writes until a member returns. This is why three AZs (or at
   least three nodes across two AZs, with the third member able to reschedule) is
diff --git a/docs/v0.6.1/index.md b/docs/v0.6.1/index.md
index cb2333c5c9..814711d7a3 100644
--- a/docs/v0.6.1/index.md
+++ b/docs/v0.6.1/index.md
@@ -10,8 +10,6 @@ This guide provides information for deploying and operating NVCF in self-managed
   : Connect GPU clusters to the NVCF control plane.
 - [Configuration](./optional-enhancements.md)
   : Configure gateway routing, registries, and optional enhancements.
-- [High Availability](./high-availability.md)
-  : Run the control plane across nodes and availability zones to survive node/AZ loss.
 - [Using Cloud Functions](./api.md)
   : Create and invoke functions using the NVCF API and CLI.
 - [Managed (Legacy)](../ngc-managed/cluster-management/ngc-managed.md)
diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml
index e05048c70d..12f8b93cfc 100644
--- a/fern/versions/dev.yml
+++ b/fern/versions/dev.yml
@@ -55,6 +55,8 @@ navigation:
         path: ../../docs/user/optional-enhancements.md
       - page: Pod Disruption Budgets
         path: ../../docs/user/pod-disruption-budgets.md
+      - page: High Availability
+        path: ../../docs/user/high-availability.md
       - page: LLM Function Enablement
         path: ../../docs/user/llm-function-enablement.md
       - page: LLM Request Router Load Balancing