Recovery: drain handling for GPU reset

charts/gpu-base-operator/README.md

@@ -39,6 +39,7 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 | `operator.resources.limits.memory` | 128Mi | Memory limit for operator pod |
 | `operator.resources.requests.cpu` | 10m | CPU request for operator pod |
 | `operator.resources.requests.memory` | 64Mi | Memory request for operator pod |
+| `operator.tolerations` | [] | Extra tolerations for the operator pod, added on top of the built-in one for `gpurecoveryplan.intel.com/recovery`. Needed for `GPUFirmwareUpdate`, whose taint key is set by `spec.updateTaint`. |
 | `metrics.enabled` | false | Serve the operator's own metrics endpoint |
 | `metrics.port` | 8443 | Port for the metrics endpoint |
 | `metrics.secure` | true | Serve metrics over HTTPS behind the authn/authz filter |

charts/gpu-base-operator/templates/manager.yaml

@@ -40,6 +40,25 @@ spec:
       #             operator: In
       #             values:
       #               - linux
+      # The operator must tolerate the taints it applies itself, because it is the only thing
+      # that removes them. A GPURecoveryPlan reset cordons the node it is recovering; if that
+      # reset wedges the host and the operator pod is later evicted, its replacement has to be
+      # schedulable somewhere. On a single-node cluster — or one where every other node is full
+      # or tainted — the only candidate is the cordoned node itself, and without this toleration
+      # the operator can never come back to lift its own taint.
+      #
+      # NoSchedule only: the drain deliberately never evicts the operator's own namespace, so
+      # this covers scheduling a *replacement* pod rather than protecting the running one.
+      #
+      # GPUFirmwareUpdate's taint key comes from spec.updateTaint and has no fixed value, so it
+      # cannot be listed here — add it via operator.tolerations when using that CRD.
+      tolerations:
+      - key: gpurecoveryplan.intel.com/recovery
+        operator: Exists
+        effect: NoSchedule
+      {{- with .Values.operator.tolerations }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
       securityContext:
         # Projects are configured by default to adhere to the "restricted" Pod Security Standards.
         # This ensures that deployments meet the highest security requirements for Kubernetes.

charts/gpu-base-operator/values.yaml

@@ -18,6 +18,13 @@ operator:
     requests:
       cpu: 100m
       memory: 256Mi
+  # Extra tolerations for the operator pod, on top of the built-in one for the recovery taint
+  # it applies itself. Set this when using GPUFirmwareUpdate, whose taint key is whatever
+  # spec.updateTaint says and so cannot be tolerated by default:
+  #   - key: gpu-update-in-progress
+  #     operator: Exists
+  #     effect: NoSchedule
+  tolerations: []
 
 metrics:
   # Serve the operator's own metrics (the controller-runtime and workqueue metrics).

config/manager/manager.yaml

@@ -51,6 +51,22 @@ spec:
       #             operator: In
       #             values:
       #               - linux
+      # The operator must tolerate the taints it applies itself, because it is the only thing
+      # that removes them. A GPURecoveryPlan reset cordons the node it is recovering; if that
+      # reset wedges the host and the operator pod is later evicted, its replacement has to be
+      # schedulable somewhere. On a single-node cluster — or one where every other node is full
+      # or tainted — the only candidate is the cordoned node itself, and without this toleration
+      # the operator can never come back to lift its own taint.
+      #
+      # NoSchedule only: the drain deliberately never evicts the operator's own namespace, so
+      # this covers scheduling a *replacement* pod rather than protecting the running one.
+      #
+      # GPUFirmwareUpdate's taint key comes from spec.updateTaint and has no fixed value, so it
+      # cannot be listed here; add a toleration for it when using that CRD.
+      tolerations:
+      - key: gpurecoveryplan.intel.com/recovery
+        operator: Exists
+        effect: NoSchedule
       securityContext:
         # Projects are configured by default to adhere to the "restricted" Pod Security Standards.
         # This ensures that deployments meet the highest security requirements for Kubernetes.

internal/controller/gpurecoveryplan_const.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package controller
 
+import "time"
+
 const (
 	// recoveryPlanFinalizer is set on every live GPURecoveryPlan. It holds the object in place
 	// while a recovery Job is still running, so the Jobs are cleaned up rather than orphaned.
@@ -64,6 +66,17 @@ const (
 	// maxStateMessageLen caps status.events[].stateMessage.
 	maxStateMessageLen = 200
 
+	// maxPodsBlockingDrainReported caps status.events[].podsBlockingDrain and
+	// status.events[].claimsBlockingReset. A node can hold hundreds of pods, and the field exists
+	// to tell an admin what to go and look at, not to mirror the whole node into the CR.
+	maxPodsBlockingDrainReported = 10
+
+	// recoveryTaintKey is the node taint the operator applies to the node for GPU reset.
+	recoveryTaintKey = "gpurecoveryplan.intel.com/recovery"
+
+	// defaultDrainTimeout mirrors the CRD default for spec.drain.timeoutSeconds.
+	defaultDrainTimeout = 300 * time.Second
+
 	// maxRecoveryNameLen is the hard ceiling on a recovery Job name, and therefore on the
 	// event ID it is built from.
 	//