Recovery: approvals and reset job creation

.trivyignore.yaml

@@ -6,6 +6,7 @@ misconfigurations:
       NOTE: FW update could also use monitoring/admin access from GPU DP or DRA, and then drop privileged access.
     paths:
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0005
     statement: "XPUM requires SYS_ADMIN for some GPU metrics"
@@ -19,11 +20,13 @@ misconfigurations:
       - dp/dp.yaml
       - xpum/xpum.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0017
     statement: "DP levelzero & fwupdate requires privileged access / allowPrivilegeEscalation"
     paths:
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0022
     statement: "XPUM requires SYS_ADMIN for some GPU metrics"
@@ -37,6 +40,7 @@ misconfigurations:
       - dra/daemonset.yaml
       - xpum/xpum.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0025
     statement: "container_device_plugin_t is a valid SELinux profile"
@@ -68,6 +72,7 @@ misconfigurations:
       - dp/dp.yaml
       - xpum/xpum.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0118
     statement: "DP is not deployed to default NS but to where-ever the operator is deployed"
@@ -76,6 +81,7 @@ misconfigurations:
       - dra/daemonset.yaml
       - xpum/xpum.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0125
     statement: "Dockerhub.io is ok"
@@ -84,12 +90,14 @@ misconfigurations:
       - xpum/xpum.yaml
       - dra/daemonset.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-KSV-0121
     statement: "/sys is required for DRA"
     paths:
       - dra/daemonset.yaml
       - xpum/xpum-fwupdate-job.yaml
+      - xpum/xpum-reset-job.yaml
 
   - id: AVD-DS-0002
     statement: ""

charts/gpu-base-operator/templates/role.yaml

@@ -73,6 +73,7 @@ rules:
   resources:
   - clusterpolicies/finalizers
   - gpufirmwareupdates/finalizers
+  - gpurecoveryplans/finalizers
   verbs:
   - update
 - apiGroups:
@@ -92,6 +93,8 @@ rules:
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - kmm.sigs.x-k8s.io

config/deployments/deployments.go

@@ -169,6 +169,13 @@ func XpuManagerFWUpdateJob() *batch.Job {
 	return getJob(xpumFWUpdateJob).DeepCopy()
 }
 
+//go:embed xpum/xpum-reset-job.yaml
+var xpumResetJob []byte
+
+func XpuManagerResetJob() *batch.Job {
+	return getJob(xpumResetJob).DeepCopy()
+}
+
 // generic functions
 
 func getDaemonset(content []byte) *apps.DaemonSet {

config/deployments/deployments_test.go

@@ -149,6 +149,85 @@ func TestXpuFwUpdateJob(t *testing.T) {
 	}
 }
 
+func TestXpuResetJob(t *testing.T) {
+	job := XpuManagerResetJob()
+	if job == nil {
+		t.Error("XpuManagerResetJob returned nil")
+	}
+}
+
+// The operator only ever overwrites the resetter container's image and args, so everything else
+// the reset needs has to be right in the template: the /sys mount the PCIe reset writes through,
+// and a restart policy that does not re-run a reset the Job controller already gave up on.
+func TestXpuResetJob_ResetterContainer(t *testing.T) {
+	job := XpuManagerResetJob()
+
+	c := findContainer(job.Spec.Template.Spec.Containers, "resetter")
+	if c == nil {
+		t.Fatal("resetter container not found")
+	}
+
+	found := false
+	for _, m := range c.VolumeMounts {
+		if m.MountPath == "/sys" {
+			found = true
+			if m.ReadOnly {
+				t.Error("/sys must be mounted writable: the reset is issued by writing to sysfs")
+			}
+		}
+	}
+	if !found {
+		t.Error("resetter has no /sys mount")
+	}
+
+	if job.Spec.Template.Spec.RestartPolicy != core.RestartPolicyNever {
+		t.Errorf("restartPolicy: got %v, want Never", job.Spec.Template.Spec.RestartPolicy)
+	}
+
+	if job.Spec.Template.Spec.AutomountServiceAccountToken == nil {
+		t.Fatal("automountServiceAccountToken must be set (non-nil)")
+	}
+	if *job.Spec.Template.Spec.AutomountServiceAccountToken != false {
+		t.Error("automountServiceAccountToken must be false")
+	}
+}
+
+func TestXpuResetJob_ResetterSecurityContext(t *testing.T) {
+	job := XpuManagerResetJob()
+
+	c := findContainer(job.Spec.Template.Spec.Containers, "resetter")
+	if c == nil {
+		t.Fatal("resetter container not found")
+	}
+	if c.SecurityContext == nil {
+		t.Fatal("SecurityContext must be set on resetter")
+	}
+	// Privileged is deliberate here, unlike everywhere else in this file: resetting a GPU means
+	// writing to the device's PCIe config space through sysfs.
+	if c.SecurityContext.Privileged == nil || !*c.SecurityContext.Privileged {
+		t.Error("resetter must be privileged to issue a PCIe reset")
+	}
+	if c.SecurityContext.SeccompProfile == nil {
+		t.Fatal("SeccompProfile must be set on resetter")
+	}
+	if c.SecurityContext.SeccompProfile.Type != core.SeccompProfileTypeRuntimeDefault {
+		t.Errorf("SeccompProfile.Type: got %v, want RuntimeDefault on resetter", c.SecurityContext.SeccompProfile.Type)
+	}
+	if c.SecurityContext.Capabilities == nil {
+		t.Fatal("Capabilities must be set on resetter")
+	}
+	found := false
+	for _, cap := range c.SecurityContext.Capabilities.Drop {
+		if cap == allCaps {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Error("capabilities.drop must contain ALL on resetter")
+	}
+}
+
 func TestOTelConfig(t *testing.T) {
 	cfg := XpuManagerOTelConfig()
 	if cfg == nil {

config/deployments/xpum/xpum-reset-job.yaml

@@ -0,0 +1,69 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+# @file xpum-reset-job.yaml
+#
+# Template for GPU hardware reset recovery Jobs.
+# The operator fills in: spec.template.spec.nodeName, containers[resetter].image,
+# and containers[resetter].args with the appropriate xpu-smi reset command.
+#
+# Supported commands (set by the operator from the event's recoveryType):
+#   sbr:   xpu-smi config -d <BDF> --reset
+#   slot:  xpu-smi config -d <BDF> --coldreset
+#   amc:   xpu-smi amc --gpureset -d <BDF> -y
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: intel-reset-job
+spec:
+  podFailurePolicy:
+    rules:
+    - action: FailJob
+      onExitCodes:
+        containerName: resetter
+        operator: NotIn
+        values: [0]
+    - action: FailJob
+      onPodConditions:
+      - type: ConfigIssue
+  activeDeadlineSeconds: 300
+  template:
+    spec:
+      volumes:
+      - name: host-sys
+        hostPath:
+          path: /sys
+          type: Directory
+      automountServiceAccountToken: false
+      containers:
+      - name: resetter
+        image: intel/gpu-fwupdater-mock:devel # replaced by the operator (spec.xpuSmi.image)
+        imagePullPolicy: IfNotPresent
+        command: [ "/usr/local/bin/xpu-smi" ]
+        args:
+          - "config"
+          - "-d"
+          - "0000:00:00.0"  # replaced by the operator
+          - "--reset"       # replaced by the operator
+        resources:
+          requests:
+            cpu: 50m
+            memory: 32Mi
+          limits:
+            cpu: 100m
+            memory: 64Mi
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: true
+          runAsUser: 0
+          privileged: true
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop: [ "ALL" ]
+        volumeMounts:
+        - name: host-sys
+          mountPath: /sys
+          readOnly: false
+      imagePullSecrets:
+      restartPolicy: Never

config/rbac/role.yaml

@@ -73,6 +73,7 @@ rules:
   resources:
   - clusterpolicies/finalizers
   - gpufirmwareupdates/finalizers
+  - gpurecoveryplans/finalizers
   verbs:
   - update
 - apiGroups:
@@ -92,6 +93,8 @@ rules:
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - kmm.sigs.x-k8s.io

internal/controller/gpurecoveryplan_const.go

@@ -17,6 +17,19 @@ limitations under the License.
 package controller
 
 const (
+	// recoveryPlanFinalizer is set on every live GPURecoveryPlan. It holds the object in place
+	// while a recovery Job is still running, so the Jobs are cleaned up rather than orphaned.
+	recoveryPlanFinalizer = "gpurecoveryplan.intel.com/finalizer"
+
+	// recoveryJobLabelPlan is the label key placed on every recovery Job; its value is the name
+	// of the owning GPURecoveryPlan. It is what finds a plan's Jobs when status.events no longer
+	// names them.
+	recoveryJobLabelPlan = "gpurecoveryplan.intel.com/plan"
+
+	// recoveryJobLabelEvent is the label key placed on every recovery Job; its value is the ID of
+	// the RecoveryEvent the Job was created for.
+	recoveryJobLabelEvent = "gpurecoveryplan.intel.com/event"
+
 	// DRA's device attributes
 	// deviceAttrDeviceID is the ResourceSlice device attribute name for the PCI device ID.
 	deviceAttrDeviceID = "pciId"