fix(nvca): storage capability catalog reaches the agent, with Weka and OCI FSS enabled

deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml

@@ -45,16 +45,23 @@ drivers:
       - nouuid
   - name: csi.weka.io
     provider: weka
-    # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache
-    # workflow was not, so nothing is qualified yet.
-    accessModes: []
+    # Weka is a shared filesystem: one ReadWriteMany claim per cache handle,
+    # populated once and mounted read-only by every reader, with no derived
+    # reader PV, so readerMountOptions stays empty. Enabled so the cache
+    # workflows can be exercised; record the qualification run in the pull
+    # request that flips this entry.
+    accessModes:
+      - ReadWriteMany
     readerMountOptions: []
   - name: fss.csi.oraclecloud.com
     provider: ociFss
-    # A ReadWriteMany claim was tested; its readers used read-only Pod mounts,
-    # which is not evidence for a ReadOnlyMany claim. No cache workflow was
-    # qualified.
-    accessModes: []
+    # OCI File Storage is NFS: one ReadWriteMany claim per cache handle,
+    # populated once and mounted read-only by every reader, with no derived
+    # reader PV, so readerMountOptions stays empty. Enabled so the cache
+    # workflows can be exercised; record the qualification run in the pull
+    # request that flips this entry.
+    accessModes:
+      - ReadWriteMany
     readerMountOptions: []
   - name: lustre.csi.oraclecloud.com
     provider: ociLustre

docs/dev/sdd-storage-agnostic-cache-architecture.md

@@ -50,8 +50,12 @@ Four pieces:
 
 ## Capability catalog
 
-Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities`, validated
-by a packaged JSON Schema and by the Go loader with the same rules.
+Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities` in the
+operator's namespace; the operator mirrors it into the agent's namespace, where
+the agent and the storage controller read it, and re-mirrors on every edit. A
+copy of the shipped catalog is compiled into NVCA and used only while the
+ConfigMap is absent. Validated by a packaged JSON Schema and by the Go loader
+with the same rules.
 
 ```yaml
 drivers:
@@ -62,7 +66,7 @@ drivers:
     encryptionSupported: true
   - name: csi.weka.io
     provider: weka
-    accessModes: []
+    accessModes: [ReadWriteMany]
     readerMountOptions: []
 ```
 
@@ -215,6 +219,7 @@ a `Retain` class must be created for the cache.
 |---|---|
 | Catalog, class, or gate changes after the binding exists | Binding stays authoritative |
 | Class or catalog drifts before the binding exists | Fail before any side effect |
+| Catalog ConfigMap is absent (agent ahead of its chart) | Resolve against the catalog compiled into NVCA, warn, count; the ConfigMap is authoritative once present |
 | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind |
 | Object has foreign or missing ownership | Never adopt or delete it |
 | Reader PV and claim disagree on class | Never binds; prevented by construction |

src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml

@@ -45,16 +45,23 @@ drivers:
       - nouuid
   - name: csi.weka.io
     provider: weka
-    # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache
-    # workflow was not, so nothing is qualified yet.
-    accessModes: []
+    # Weka is a shared filesystem: one ReadWriteMany claim per cache handle,
+    # populated once and mounted read-only by every reader, with no derived
+    # reader PV, so readerMountOptions stays empty. Enabled so the cache
+    # workflows can be exercised; record the qualification run in the pull
+    # request that flips this entry.
+    accessModes:
+      - ReadWriteMany
     readerMountOptions: []
   - name: fss.csi.oraclecloud.com
     provider: ociFss
-    # A ReadWriteMany claim was tested; its readers used read-only Pod mounts,
-    # which is not evidence for a ReadOnlyMany claim. No cache workflow was
-    # qualified.
-    accessModes: []
+    # OCI File Storage is NFS: one ReadWriteMany claim per cache handle,
+    # populated once and mounted read-only by every reader, with no derived
+    # reader PV, so readerMountOptions stays empty. Enabled so the cache
+    # workflows can be exercised; record the qualification run in the pull
+    # request that flips this entry.
+    accessModes:
+      - ReadWriteMany
     readerMountOptions: []
   - name: lustre.csi.oraclecloud.com
     provider: ociLustre

src/compute-plane-services/nvca/internal/metrics/METRICS.md

@@ -583,6 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1
 | Reason | Description |
 |--------|-------------|
 | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) |
+| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request was resolved against the catalog built into NVCA |
 | `pvc_setup_failed` | Primary PV/PVC setup failures |
 | `pvc_bind_failed` | RO PVC bind failures |
 | `rw_pvc_bind_failed` | RW PVC bind failures |

src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go

@@ -25,7 +25,10 @@ const (
 
 // Failure reason values for model cache metrics
 const (
-	ReasonCacheSpecInvalid   = "cache_spec_invalid"
+	ReasonCacheSpecInvalid = "cache_spec_invalid"
+	// ReasonCatalogMissing: the storage capability catalog ConfigMap is absent,
+	// so the request was created without a durable cache.
+	ReasonCatalogMissing     = "catalog_missing"
 	ReasonPVCSetupFailed     = "pvc_setup_failed"
 	ReasonPVCBindFailed      = "pvc_bind_failed"
 	ReasonRWPVCBindFailed    = "rw_pvc_bind_failed"
@@ -49,6 +52,7 @@ const (
 // Used for pre-initializing Prometheus counters to zero.
 var AllFailureReasons = []string{
 	ReasonCacheSpecInvalid,
+	ReasonCatalogMissing,
 	ReasonPVCSetupFailed,
 	ReasonPVCBindFailed,
 	ReasonRWPVCBindFailed,

src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go

@@ -111,7 +111,11 @@ func validatePersistedModelCacheStorageRequest(
 		existing.Spec.Type != nvcav2beta1.ModelCacheRequest {
 		return conflict("name or type does not match")
 	}
-	if backend != nvcastorage.HelmCacheBackendNVMesh {
+	switch backend {
+	case nvcastorage.HelmCacheBackendNVMesh, nvcastorage.HelmCacheBackendSharedFS, nvcastorage.HelmCacheBackendSamba:
+		// The backends that create a model cache StorageRequest; keep in step
+		// with makeStorageRequests.
+	default:
 		return conflict(fmt.Sprintf("backend %q does not create a StorageRequest", backend))
 	}
 	if existing.Spec.ModelCache == nil {

src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go

@@ -99,6 +99,40 @@ func TestSelectHelmCacheBackend(t *testing.T) {
 		assert.Equal(t, nvcastorage.HelmCacheBackendNVMesh, got)
 	})
 
+	t.Run("matching persisted shared-filesystem StorageRequest is adopted", func(t *testing.T) {
+		// Weka and OCI FSS persist a ReadWriteMany selection that routes to the
+		// shared-filesystem backend. The StorageRequest it creates must validate
+		// on the next reconcile instead of being rejected as a backend that
+		// creates none.
+		request := requestWithRWXModelCacheSelection(t)
+		raw := request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]
+		existing := &nvcav2beta1.StorageRequest{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      nvcav2beta1.ModelCacheRequest.Name(),
+				Namespace: instanceNamespace,
+				Annotations: map[string]string{
+					nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw,
+					nvcastorage.ICMSRequestUIDAnnotationKey:             string(request.UID),
+				},
+			},
+			Spec: nvcav2beta1.StorageRequestSpec{
+				Type:             nvcav2beta1.ModelCacheRequest,
+				RequestName:      request.Name,
+				RequestNamespace: request.Namespace,
+				ModelCache: &nvcav2beta1.ModelCacheSpec{
+					Backend:     string(nvcastorage.HelmCacheBackendSharedFS),
+					CacheHandle: helmModelCacheHandle(request),
+				},
+			},
+		}
+		r := newModelCacheSelectionReconciler(t, existing)
+
+		got, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace)
+
+		require.NoError(t, err)
+		assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, got)
+	})
+
 	t.Run("ephemeral selection rejects a stale durable StorageRequest", func(t *testing.T) {
 		existing := &nvcav2beta1.StorageRequest{
 			ObjectMeta: metav1.ObjectMeta{
@@ -368,3 +402,27 @@ func (c *getErrorClient) Get(
 ) error {
 	return c.err
 }
+
+// requestWithRWXModelCacheSelection is a Helm request whose persisted selection
+// is the ReadWriteMany shape a shared filesystem such as Weka resolves to.
+func requestWithRWXModelCacheSelection(t *testing.T) *nvcav2beta1.ICMSRequest {
+	t.Helper()
+	resolved := &nvcastorage.ModelCacheStorageSelection{
+		StorageClassName:    nvcastorage.DefaultModelCacheStorageClassName,
+		StorageClassUID:     types.UID("storage-class-uid"),
+		StorageClassDigest:  "storage-class-digest",
+		ProfileDigest:       "catalog-digest",
+		Provider:            "weka",
+		Provisioner:         "csi.weka.io",
+		Transition:          nvcastorage.ModelCacheTransitionRWXReadOnly,
+		RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany},
+	}
+	selection, err := nvcastorage.NewPersistedModelCacheStorageSelection(
+		nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable, resolved)
+	require.NoError(t, err)
+	payload, err := selection.Marshal()
+	require.NoError(t, err)
+	request := requestWithModelCacheSelection(t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionEphemeral)
+	request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload
+	return request
+}

src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go

@@ -21,6 +21,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/logging"
+	nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics"
+	modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes"
+	"github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core"
+	"github.com/sirupsen/logrus"
 
 	"github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common"
 
@@ -75,6 +80,9 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection(
 		var err error
 		resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset(
 			ctx, c.clients.K8s, c.systemNamespace, workflow)
+		if err == nil && resolved.CatalogBuiltin {
+			c.noteBuiltinCatalog(ctx, req, workflow)
+		}
 		switch {
 		case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound):
 			if workflow == nvcastorage.ModelCacheWorkflowHelm {
@@ -86,10 +94,12 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection(
 			if workflow == nvcastorage.ModelCacheWorkflowHelm {
 				mode = nvcastorage.ModelCacheSelectionEphemeral
 			}
-		case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly:
-			mode = nvcastorage.ModelCacheSelectionDurable
-		case resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly &&
-			workflow == nvcastorage.ModelCacheWorkflowRegular:
+		case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly,
+			resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly:
+			// Both shapes are durable for both workflows. Helm routes the
+			// ReadWriteMany shape to the shared-filesystem backend through
+			// HelmCacheBackendFromSelection; the regular workflow serves it from
+			// one shared claim per cache handle.
 			mode = nvcastorage.ModelCacheSelectionDurable
 		default:
 			return fmt.Errorf("unsupported model cache transition %q", resolved.Transition)
@@ -120,3 +130,20 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection(
 	req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload
 	return nil
 }
+
+// noteBuiltinCatalog records that a request was resolved against the catalog
+// compiled into NVCA because the nvcf-storage-capabilities ConfigMap is absent.
+// The selection is the one a converged install would make; the warning and the
+// counter exist so the chart rollout gap is visible.
+func (c *BackendK8sCache) noteBuiltinCatalog(
+	ctx context.Context, req *nvcav2beta1.ICMSRequest, workflow nvcastorage.ModelCacheWorkflow,
+) {
+	logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithFields(logrus.Fields{
+		"configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName,
+		"workflow":  workflow,
+	}).Warn("storage capability catalog ConfigMap is missing, using the catalog built into NVCA")
+	if m := nvcametrics.FromContext(ctx); m != nil {
+		// The backend label is empty: the selection has not chosen a backend yet.
+		m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "")
+	}
+}

src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go

@@ -210,6 +210,29 @@ func TestPersistModelCacheStorageSelection(t *testing.T) {
 			wantProvider:      "weka",
 			wantProvisioner:   "csi.weka.io",
 		},
+		{
+			// Weka and OCI FSS enable this shape; a Helm request on them must
+			// persist a durable selection that HelmCacheBackendFromSelection routes
+			// to the shared-filesystem backend, not fail the creation message.
+			name: "Helm durable provider-neutral RWX",
+			helm: true,
+			objects: func() []runtime.Object {
+				return []runtime.Object{
+					selectionStorageClassForProvisioner("csi.weka.io"),
+					selectionCatalogConfigMap(selectionCatalogRWXReadOnly),
+				}
+			},
+			flags: []*featureflag.FeatureFlag{
+				featureflag.CachingSupport,
+				featureflag.HelmModelCaching,
+			},
+			wantWorkflow:      nvcastorage.ModelCacheWorkflowHelm,
+			wantMode:          nvcastorage.ModelCacheSelectionDurable,
+			wantTransition:    nvcastorage.ModelCacheTransitionRWXReadOnly,
+			wantResolvedState: true,
+			wantProvider:      "weka",
+			wantProvisioner:   "csi.weka.io",
+		},
 		{
 			name: "disabled regular cache persists none",
 			objects: func() []runtime.Object {
@@ -249,6 +272,36 @@ func TestPersistModelCacheStorageSelection(t *testing.T) {
 			wantWorkflow: nvcastorage.ModelCacheWorkflowRegular,
 			wantMode:     nvcastorage.ModelCacheSelectionNone,
 		},
+		{
+			name: "missing catalog ConfigMap resolves against the built-in catalog",
+			objects: func() []runtime.Object {
+				return []runtime.Object{selectionStorageClass()}
+			},
+			flags:             []*featureflag.FeatureFlag{featureflag.CachingSupport},
+			wantWorkflow:      nvcastorage.ModelCacheWorkflowRegular,
+			wantMode:          nvcastorage.ModelCacheSelectionDurable,
+			wantTransition:    nvcastorage.ModelCacheTransitionROXReadOnly,
+			wantResolvedState: true,
+			wantProvider:      nvcastorage.ModelCacheProviderNVMesh,
+			wantProvisioner:   nvcastorage.NVMeshStorageClassProvisioner,
+		},
+		{
+			name: "missing catalog ConfigMap resolves Helm against the built-in catalog",
+			helm: true,
+			objects: func() []runtime.Object {
+				return []runtime.Object{selectionStorageClass()}
+			},
+			flags: []*featureflag.FeatureFlag{
+				featureflag.CachingSupport,
+				featureflag.HelmModelCaching,
+			},
+			wantWorkflow:      nvcastorage.ModelCacheWorkflowHelm,
+			wantMode:          nvcastorage.ModelCacheSelectionDurable,
+			wantTransition:    nvcastorage.ModelCacheTransitionROXReadOnly,
+			wantResolvedState: true,
+			wantProvider:      nvcastorage.ModelCacheProviderNVMesh,
+			wantProvisioner:   nvcastorage.NVMeshStorageClassProvisioner,
+		},
 		{
 			name: "missing StorageClass falls Helm back to ephemeral",
 			helm: true,
@@ -326,3 +379,56 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test
 	require.NoError(t, listErr)
 	assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call")
 }
+
+// A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap
+// did not exist, every creation message failed before the ICMSRequest was
+// created, and the queue retried it forever. The agent must resolve against
+// the catalog it was built with instead.
+func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *testing.T) {
+	objects := []runtime.Object{selectionStorageClass()}
+	cache, _ := selectionBackendCache(objects, featureflag.CachingSupport)
+	cache.clients = mockKubeClients(objects...)
+	cache.requestsNamespace = RequestsNamespace
+
+	msg := function.CreationQueueMessage{
+		CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{
+			RequestID: "missing-catalog-request",
+			NCAID:     "test-nca",
+			Action:    common.FunctionCreationAction,
+		},
+		Details: function.Details{
+			FunctionID:        "function-id",
+			FunctionVersionID: "function-version-id",
+		},
+		LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification,
+	}
+
+	created, err := cache.CreateICMSCreationMessageRequest(
+		newTestContext(), msg, "receipt", "message-id", "queue")
+	require.NoError(t, err, "a missing catalog must not fail the creation message")
+	require.NotNil(t, created)
+
+	requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace).
+		List(t.Context(), metav1.ListOptions{})
+	require.NoError(t, listErr)
+	require.Len(t, requests.Items, 1)
+	selection := parseRequestStorageSelection(t, &requests.Items[0])
+	assert.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode)
+	assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider)
+	assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName)
+}
+
+// The persisted Helm selection on a ReadWriteMany provider must route to the
+// shared-filesystem backend, the end-to-end contract this fix restores.
+func TestHelmRWXSelectionRoutesToSharedFS(t *testing.T) {
+	cache, _ := selectionBackendCache([]runtime.Object{
+		selectionStorageClassForProvisioner("csi.weka.io"),
+		selectionCatalogConfigMap(selectionCatalogRWXReadOnly),
+	}, featureflag.CachingSupport, featureflag.HelmModelCaching)
+	req := selectionRequest(true)
+	require.NoError(t, cache.persistModelCacheStorageSelection(t.Context(), req))
+	selection := parseRequestStorageSelection(t, req)
+	backend, err := nvcastorage.HelmCacheBackendFromSelection(selection)
+	require.NoError(t, err)
+	assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, backend)
+}