fix(nvca): resolve against the built-in storage catalog when the ConfigMap is absent

docs/dev/sdd-storage-agnostic-cache-architecture.md

@@ -215,6 +215,7 @@ a `Retain` class must be created for the cache.
 |---|---|
 | Catalog, class, or gate changes after the binding exists | Binding stays authoritative |
 | Class or catalog drifts before the binding exists | Fail before any side effect |
+| Catalog ConfigMap is absent (agent ahead of its chart) | Resolve against the catalog compiled into NVCA, warn, count; the ConfigMap is authoritative once present |
 | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind |
 | Object has foreign or missing ownership | Never adopt or delete it |
 | Reader PV and claim disagree on class | Never binds; prevented by construction |

src/compute-plane-services/nvca/internal/metrics/METRICS.md

@@ -583,6 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1
 | Reason | Description |
 |--------|-------------|
 | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) |
+| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request was resolved against the catalog built into NVCA |
 | `pvc_setup_failed` | Primary PV/PVC setup failures |
 | `pvc_bind_failed` | RO PVC bind failures |
 | `rw_pvc_bind_failed` | RW PVC bind failures |

src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go

@@ -25,7 +25,10 @@ const (
 
 // Failure reason values for model cache metrics
 const (
-	ReasonCacheSpecInvalid   = "cache_spec_invalid"
+	ReasonCacheSpecInvalid = "cache_spec_invalid"
+	// ReasonCatalogMissing: the storage capability catalog ConfigMap is absent,
+	// so the request was created without a durable cache.
+	ReasonCatalogMissing     = "catalog_missing"
 	ReasonPVCSetupFailed     = "pvc_setup_failed"
 	ReasonPVCBindFailed      = "pvc_bind_failed"
 	ReasonRWPVCBindFailed    = "rw_pvc_bind_failed"
@@ -49,6 +52,7 @@ const (
 // Used for pre-initializing Prometheus counters to zero.
 var AllFailureReasons = []string{
 	ReasonCacheSpecInvalid,
+	ReasonCatalogMissing,
 	ReasonPVCSetupFailed,
 	ReasonPVCBindFailed,
 	ReasonRWPVCBindFailed,

src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go

@@ -21,6 +21,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/logging"
+	nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics"
+	modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes"
+	"github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core"
+	"github.com/sirupsen/logrus"
 
 	"github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common"
 
@@ -75,6 +80,9 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection(
 		var err error
 		resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset(
 			ctx, c.clients.K8s, c.systemNamespace, workflow)
+		if err == nil && resolved.CatalogBuiltin {
+			c.noteBuiltinCatalog(ctx, req, workflow)
+		}
 		switch {
 		case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound):
 			if workflow == nvcastorage.ModelCacheWorkflowHelm {
@@ -120,3 +128,20 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection(
 	req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload
 	return nil
 }
+
+// noteBuiltinCatalog records that a request was resolved against the catalog
+// compiled into NVCA because the nvcf-storage-capabilities ConfigMap is absent.
+// The selection is the one a converged install would make; the warning and the
+// counter exist so the chart rollout gap is visible.
+func (c *BackendK8sCache) noteBuiltinCatalog(
+	ctx context.Context, req *nvcav2beta1.ICMSRequest, workflow nvcastorage.ModelCacheWorkflow,
+) {
+	logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithFields(logrus.Fields{
+		"configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName,
+		"workflow":  workflow,
+	}).Warn("storage capability catalog ConfigMap is missing, using the catalog built into NVCA")
+	if m := nvcametrics.FromContext(ctx); m != nil {
+		// The backend label is empty: the selection has not chosen a backend yet.
+		m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "")
+	}
+}

src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go

@@ -249,6 +249,36 @@ func TestPersistModelCacheStorageSelection(t *testing.T) {
 			wantWorkflow: nvcastorage.ModelCacheWorkflowRegular,
 			wantMode:     nvcastorage.ModelCacheSelectionNone,
 		},
+		{
+			name: "missing catalog ConfigMap resolves against the built-in catalog",
+			objects: func() []runtime.Object {
+				return []runtime.Object{selectionStorageClass()}
+			},
+			flags:             []*featureflag.FeatureFlag{featureflag.CachingSupport},
+			wantWorkflow:      nvcastorage.ModelCacheWorkflowRegular,
+			wantMode:          nvcastorage.ModelCacheSelectionDurable,
+			wantTransition:    nvcastorage.ModelCacheTransitionROXReadOnly,
+			wantResolvedState: true,
+			wantProvider:      nvcastorage.ModelCacheProviderNVMesh,
+			wantProvisioner:   nvcastorage.NVMeshStorageClassProvisioner,
+		},
+		{
+			name: "missing catalog ConfigMap resolves Helm against the built-in catalog",
+			helm: true,
+			objects: func() []runtime.Object {
+				return []runtime.Object{selectionStorageClass()}
+			},
+			flags: []*featureflag.FeatureFlag{
+				featureflag.CachingSupport,
+				featureflag.HelmModelCaching,
+			},
+			wantWorkflow:      nvcastorage.ModelCacheWorkflowHelm,
+			wantMode:          nvcastorage.ModelCacheSelectionDurable,
+			wantTransition:    nvcastorage.ModelCacheTransitionROXReadOnly,
+			wantResolvedState: true,
+			wantProvider:      nvcastorage.ModelCacheProviderNVMesh,
+			wantProvisioner:   nvcastorage.NVMeshStorageClassProvisioner,
+		},
 		{
 			name: "missing StorageClass falls Helm back to ephemeral",
 			helm: true,
@@ -326,3 +356,41 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test
 	require.NoError(t, listErr)
 	assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call")
 }
+
+// A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap
+// did not exist, every creation message failed before the ICMSRequest was
+// created, and the queue retried it forever. The agent must resolve against
+// the catalog it was built with instead.
+func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *testing.T) {
+	objects := []runtime.Object{selectionStorageClass()}
+	cache, _ := selectionBackendCache(objects, featureflag.CachingSupport)
+	cache.clients = mockKubeClients(objects...)
+	cache.requestsNamespace = RequestsNamespace
+
+	msg := function.CreationQueueMessage{
+		CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{
+			RequestID: "missing-catalog-request",
+			NCAID:     "test-nca",
+			Action:    common.FunctionCreationAction,
+		},
+		Details: function.Details{
+			FunctionID:        "function-id",
+			FunctionVersionID: "function-version-id",
+		},
+		LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification,
+	}
+
+	created, err := cache.CreateICMSCreationMessageRequest(
+		newTestContext(), msg, "receipt", "message-id", "queue")
+	require.NoError(t, err, "a missing catalog must not fail the creation message")
+	require.NotNil(t, created)
+
+	requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace).
+		List(t.Context(), metav1.ListOptions{})
+	require.NoError(t, listErr)
+	require.Len(t, requests.Items, 1)
+	selection := parseRequestStorageSelection(t, &requests.Items[0])
+	assert.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode)
+	assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider)
+	assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName)
+}

src/compute-plane-services/nvca/pkg/storage/BUILD.bazel

@@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "storage",
+    embedsrcs = ["nvcf-storage-capabilities-v1alpha1.yaml"],
     srcs = [
         "modelcache_selection.go",
         "cachebackend.go",
@@ -20,6 +21,7 @@ go_library(
         "sharedstorage.go",
         "smbcsidriver.go",
         "storage_capabilities.go",
+        "storage_capabilities_builtin.go",
         "storage_request_api.go",
         "storagerequest.go",
         "translate_workload.go",

src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NVCA owns this catalog and installs it with the NVCA chart. Each entry is
+# named by exact CSI provisioner and records the PVC access modes qualified end to end in an
+# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs
+# from these modes.
+#
+#   ReadWriteMany              one shared claim; readers mount it read-only
+#   ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own
+#
+# An access mode a driver merely accepts is not a qualification. A claim that
+# binds is not either. An empty accessModes list means nothing is qualified
+# yet, so caching stays off for that driver, and a provisioner absent from this
+# file is unsupported. Enabling a backend is an edit here, backed by a
+# qualification run.
+apiVersion: storage.nvcf.nvidia.com/v1alpha1
+kind: StorageCapabilityCatalog
+drivers:
+  - name: nvmesh-csi.excelero.com
+    provider: nvmesh
+    # Encrypted caches are qualified on NVMesh, through a derived StorageClass
+    # and a per-sharing-domain Secret.
+    encryptionSupported: true
+    accessModes:
+      - ReadWriteOnce
+      - ReadOnlyMany
+    # The reader PV is XFS on the same filesystem as the writer, so it needs
+    # nouuid and norecovery or the mount fails outright.
+    readerMountOptions:
+      - ro
+      - norecovery
+      - nouuid
+  - name: csi.weka.io
+    provider: weka
+    # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache
+    # workflow was not, so nothing is qualified yet.
+    accessModes: []
+    readerMountOptions: []
+  - name: fss.csi.oraclecloud.com
+    provider: ociFss
+    # A ReadWriteMany claim was tested; its readers used read-only Pod mounts,
+    # which is not evidence for a ReadOnlyMany claim. No cache workflow was
+    # qualified.
+    accessModes: []
+    readerMountOptions: []
+  - name: lustre.csi.oraclecloud.com
+    provider: ociLustre
+    # No PVC access mode has been qualified in an NVCF cache workflow.
+    accessModes: []
+    readerMountOptions: []

src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go

@@ -83,6 +83,9 @@ const (
 // live nvcf-sc object and the public capability catalog. It deliberately does
 // not infer behavior from a provider name or access mode.
 type ModelCacheStorageSelection struct {
+	// CatalogBuiltin is set when the ConfigMap was absent and the selection was
+	// resolved against the catalog compiled into NVCA.
+	CatalogBuiltin       bool
 	EncryptionSupported  bool
 	StorageClassName     string
 	StorageClassUID      types.UID
@@ -214,6 +217,10 @@ func loadStorageCapabilityCatalog(
 
 	cm := &corev1.ConfigMap{}
 	if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil {
+		if apierrors.IsNotFound(err) {
+			catalog, _, berr := builtinStorageCapabilityCatalog()
+			return catalog, berr
+		}
 		return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w",
 			namespace, StorageCapabilityConfigMapName, err)
 	}
@@ -227,20 +234,33 @@ func loadStorageCapabilityCatalog(
 	return parseStorageCapabilityCatalog(raw)
 }
 
+// loadStorageCapabilityCatalogSnapshot returns the catalog, its payload
+// digest, and whether the compiled-in copy was used because the ConfigMap is
+// absent. A ConfigMap that exists but is empty or malformed is an error.
 func loadStorageCapabilityCatalogSnapshot(
 	ctx context.Context,
 	c client.Client,
 	namespace string,
-) (*storageCapabilityCatalog, string, error) {
+) (*storageCapabilityCatalog, string, bool, error) {
 	if namespace == "" {
-		return nil, "", fmt.Errorf("storage capability ConfigMap namespace is empty")
+		return nil, "", false, fmt.Errorf("storage capability ConfigMap namespace is empty")
 	}
 
 	cm := &corev1.ConfigMap{}
 	if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil {
-		return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w",
+		if apierrors.IsNotFound(err) {
+			catalog, digest, berr := builtinStorageCapabilityCatalog()
+			return catalog, digest, true, berr
+		}
+		return nil, "", false, fmt.Errorf("get storage capability ConfigMap %s/%s: %w",
 			namespace, StorageCapabilityConfigMapName, err)
 	}
+	catalog, digest, err := parseStorageCapabilityConfigMap(cm, namespace)
+	return catalog, digest, false, err
+}
+
+// parseStorageCapabilityConfigMap parses a present catalog ConfigMap.
+func parseStorageCapabilityConfigMap(cm *corev1.ConfigMap, namespace string) (*storageCapabilityCatalog, string, error) {
 	raw, ok := cm.Data[StorageCapabilityConfigMapKey]
 	if !ok || raw == "" {
 		return nil, "", fmt.Errorf("storage capability ConfigMap %s/%s has no %q data",
@@ -286,11 +306,16 @@ func ResolveModelCacheStorage(
 		return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err)
 	}
 
-	catalog, catalogDigest, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace)
+	catalog, catalogDigest, builtin, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace)
+	if err != nil {
+		return nil, err
+	}
+	selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow)
 	if err != nil {
 		return nil, err
 	}
-	return selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow)
+	selection.CatalogBuiltin = builtin
+	return selection, nil
 }
 
 // ResolveModelCacheStorageWithClientset provides the same decision to the
@@ -314,22 +339,34 @@ func ResolveModelCacheStorageWithClientset(
 		}
 		return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err)
 	}
+	var (
+		catalog       *storageCapabilityCatalog
+		catalogDigest string
+		builtin       bool
+	)
 	cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get(
 		ctx, StorageCapabilityConfigMapName, metav1.GetOptions{})
-	if err != nil {
+	switch {
+	case apierrors.IsNotFound(err):
+		// The chart that ships this agent installs the ConfigMap. Until it has,
+		// resolve against the same catalog it would install.
+		catalog, catalogDigest, err = builtinStorageCapabilityCatalog()
+		builtin = true
+	case err != nil:
 		return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w",
 			catalogNamespace, StorageCapabilityConfigMapName, err)
+	default:
+		catalog, catalogDigest, err = parseStorageCapabilityConfigMap(cm, catalogNamespace)
 	}
-	raw, ok := cm.Data[StorageCapabilityConfigMapKey]
-	if !ok || raw == "" {
-		return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data",
-			catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey)
+	if err != nil {
+		return nil, err
 	}
-	catalog, err := parseStorageCapabilityCatalog(raw)
+	selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow)
 	if err != nil {
 		return nil, err
 	}
-	return selectModelCacheStorageFromObjects(sc, catalog, digestCatalogPayload(raw), workflow)
+	selection.CatalogBuiltin = builtin
+	return selection, nil
 }
 
 func selectModelCacheStorageFromObjects(

src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go

@@ -0,0 +1,42 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	_ "embed"
+	"fmt"
+)
+
+// builtinStorageCapabilityCatalogYAML is the catalog the NVCA chart ships as
+// the nvcf-storage-capabilities ConfigMap, compiled into the agent so an agent
+// whose chart has not converged resolves against the same defaults a fresh
+// install would. The ConfigMap stays authoritative when it exists; this copy is
+// read only when it is absent. TestBuiltinCatalogMatchesChart keeps the two
+// identical.
+//
+//go:embed nvcf-storage-capabilities-v1alpha1.yaml
+var builtinStorageCapabilityCatalogYAML string
+
+// builtinStorageCapabilityCatalog parses the compiled-in catalog.
+func builtinStorageCapabilityCatalog() (*storageCapabilityCatalog, string, error) {
+	catalog, err := parseStorageCapabilityCatalog(builtinStorageCapabilityCatalogYAML)
+	if err != nil {
+		return nil, "", fmt.Errorf("built-in storage capability catalog: %w", err)
+	}
+	return catalog, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), nil
+}