From aaec46b85ab5ce44cee8b7a2efbe1b5fba64871e Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Thu, 17 Sep 2026 10:43:03 -0700 Subject: [PATCH 1/5] fix(nvca): deploy without a durable cache when the storage catalog is missing The agent read the storage capability catalog ConfigMap at the top of every creation message that requests a cache, and any error, including NotFound, failed the message. On a cluster whose agent image had moved ahead of the chart that installs the catalog, every new container function stayed in DEPLOYING while the queue retried the message forever and no ICMSRequest was ever created. A missing catalog now returns ErrStorageCapabilityCatalogNotFound and is handled like a missing nvcf-sc: warn with the ConfigMap name, count it under failure_reason=catalog_missing, and record a non-durable selection so the function deploys uncached. A present but malformed catalog still fails, since that is a configuration error and not a rollout gap. Co-Authored-By: Balaji Ganesan --- ...sdd-storage-agnostic-cache-architecture.md | 1 + .../nvca/internal/metrics/METRICS.md | 1 + .../internal/metrics/modelcachetypes/types.go | 6 +- .../pkg/nvca/modelcache_storage_selection.go | 19 ++++++ .../nvca/modelcache_storage_selection_test.go | 58 +++++++++++++++++++ .../nvca/pkg/storage/storage_capabilities.go | 16 +++++ .../pkg/storage/storage_capabilities_test.go | 19 ++++++ 7 files changed, 119 insertions(+), 1 deletion(-) diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index 5f468da598..c22e9ace31 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -215,6 +215,7 @@ a `Retain` class must be created for the cache. |---|---| | Catalog, class, or gate changes after the binding exists | Binding stays authoritative | | Class or catalog drifts before the binding exists | Fail before any side effect | +| Catalog ConfigMap is absent (agent ahead of its chart) | Warn, count, deploy without a durable cache; Helm falls back to ephemeral | | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind | | Object has foreign or missing ownership | Never adopt or delete it | | Reader PV and claim disagree on class | Never binds; prevented by construction | diff --git a/src/compute-plane-services/nvca/internal/metrics/METRICS.md b/src/compute-plane-services/nvca/internal/metrics/METRICS.md index 9d7a26fc31..61fc2aa2b2 100644 --- a/src/compute-plane-services/nvca/internal/metrics/METRICS.md +++ b/src/compute-plane-services/nvca/internal/metrics/METRICS.md @@ -583,6 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1 | Reason | Description | |--------|-------------| | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) | +| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request proceeds without a durable cache | | `pvc_setup_failed` | Primary PV/PVC setup failures | | `pvc_bind_failed` | RO PVC bind failures | | `rw_pvc_bind_failed` | RW PVC bind failures | diff --git a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go index e255904266..acb0d0b960 100644 --- a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go +++ b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go @@ -25,7 +25,10 @@ const ( // Failure reason values for model cache metrics const ( - ReasonCacheSpecInvalid = "cache_spec_invalid" + ReasonCacheSpecInvalid = "cache_spec_invalid" + // ReasonCatalogMissing: the storage capability catalog ConfigMap is absent, + // so the request was created without a durable cache. + ReasonCatalogMissing = "catalog_missing" ReasonPVCSetupFailed = "pvc_setup_failed" ReasonPVCBindFailed = "pvc_bind_failed" ReasonRWPVCBindFailed = "rw_pvc_bind_failed" @@ -49,6 +52,7 @@ const ( // Used for pre-initializing Prometheus counters to zero. var AllFailureReasons = []string{ ReasonCacheSpecInvalid, + ReasonCatalogMissing, ReasonPVCSetupFailed, ReasonPVCBindFailed, ReasonRWPVCBindFailed, diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index b30e53f109..fe24625053 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -21,6 +21,10 @@ import ( "context" "errors" "fmt" + nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" + modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" + "github.com/sirupsen/logrus" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" @@ -80,6 +84,21 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( if workflow == nvcastorage.ModelCacheWorkflowHelm { mode = nvcastorage.ModelCacheSelectionEphemeral } + case errors.Is(err, nvcastorage.ErrStorageCapabilityCatalogNotFound): + // The chart that ships this agent also installs the catalog, so + // its absence means the rollout has not converged. Blocking every + // deployment until it does helps nobody; run uncached and say so. + core.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ + "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, + "workflow": workflow, + }).Warn("storage capability catalog is missing, deploying without a durable model cache") + if m := nvcametrics.FromContext(ctx); m != nil { + m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, + string(nvcastorage.HelmCacheBackendNone)) + } + if workflow == nvcastorage.ModelCacheWorkflowHelm { + mode = nvcastorage.ModelCacheSelectionEphemeral + } case err != nil: return fmt.Errorf("resolve model cache storage: %w", err) case resolved.Transition == nvcastorage.ModelCacheTransitionDisabled: diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go index 0b98e2ee64..c15a7bd096 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -249,6 +249,28 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, wantMode: nvcastorage.ModelCacheSelectionNone, }, + { + name: "missing catalog ConfigMap disables regular cache", + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionNone, + }, + { + name: "missing catalog ConfigMap falls Helm back to ephemeral", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionEphemeral, + }, { name: "missing StorageClass falls Helm back to ephemeral", helm: true, @@ -326,3 +348,39 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test require.NoError(t, listErr) assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call") } + +// A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap +// did not exist, every creation message failed before the ICMSRequest was +// created, and the queue retried it forever. Absence must degrade, not block. +func TestCreateICMSCreationMessageRequestMissingCatalogDeploysUncached(t *testing.T) { + objects := []runtime.Object{selectionStorageClass()} + cache, _ := selectionBackendCache(objects, featureflag.CachingSupport) + cache.clients = mockKubeClients(objects...) + cache.requestsNamespace = RequestsNamespace + + msg := function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + RequestID: "missing-catalog-request", + NCAID: "test-nca", + Action: common.FunctionCreationAction, + }, + Details: function.Details{ + FunctionID: "function-id", + FunctionVersionID: "function-version-id", + }, + LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification, + } + + created, err := cache.CreateICMSCreationMessageRequest( + newTestContext(), msg, "receipt", "message-id", "queue") + require.NoError(t, err, "a missing catalog must not fail the creation message") + require.NotNil(t, created) + + requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + require.Len(t, requests.Items, 1) + selection := parseRequestStorageSelection(t, &requests.Items[0]) + assert.Equal(t, nvcastorage.ModelCacheSelectionNone, selection.Mode) + assert.Empty(t, selection.StorageClassName) +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index fab9243c99..3966d113b5 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -65,6 +65,12 @@ var ( // ErrModelCacheStorageClassNotFound means nvcf-sc is absent. Callers may map // this to a documented non-durable fallback. ErrModelCacheStorageClassNotFound = errors.New("model cache StorageClass not found") + // ErrStorageCapabilityCatalogNotFound means the catalog ConfigMap is absent, + // which happens when the agent image runs ahead of the chart that installs + // it. Callers treat it like an absent nvcf-sc: nothing is qualified, so the + // request proceeds without a durable cache. A present but malformed catalog + // is a different error and stays fatal. + ErrStorageCapabilityCatalogNotFound = errors.New("storage capability catalog ConfigMap not found") // ErrModelCacheStorageSelectionDrift marks a deterministic mismatch between // a persisted selection and its live StorageClass or catalog input. Callers // can fail the request without treating transient API errors as drift. @@ -214,6 +220,9 @@ func loadStorageCapabilityCatalog( cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + err = fmt.Errorf("%w: %w", ErrStorageCapabilityCatalogNotFound, err) + } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } @@ -238,6 +247,9 @@ func loadStorageCapabilityCatalogSnapshot( cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + err = fmt.Errorf("%w: %w", ErrStorageCapabilityCatalogNotFound, err) + } return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } @@ -317,6 +329,10 @@ func ResolveModelCacheStorageWithClientset( cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) if err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + catalogNamespace, StorageCapabilityConfigMapName, ErrStorageCapabilityCatalogNotFound) + } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", catalogNamespace, StorageCapabilityConfigMapName, err) } diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index e23fd7fad0..d8c7665099 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -18,7 +18,9 @@ limitations under the License. package storage import ( + "context" "encoding/json" + fakek8sclient "k8s.io/client-go/kubernetes/fake" "os" "path/filepath" "strings" @@ -545,3 +547,20 @@ func TestStorageCapabilityCatalogEncryptionSupported(t *testing.T) { digestDriverProfile(NVMeshStorageClassProvisioner, without, ModelCacheWorkflowHelm, ModelCacheTransitionROXReadOnly), "encryption support is part of the driver profile digest") } + +func TestResolveModelCacheStorageWithClientsetMissingCatalogIsSentinel(t *testing.T) { + sc := storageClassWithProvisioner(DefaultModelCacheStorageClassName, NVMeshStorageClassProvisioner) + k8s := fakek8sclient.NewSimpleClientset(sc) + + _, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.ErrorIs(t, err, ErrStorageCapabilityCatalogNotFound) + assert.NotErrorIs(t, err, ErrModelCacheStorageClassNotFound) + + // A present but empty catalog is a configuration error, not absence. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + }) + _, err = ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.Error(t, err) + assert.NotErrorIs(t, err, ErrStorageCapabilityCatalogNotFound) +} From aa600d64810f67db23d12e28aaf15da90c8560c8 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Thu, 17 Sep 2026 11:04:22 -0700 Subject: [PATCH 2/5] fix(nvca): carry request fields, empty backend, and NotFound cause in the catalog warning Review follow-ups: the warning uses the ICMSRequest field logger so the request and function identifiers are on the line, the metric uses the documented empty backend label since no backend is known without a catalog, and the wrapped error keeps the Kubernetes NotFound cause. Co-Authored-By: Balaji Ganesan --- .../nvca/pkg/nvca/modelcache_storage_selection.go | 7 ++++--- .../nvca/pkg/storage/storage_capabilities.go | 4 ++-- .../nvca/pkg/storage/storage_capabilities_test.go | 2 ++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index fe24625053..da98c0367e 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/logging" nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" @@ -88,13 +89,13 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( // The chart that ships this agent also installs the catalog, so // its absence means the rollout has not converged. Blocking every // deployment until it does helps nobody; run uncached and say so. - core.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ + logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithError(err).WithFields(logrus.Fields{ "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, "workflow": workflow, }).Warn("storage capability catalog is missing, deploying without a durable model cache") if m := nvcametrics.FromContext(ctx); m != nil { - m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, - string(nvcastorage.HelmCacheBackendNone)) + // The backend label is empty: no catalog means no backend is known. + m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "") } if workflow == nvcastorage.ModelCacheWorkflowHelm { mode = nvcastorage.ModelCacheSelectionEphemeral diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index 3966d113b5..bd5cdadd80 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -330,8 +330,8 @@ func ResolveModelCacheStorageWithClientset( ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", - catalogNamespace, StorageCapabilityConfigMapName, ErrStorageCapabilityCatalogNotFound) + return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w: %w", + catalogNamespace, StorageCapabilityConfigMapName, ErrStorageCapabilityCatalogNotFound, err) } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", catalogNamespace, StorageCapabilityConfigMapName, err) diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index d8c7665099..95d4c703f6 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -20,6 +20,7 @@ package storage import ( "context" "encoding/json" + apierrors "k8s.io/apimachinery/pkg/api/errors" fakek8sclient "k8s.io/client-go/kubernetes/fake" "os" "path/filepath" @@ -555,6 +556,7 @@ func TestResolveModelCacheStorageWithClientsetMissingCatalogIsSentinel(t *testin _, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) require.ErrorIs(t, err, ErrStorageCapabilityCatalogNotFound) assert.NotErrorIs(t, err, ErrModelCacheStorageClassNotFound) + assert.True(t, apierrors.IsNotFound(err), "the Kubernetes NotFound cause must survive wrapping") // A present but empty catalog is a configuration error, not absence. k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ From f9de809d85dc5a37fbbf1eece51c1785e4927e45 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Thu, 17 Sep 2026 12:21:33 -0700 Subject: [PATCH 3/5] fix(nvca): resolve against the built-in catalog when the ConfigMap is absent Replaces the deploy-uncached branch with the catalog itself. The YAML the chart installs as nvcf-storage-capabilities is compiled into NVCA, and a test keeps the two byte-identical. When the ConfigMap is absent the resolver uses the built-in copy, so an agent whose chart has not converged makes the same selection a fresh install would: NVMesh cached, other providers off. The selection is flagged CatalogBuiltin, the agent warns with the request fields, and the catalog_missing counter still counts the rollout gap. A present ConfigMap stays authoritative, and a present but malformed one still fails. Co-Authored-By: Balaji Ganesan --- ...sdd-storage-agnostic-cache-architecture.md | 2 +- .../nvca/internal/metrics/METRICS.md | 2 +- .../pkg/nvca/modelcache_storage_selection.go | 35 ++++++---- .../nvca/modelcache_storage_selection_test.go | 32 ++++++--- .../nvca/pkg/storage/BUILD.bazel | 2 + .../nvcf-storage-capabilities-v1alpha1.yaml | 63 +++++++++++++++++ .../nvca/pkg/storage/storage_capabilities.go | 69 ++++++++++++------- .../storage/storage_capabilities_builtin.go | 42 +++++++++++ .../pkg/storage/storage_capabilities_test.go | 61 +++++++++++----- .../pkg/storage/storage_resolution_test.go | 14 ---- 10 files changed, 240 insertions(+), 82 deletions(-) create mode 100644 src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml create mode 100644 src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index c22e9ace31..a808147a6f 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -215,7 +215,7 @@ a `Retain` class must be created for the cache. |---|---| | Catalog, class, or gate changes after the binding exists | Binding stays authoritative | | Class or catalog drifts before the binding exists | Fail before any side effect | -| Catalog ConfigMap is absent (agent ahead of its chart) | Warn, count, deploy without a durable cache; Helm falls back to ephemeral | +| Catalog ConfigMap is absent (agent ahead of its chart) | Resolve against the catalog compiled into NVCA, warn, count; the ConfigMap is authoritative once present | | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind | | Object has foreign or missing ownership | Never adopt or delete it | | Reader PV and claim disagree on class | Never binds; prevented by construction | diff --git a/src/compute-plane-services/nvca/internal/metrics/METRICS.md b/src/compute-plane-services/nvca/internal/metrics/METRICS.md index 61fc2aa2b2..ae303e3167 100644 --- a/src/compute-plane-services/nvca/internal/metrics/METRICS.md +++ b/src/compute-plane-services/nvca/internal/metrics/METRICS.md @@ -583,7 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1 | Reason | Description | |--------|-------------| | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) | -| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request proceeds without a durable cache | +| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request was resolved against the catalog built into NVCA | | `pvc_setup_failed` | Primary PV/PVC setup failures | | `pvc_bind_failed` | RO PVC bind failures | | `rw_pvc_bind_failed` | RW PVC bind failures | diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index da98c0367e..2c101ba042 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -80,26 +80,14 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( var err error resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset( ctx, c.clients.K8s, c.systemNamespace, workflow) + if err == nil && resolved.CatalogBuiltin { + c.noteBuiltinCatalog(ctx, req, workflow) + } switch { case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound): if workflow == nvcastorage.ModelCacheWorkflowHelm { mode = nvcastorage.ModelCacheSelectionEphemeral } - case errors.Is(err, nvcastorage.ErrStorageCapabilityCatalogNotFound): - // The chart that ships this agent also installs the catalog, so - // its absence means the rollout has not converged. Blocking every - // deployment until it does helps nobody; run uncached and say so. - logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithError(err).WithFields(logrus.Fields{ - "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, - "workflow": workflow, - }).Warn("storage capability catalog is missing, deploying without a durable model cache") - if m := nvcametrics.FromContext(ctx); m != nil { - // The backend label is empty: no catalog means no backend is known. - m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "") - } - if workflow == nvcastorage.ModelCacheWorkflowHelm { - mode = nvcastorage.ModelCacheSelectionEphemeral - } case err != nil: return fmt.Errorf("resolve model cache storage: %w", err) case resolved.Transition == nvcastorage.ModelCacheTransitionDisabled: @@ -140,3 +128,20 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload return nil } + +// noteBuiltinCatalog records that a request was resolved against the catalog +// compiled into NVCA because the nvcf-storage-capabilities ConfigMap is absent. +// The selection is the one a converged install would make; the warning and the +// counter exist so the chart rollout gap is visible. +func (c *BackendK8sCache) noteBuiltinCatalog( + ctx context.Context, req *nvcav2beta1.ICMSRequest, workflow nvcastorage.ModelCacheWorkflow, +) { + logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithFields(logrus.Fields{ + "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, + "workflow": workflow, + }).Warn("storage capability catalog ConfigMap is missing, using the catalog built into NVCA") + if m := nvcametrics.FromContext(ctx); m != nil { + // The backend label is empty: the selection has not chosen a backend yet. + m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "") + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go index c15a7bd096..324270f3b6 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -250,16 +250,20 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantMode: nvcastorage.ModelCacheSelectionNone, }, { - name: "missing catalog ConfigMap disables regular cache", + name: "missing catalog ConfigMap resolves against the built-in catalog", objects: func() []runtime.Object { return []runtime.Object{selectionStorageClass()} }, - flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, - wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, - wantMode: nvcastorage.ModelCacheSelectionNone, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, }, { - name: "missing catalog ConfigMap falls Helm back to ephemeral", + name: "missing catalog ConfigMap resolves Helm against the built-in catalog", helm: true, objects: func() []runtime.Object { return []runtime.Object{selectionStorageClass()} @@ -268,8 +272,12 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { featureflag.CachingSupport, featureflag.HelmModelCaching, }, - wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, - wantMode: nvcastorage.ModelCacheSelectionEphemeral, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, }, { name: "missing StorageClass falls Helm back to ephemeral", @@ -351,8 +359,9 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test // A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap // did not exist, every creation message failed before the ICMSRequest was -// created, and the queue retried it forever. Absence must degrade, not block. -func TestCreateICMSCreationMessageRequestMissingCatalogDeploysUncached(t *testing.T) { +// created, and the queue retried it forever. The agent must resolve against +// the catalog it was built with instead. +func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *testing.T) { objects := []runtime.Object{selectionStorageClass()} cache, _ := selectionBackendCache(objects, featureflag.CachingSupport) cache.clients = mockKubeClients(objects...) @@ -381,6 +390,7 @@ func TestCreateICMSCreationMessageRequestMissingCatalogDeploysUncached(t *testin require.NoError(t, listErr) require.Len(t, requests.Items, 1) selection := parseRequestStorageSelection(t, &requests.Items[0]) - assert.Equal(t, nvcastorage.ModelCacheSelectionNone, selection.Mode) - assert.Empty(t, selection.StorageClassName) + assert.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode) + assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName) } diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index ace6bc398f..79e3397ab7 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "storage", + embedsrcs = ["nvcf-storage-capabilities-v1alpha1.yaml"], srcs = [ "modelcache_selection.go", "cachebackend.go", @@ -20,6 +21,7 @@ go_library( "sharedstorage.go", "smbcsidriver.go", "storage_capabilities.go", + "storage_capabilities_builtin.go", "storage_request_api.go", "storagerequest.go", "translate_workload.go", diff --git a/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 0000000000..214ff92398 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVCA owns this catalog and installs it with the NVCA chart. Each entry is +# named by exact CSI provisioner and records the PVC access modes qualified end to end in an +# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs +# from these modes. +# +# ReadWriteMany one shared claim; readers mount it read-only +# ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own +# +# An access mode a driver merely accepts is not a qualification. A claim that +# binds is not either. An empty accessModes list means nothing is qualified +# yet, so caching stays off for that driver, and a provisioner absent from this +# file is unsupported. Enabling a backend is an edit here, backed by a +# qualification run. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + - name: nvmesh-csi.excelero.com + provider: nvmesh + # Encrypted caches are qualified on NVMesh, through a derived StorageClass + # and a per-sharing-domain Secret. + encryptionSupported: true + accessModes: + - ReadWriteOnce + - ReadOnlyMany + # The reader PV is XFS on the same filesystem as the writer, so it needs + # nouuid and norecovery or the mount fails outright. + readerMountOptions: + - ro + - norecovery + - nouuid + - name: csi.weka.io + provider: weka + # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache + # workflow was not, so nothing is qualified yet. + accessModes: [] + readerMountOptions: [] + - name: fss.csi.oraclecloud.com + provider: ociFss + # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, + # which is not evidence for a ReadOnlyMany claim. No cache workflow was + # qualified. + accessModes: [] + readerMountOptions: [] + - name: lustre.csi.oraclecloud.com + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + readerMountOptions: [] diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index bd5cdadd80..1471f61e38 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -65,12 +65,6 @@ var ( // ErrModelCacheStorageClassNotFound means nvcf-sc is absent. Callers may map // this to a documented non-durable fallback. ErrModelCacheStorageClassNotFound = errors.New("model cache StorageClass not found") - // ErrStorageCapabilityCatalogNotFound means the catalog ConfigMap is absent, - // which happens when the agent image runs ahead of the chart that installs - // it. Callers treat it like an absent nvcf-sc: nothing is qualified, so the - // request proceeds without a durable cache. A present but malformed catalog - // is a different error and stays fatal. - ErrStorageCapabilityCatalogNotFound = errors.New("storage capability catalog ConfigMap not found") // ErrModelCacheStorageSelectionDrift marks a deterministic mismatch between // a persisted selection and its live StorageClass or catalog input. Callers // can fail the request without treating transient API errors as drift. @@ -89,6 +83,9 @@ const ( // live nvcf-sc object and the public capability catalog. It deliberately does // not infer behavior from a provider name or access mode. type ModelCacheStorageSelection struct { + // CatalogBuiltin is set when the ConfigMap was absent and the selection was + // resolved against the catalog compiled into NVCA. + CatalogBuiltin bool EncryptionSupported bool StorageClassName string StorageClassUID types.UID @@ -221,7 +218,8 @@ func loadStorageCapabilityCatalog( cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { if apierrors.IsNotFound(err) { - err = fmt.Errorf("%w: %w", ErrStorageCapabilityCatalogNotFound, err) + catalog, _, berr := builtinStorageCapabilityCatalog() + return catalog, berr } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) @@ -236,23 +234,33 @@ func loadStorageCapabilityCatalog( return parseStorageCapabilityCatalog(raw) } +// loadStorageCapabilityCatalogSnapshot returns the catalog, its payload +// digest, and whether the compiled-in copy was used because the ConfigMap is +// absent. A ConfigMap that exists but is empty or malformed is an error. func loadStorageCapabilityCatalogSnapshot( ctx context.Context, c client.Client, namespace string, -) (*storageCapabilityCatalog, string, error) { +) (*storageCapabilityCatalog, string, bool, error) { if namespace == "" { - return nil, "", fmt.Errorf("storage capability ConfigMap namespace is empty") + return nil, "", false, fmt.Errorf("storage capability ConfigMap namespace is empty") } cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { if apierrors.IsNotFound(err) { - err = fmt.Errorf("%w: %w", ErrStorageCapabilityCatalogNotFound, err) + catalog, digest, berr := builtinStorageCapabilityCatalog() + return catalog, digest, true, berr } - return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + return nil, "", false, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } + catalog, digest, err := parseStorageCapabilityConfigMap(cm, namespace) + return catalog, digest, false, err +} + +// parseStorageCapabilityConfigMap parses a present catalog ConfigMap. +func parseStorageCapabilityConfigMap(cm *corev1.ConfigMap, namespace string) (*storageCapabilityCatalog, string, error) { raw, ok := cm.Data[StorageCapabilityConfigMapKey] if !ok || raw == "" { return nil, "", fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", @@ -298,11 +306,16 @@ func ResolveModelCacheStorage( return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } - catalog, catalogDigest, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + catalog, catalogDigest, builtin, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + if err != nil { + return nil, err + } + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) + selection.CatalogBuiltin = builtin + return selection, nil } // ResolveModelCacheStorageWithClientset provides the same decision to the @@ -326,26 +339,34 @@ func ResolveModelCacheStorageWithClientset( } return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } + var ( + catalog *storageCapabilityCatalog + catalogDigest string + builtin bool + ) cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w: %w", - catalogNamespace, StorageCapabilityConfigMapName, ErrStorageCapabilityCatalogNotFound, err) - } + switch { + case apierrors.IsNotFound(err): + // The chart that ships this agent installs the ConfigMap. Until it has, + // resolve against the same catalog it would install. + catalog, catalogDigest, err = builtinStorageCapabilityCatalog() + builtin = true + case err != nil: return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", catalogNamespace, StorageCapabilityConfigMapName, err) + default: + catalog, catalogDigest, err = parseStorageCapabilityConfigMap(cm, catalogNamespace) } - raw, ok := cm.Data[StorageCapabilityConfigMapKey] - if !ok || raw == "" { - return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", - catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + if err != nil { + return nil, err } - catalog, err := parseStorageCapabilityCatalog(raw) + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, digestCatalogPayload(raw), workflow) + selection.CatalogBuiltin = builtin + return selection, nil } func selectModelCacheStorageFromObjects( diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go new file mode 100644 index 0000000000..eca965453b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go @@ -0,0 +1,42 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + _ "embed" + "fmt" +) + +// builtinStorageCapabilityCatalogYAML is the catalog the NVCA chart ships as +// the nvcf-storage-capabilities ConfigMap, compiled into the agent so an agent +// whose chart has not converged resolves against the same defaults a fresh +// install would. The ConfigMap stays authoritative when it exists; this copy is +// read only when it is absent. TestBuiltinCatalogMatchesChart keeps the two +// identical. +// +//go:embed nvcf-storage-capabilities-v1alpha1.yaml +var builtinStorageCapabilityCatalogYAML string + +// builtinStorageCapabilityCatalog parses the compiled-in catalog. +func builtinStorageCapabilityCatalog() (*storageCapabilityCatalog, string, error) { + catalog, err := parseStorageCapabilityCatalog(builtinStorageCapabilityCatalogYAML) + if err != nil { + return nil, "", fmt.Errorf("built-in storage capability catalog: %w", err) + } + return catalog, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index 95d4c703f6..29c5f27660 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -20,7 +20,6 @@ package storage import ( "context" "encoding/json" - apierrors "k8s.io/apimachinery/pkg/api/errors" fakek8sclient "k8s.io/client-go/kubernetes/fake" "os" "path/filepath" @@ -96,9 +95,6 @@ func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { } func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { - missingConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: testCatalogNamespace}, - } missingData := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, } @@ -121,10 +117,6 @@ func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { name: "empty namespace", namespace: "", configMap: capabilityCatalogConfigMap(validCatalog), want: "namespace is empty", }, - { - name: "missing ConfigMap", namespace: testCatalogNamespace, - configMap: missingConfigMap, want: "get storage capability ConfigMap", - }, { name: "missing data key", namespace: testCatalogNamespace, configMap: missingData, want: "has no", @@ -549,20 +541,57 @@ func TestStorageCapabilityCatalogEncryptionSupported(t *testing.T) { "encryption support is part of the driver profile digest") } -func TestResolveModelCacheStorageWithClientsetMissingCatalogIsSentinel(t *testing.T) { - sc := storageClassWithProvisioner(DefaultModelCacheStorageClassName, NVMeshStorageClassProvisioner) +func TestResolveModelCacheStorageWithClientsetMissingCatalogUsesBuiltin(t *testing.T) { + sc := testModelCacheStorageClass() k8s := fakek8sclient.NewSimpleClientset(sc) - _, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) - require.ErrorIs(t, err, ErrStorageCapabilityCatalogNotFound) - assert.NotErrorIs(t, err, ErrModelCacheStorageClassNotFound) - assert.True(t, apierrors.IsNotFound(err), "the Kubernetes NotFound cause must survive wrapping") + selection, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err, "an absent catalog ConfigMap must resolve against the built-in catalog") + assert.True(t, selection.CatalogBuiltin) + assert.Equal(t, ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, ModelCacheTransitionROXReadOnly, selection.Transition) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), selection.CatalogRevision) + + // With the ConfigMap present the selection is identical except for the flag. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + Data: map[string]string{StorageCapabilityConfigMapKey: builtinStorageCapabilityCatalogYAML}, + }) + fromConfigMap, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err) + assert.False(t, fromConfigMap.CatalogBuiltin) + fromConfigMap.CatalogBuiltin = true + assert.Equal(t, selection, fromConfigMap) // A present but empty catalog is a configuration error, not absence. k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, }) _, err = ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) - require.Error(t, err) - assert.NotErrorIs(t, err, ErrStorageCapabilityCatalogNotFound) + require.ErrorContains(t, err, "has no") +} + +// The built-in catalog must be the one the chart installs, or an agent ahead +// of its chart would resolve differently from a converged install. +func TestBuiltinCatalogMatchesChart(t *testing.T) { + chartCopy, err := os.ReadFile(filepath.Join("..", "..", "deployments", "nvca-operator", "files", + "nvcf-storage-capabilities-v1alpha1.yaml")) + if err != nil { + t.Skipf("chart catalog not reachable from this test root: %v", err) + } + assert.Equal(t, string(chartCopy), builtinStorageCapabilityCatalogYAML) + _, _, err = builtinStorageCapabilityCatalog() + require.NoError(t, err) +} + +func TestLoadStorageCapabilityCatalogMissingConfigMapUsesBuiltin(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + catalog, digest, builtin, err := loadStorageCapabilityCatalogSnapshot(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + assert.True(t, builtin) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), digest) + _, ok := catalog.Drivers[NVMeshStorageClassProvisioner] + assert.True(t, ok, "the built-in catalog must qualify NVMesh") } diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go index 6a8ee98be1..09f99c42dc 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go @@ -157,13 +157,6 @@ func TestResolveModelCacheStorageErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []client.Object { return []client.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "catalog ConfigMap missing data key", namespace: testCatalogNamespace, @@ -315,13 +308,6 @@ func TestResolveModelCacheStorageWithClientsetErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []runtime.Object { return []runtime.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "empty catalog namespace", namespace: "", From 88c8933ea744556f36da6797e0d5335b78550441 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Thu, 17 Sep 2026 12:33:12 -0700 Subject: [PATCH 4/5] test(storage): fail, not skip, when the chart catalog is unreadable The storage test target already carries the chart catalog as Bazel data with the package as run dir, so the file is reachable under both runners. Co-Authored-By: Balaji Ganesan --- .../nvca/pkg/storage/storage_capabilities_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index 29c5f27660..0fd630ae53 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -574,11 +574,11 @@ func TestResolveModelCacheStorageWithClientsetMissingCatalogUsesBuiltin(t *testi // The built-in catalog must be the one the chart installs, or an agent ahead // of its chart would resolve differently from a converged install. func TestBuiltinCatalogMatchesChart(t *testing.T) { + // Reachable under go test from the package dir and under Bazel through the + // storage-capability-catalog data dependency with rundir ".". chartCopy, err := os.ReadFile(filepath.Join("..", "..", "deployments", "nvca-operator", "files", "nvcf-storage-capabilities-v1alpha1.yaml")) - if err != nil { - t.Skipf("chart catalog not reachable from this test root: %v", err) - } + require.NoError(t, err, "the chart catalog must be readable; the built-in copy is checked against it") assert.Equal(t, string(chartCopy), builtinStorageCapabilityCatalogYAML) _, _, err = builtinStorageCapabilityCatalog() require.NoError(t, err) From fb2977f5e3698dd2f8561e6f43700adbd10242d9 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Thu, 17 Sep 2026 12:45:10 -0700 Subject: [PATCH 5/5] test(storage): locate the chart catalog the same way under Bazel and go test Share the runfiles-aware chart directory lookup the shipped-catalog test already uses, so the built-in catalog check passes under bazel test. Co-Authored-By: Balaji Ganesan --- .../pkg/storage/storage_capabilities_test.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index 0fd630ae53..b4a6a659b1 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -358,11 +358,20 @@ func TestValidateStorageCapabilityCatalogAllowsNothingQualified(t *testing.T) { require.NoError(t, validateStorageCapabilityCatalog(catalog)) } -func TestShippedStorageCapabilityCatalog(t *testing.T) { +// shippedChartDir locates the operator chart: Bazel runs the test from the +// runfiles root with the storage-capability-catalog data dependency, go test +// runs it from the package directory. +func shippedChartDir(t *testing.T) string { + t.Helper() chartDir := filepath.Join("src", "compute-plane-services", "nvca", "deployments", "nvca-operator") if _, err := os.Stat(chartDir); os.IsNotExist(err) { chartDir = filepath.Join("..", "..", "deployments", "nvca-operator") } + return chartDir +} + +func TestShippedStorageCapabilityCatalog(t *testing.T) { + chartDir := shippedChartDir(t) raw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.yaml")) require.NoError(t, err) c := capabilityClient(t, capabilityCatalogConfigMap(string(raw))).Build() @@ -574,10 +583,7 @@ func TestResolveModelCacheStorageWithClientsetMissingCatalogUsesBuiltin(t *testi // The built-in catalog must be the one the chart installs, or an agent ahead // of its chart would resolve differently from a converged install. func TestBuiltinCatalogMatchesChart(t *testing.T) { - // Reachable under go test from the package dir and under Bazel through the - // storage-capability-catalog data dependency with rundir ".". - chartCopy, err := os.ReadFile(filepath.Join("..", "..", "deployments", "nvca-operator", "files", - "nvcf-storage-capabilities-v1alpha1.yaml")) + chartCopy, err := os.ReadFile(filepath.Join(shippedChartDir(t), "files", "nvcf-storage-capabilities-v1alpha1.yaml")) require.NoError(t, err, "the chart catalog must be readable; the built-in copy is checked against it") assert.Equal(t, string(chartCopy), builtinStorageCapabilityCatalogYAML) _, _, err = builtinStorageCapabilityCatalog()