From ca23be7950c60cb1da58c7b61f4513eced4f354a Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 09:02:28 -0700 Subject: [PATCH 1/6] fix(nvca-operator): mirror the storage capability catalog into the agent namespace The chart renders nvcf-storage-capabilities into the operator's release namespace, but the agent and the storage controller read it from the agent's system namespace, and nothing copied it across. The prototype had the operator mirror; it was dropped when the catalog was split out into #1334. A cluster on 3.7.1 with the chart fully synced therefore still failed every cache-requesting deployment on "ConfigMap not found". The NVCFBackend reconcile now mirrors the ConfigMap the same way it does the custom annotations, and an edit to the chart ConfigMap forces a reconcile so the agent copy follows. An absent source is skipped with a warning rather than failing the reconcile, since the agent falls back to its compiled-in catalog until the chart converges. Co-Authored-By: Balaji Ganesan --- ...sdd-storage-agnostic-cache-architecture.md | 8 +- .../nvca/pkg/operator/reconcile/BUILD.bazel | 3 +- .../pkg/operator/reconcile/backendk8scache.go | 4 +- .../operator/reconcile/nvcaagent_reconcile.go | 38 +++++++ .../storage_capabilities_configmap_test.go | 105 ++++++++++++++++++ 5 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index 5f468da598..a33fd3ce8b 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -50,8 +50,12 @@ Four pieces: ## Capability catalog -Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities`, validated -by a packaged JSON Schema and by the Go loader with the same rules. +Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities` in the +operator's namespace; the operator mirrors it into the agent's namespace, where +the agent and the storage controller read it, and re-mirrors on every edit. A +copy of the shipped catalog is compiled into NVCA and used only while the +ConfigMap is absent. Validated by a packaged JSON Schema and by the Go loader +with the same rules. ```yaml drivers: diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel index 156962fc69..8eb3db1cb4 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel @@ -133,15 +133,16 @@ go_test( "gpu_profiling_configmap_test.go", "miniservice_restrictions_test.go", "namespace_helpers_test.go", + "nvca_config_mapper_test.go", "nvcaagent_reconcile_test.go", "nvcaagent_selfmanaged_test.go", - "nvca_config_mapper_test.go", "otel_reconcile_test.go", "reconcile_helpers_test.go", "rollout_test.go", "sanitize_test.go", "setup_auth_client_secrets_extended_test.go", "setup_auth_client_secrets_test.go", + "storage_capabilities_configmap_test.go", "transport_tls_config_test.go", "vault_config_data_test.go", "version_detection_test.go", diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go index 38c0898b6c..7e3fd220a5 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go @@ -62,6 +62,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/metrics" nvcaopotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/otel" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) const ( @@ -581,7 +582,8 @@ func configMapUpdateForcesNVCAReconcile(name string) bool { nvcfGPUProfilingConfigMapName, nvcfBackendChartDefaultsConfigMapName, agentConfigMergeConfigMapName, - nvcaOperatorConfigMapName: + nvcaOperatorConfigMapName, + nvcastorage.StorageCapabilityConfigMapName: return true default: return false diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go index 8225559478..56109e157d 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go @@ -59,6 +59,7 @@ import ( nvcaoperatorerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/internal/errors" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -577,6 +578,12 @@ func (bc *BackendK8sCache) setupNVCAAgentInfra( nb.Namespace, nb.Name, err) } + err = bc.setupStorageCapabilityCatalogConfigMap(ctx, nb) + if err != nil { + return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", nvcastorage.StorageCapabilityConfigMapName, + nb.Namespace, nb.Name, err) + } + err = bc.setupGPUProfilingConfigMap(ctx, nb) if err != nil { return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", nvcfGPUProfilingConfigMapName, @@ -1001,6 +1008,37 @@ func (bc *BackendK8sCache) mirrorConfigMap(ctx context.Context, nb *nvidiaiov1.N return bc.createOrUpdateConfigMap(ctx, &cmTemplate) } +// setupStorageCapabilityCatalogConfigMap mirrors the chart-created +// nvcf-storage-capabilities ConfigMap into the agent's system namespace, where +// the agent and the storage controller read it. The chart renders it into the +// operator's release namespace; without this copy the agent never sees it. An +// absent source is skipped with a warning rather than failing the reconcile: +// the agent falls back to the catalog compiled into it until the chart that +// ships the ConfigMap has converged. +func (bc *BackendK8sCache) setupStorageCapabilityCatalogConfigMap(ctx context.Context, nb *nvidiaiov1.NVCFBackend) error { + log := core.GetLogger(ctx) + srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + if err != nil { + if k8serr.IsNotFound(err) { + log.Warnf("%v/%v configmap not found, not mirroring the storage capability catalog into %v", + NVCAOperatorNamespace, nvcastorage.StorageCapabilityConfigMapName, getSystemNamespace(nb)) + return nil + } + return err + } + cmTemplate := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: getSystemNamespace(nb), + Annotations: getNBAnnotations(nb), + Labels: getAppLabels(), + }, + Data: srcCM.Data, + } + return bc.createOrUpdateConfigMap(ctx, &cmTemplate) +} + // setupGPUProfilingConfigMap mirrors the chart-created nvca-gpu-profiling-config ConfigMap // into the agent's system namespace, where NVCA reads it live. Unlike mirrorConfigMap it is // optional: an absent source is skipped (profiling stays off) rather than failing reconcile. diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go new file mode 100644 index 0000000000..de5a0096ca --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +// The chart renders nvcf-storage-capabilities into the operator's namespace; +// the agent reads it from its own. A 3.7.1 cluster with the chart fully synced +// still failed every cache-requesting deployment because nothing copied it. +func TestSetupStorageCapabilityCatalogConfigMap(t *testing.T) { + nb := &nvidiaiov1.NVCFBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backend", Namespace: NVCAOperatorNamespace}, + Spec: nvidiaiov1.NVCFBackendSpec{ + NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{SystemNamespace: "custom-agent-system"}, + }, + }, + } + agentNamespace := getSystemNamespace(nb) + require.NotEqual(t, NVCAOperatorNamespace, agentNamespace) + + t.Run("mirrors the chart ConfigMap into the agent namespace and follows updates", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients} + + src := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: NVCAOperatorNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "apiVersion: storage.nvcf.nvidia.com/v1alpha1"}, + } + _, err := clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Create(ctx, src, metav1.CreateOptions{}) + require.NoError(t, err) + + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + mirrored, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + + src.Data[nvcastorage.StorageCapabilityConfigMapKey] += "\ndrivers: []" + _, err = clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Update(ctx, src, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + mirrored, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data, "an edit to the chart ConfigMap must reach the agent copy") + }) + + t.Run("absent source is skipped and an existing agent copy is kept", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients} + + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb), + "a chart that has not converged must not fail the NVCFBackend reconcile") + _, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + assert.Error(t, err, "nothing to mirror, nothing created") + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName, Namespace: agentNamespace}, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "last good"}, + } + _, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Create(ctx, existing, metav1.CreateOptions{}) + require.NoError(t, err) + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + kept, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, existing.Data, kept.Data) + }) + + t.Run("an update to the chart ConfigMap forces an NVCA reconcile", func(t *testing.T) { + assert.True(t, configMapUpdateForcesNVCAReconcile(nvcastorage.StorageCapabilityConfigMapName)) + }) +} From f56920046f08b05833b08388685f946921ebfa12 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 09:22:10 -0700 Subject: [PATCH 2/6] test(nvca-operator): assert the mirrored storage catalog in the full sync tests The sync tests already prove every required object exists in the agent namespace after a reconcile; the chart-rendered catalog ConfigMap is now part of the fixture and its mirrored copy and data are asserted alongside the other ConfigMaps. Co-Authored-By: Balaji Ganesan --- .../reconcile/backendk8scache_test.go | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go index 9cb793d240..108f2ce7b0 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go @@ -63,6 +63,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) var ( @@ -532,6 +533,23 @@ func mockKubeClientsForIntegrationTests() *kubeclients.KubeClients { } } +// testStorageCapabilityCatalog is the shipped catalog shape, enough for the +// sync tests to prove the mirror carries the data through unchanged. +const testStorageCapabilityCatalog = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + - name: nvmesh-csi.excelero.com + provider: nvmesh + encryptionSupported: true + accessModes: + - ReadWriteOnce + - ReadOnlyMany + readerMountOptions: + - ro + - norecovery + - nouuid +` + func mockKubeClients() *kubeclients.KubeClients { scheme := newTestScheme() k8sClient := fakek8sclient.NewSimpleClientset( @@ -543,6 +561,15 @@ func mockKubeClients() *kubeclients.KubeClients { }, Data: map[string]string{}, }, + // Rendered by the chart into the operator namespace; the sync must + // mirror it into the agent namespace, where NVCA reads it. + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: NVCAOperatorNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: testStorageCapabilityCatalog}, + }, ) discClient := k8sClient.Discovery().(*fakediscovery.FakeDiscovery) icmsCRD := makeICMSRequestCRD() @@ -720,6 +747,10 @@ func TestBackendK8sSyncMinimal(t *testing.T) { _, err = bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) require.NoError(ct, err) + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") + require.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) + _, err = bc.clients.K8s.CoreV1().Secrets(getSystemNamespace(nb)).Get(ctx, NGCServiceAPIKeySecretName, metav1.GetOptions{}) require.NoError(ct, err) @@ -925,6 +956,10 @@ func TestBackendK8sSyncMinimalExternal(t *testing.T) { _, err = bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) require.NoError(ct, err) + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") + require.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) + nbObj, err := bc.clients.NVCAOP.NvcfV1().NVCFBackends(agentOpts.SystemNamespace).Get(ctx, nb.Name, metav1.GetOptions{}) require.NoError(ct, err) require.Equal(ct, "v1.26.6", nbObj.Spec.Version) @@ -1143,6 +1178,14 @@ func TestBackendK8sSyncAllFeatures(t *testing.T) { return } + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + if !assert.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") { + return + } + if !assert.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) { + return + } + dep, err = bc.clients.K8s.AppsV1().Deployments(getSystemNamespace(nb)).Get(ctx, nvcaoptypes.NVCAModuleName, metav1.GetOptions{}) if !assert.NoError(ct, err) { return From 47636c644dbc5066b81dd901e24a974932e47546 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 12:54:35 -0700 Subject: [PATCH 3/6] fix(nvca): resolve against the built-in catalog when the ConfigMap is absent The YAML the chart installs as nvcf-storage-capabilities is compiled into NVCA and used only while the ConfigMap is absent, so an agent whose chart has not converged, or whose operator has not mirrored the ConfigMap yet, makes the same selection a fresh install would. The selection is flagged CatalogBuiltin, the agent warns with the request fields, and the catalog_missing counter records the gap. A present ConfigMap stays authoritative; a present but malformed one still fails. Folded from #1953. Co-Authored-By: Balaji Ganesan --- ...sdd-storage-agnostic-cache-architecture.md | 1 + .../nvca/internal/metrics/METRICS.md | 1 + .../internal/metrics/modelcachetypes/types.go | 6 +- .../pkg/nvca/modelcache_storage_selection.go | 25 +++++++ .../nvca/modelcache_storage_selection_test.go | 68 ++++++++++++++++++ .../nvca/pkg/storage/BUILD.bazel | 2 + .../nvcf-storage-capabilities-v1alpha1.yaml | 63 ++++++++++++++++ .../nvca/pkg/storage/storage_capabilities.go | 61 ++++++++++++---- .../storage/storage_capabilities_builtin.go | 42 +++++++++++ .../pkg/storage/storage_capabilities_test.go | 72 ++++++++++++++++--- .../pkg/storage/storage_resolution_test.go | 14 ---- 11 files changed, 320 insertions(+), 35 deletions(-) create mode 100644 src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml create mode 100644 src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index a33fd3ce8b..d98d80570e 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -219,6 +219,7 @@ a `Retain` class must be created for the cache. |---|---| | Catalog, class, or gate changes after the binding exists | Binding stays authoritative | | Class or catalog drifts before the binding exists | Fail before any side effect | +| Catalog ConfigMap is absent (agent ahead of its chart) | Resolve against the catalog compiled into NVCA, warn, count; the ConfigMap is authoritative once present | | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind | | Object has foreign or missing ownership | Never adopt or delete it | | Reader PV and claim disagree on class | Never binds; prevented by construction | diff --git a/src/compute-plane-services/nvca/internal/metrics/METRICS.md b/src/compute-plane-services/nvca/internal/metrics/METRICS.md index 9d7a26fc31..ae303e3167 100644 --- a/src/compute-plane-services/nvca/internal/metrics/METRICS.md +++ b/src/compute-plane-services/nvca/internal/metrics/METRICS.md @@ -583,6 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1 | Reason | Description | |--------|-------------| | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) | +| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request was resolved against the catalog built into NVCA | | `pvc_setup_failed` | Primary PV/PVC setup failures | | `pvc_bind_failed` | RO PVC bind failures | | `rw_pvc_bind_failed` | RW PVC bind failures | diff --git a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go index e255904266..acb0d0b960 100644 --- a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go +++ b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go @@ -25,7 +25,10 @@ const ( // Failure reason values for model cache metrics const ( - ReasonCacheSpecInvalid = "cache_spec_invalid" + ReasonCacheSpecInvalid = "cache_spec_invalid" + // ReasonCatalogMissing: the storage capability catalog ConfigMap is absent, + // so the request was created without a durable cache. + ReasonCatalogMissing = "catalog_missing" ReasonPVCSetupFailed = "pvc_setup_failed" ReasonPVCBindFailed = "pvc_bind_failed" ReasonRWPVCBindFailed = "rw_pvc_bind_failed" @@ -49,6 +52,7 @@ const ( // Used for pre-initializing Prometheus counters to zero. var AllFailureReasons = []string{ ReasonCacheSpecInvalid, + ReasonCatalogMissing, ReasonPVCSetupFailed, ReasonPVCBindFailed, ReasonRWPVCBindFailed, diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index b30e53f109..2c101ba042 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -21,6 +21,11 @@ import ( "context" "errors" "fmt" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/logging" + nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" + modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" + "github.com/sirupsen/logrus" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" @@ -75,6 +80,9 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( var err error resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset( ctx, c.clients.K8s, c.systemNamespace, workflow) + if err == nil && resolved.CatalogBuiltin { + c.noteBuiltinCatalog(ctx, req, workflow) + } switch { case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound): if workflow == nvcastorage.ModelCacheWorkflowHelm { @@ -120,3 +128,20 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload return nil } + +// noteBuiltinCatalog records that a request was resolved against the catalog +// compiled into NVCA because the nvcf-storage-capabilities ConfigMap is absent. +// The selection is the one a converged install would make; the warning and the +// counter exist so the chart rollout gap is visible. +func (c *BackendK8sCache) noteBuiltinCatalog( + ctx context.Context, req *nvcav2beta1.ICMSRequest, workflow nvcastorage.ModelCacheWorkflow, +) { + logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithFields(logrus.Fields{ + "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, + "workflow": workflow, + }).Warn("storage capability catalog ConfigMap is missing, using the catalog built into NVCA") + if m := nvcametrics.FromContext(ctx); m != nil { + // The backend label is empty: the selection has not chosen a backend yet. + m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "") + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go index 0b98e2ee64..324270f3b6 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -249,6 +249,36 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, wantMode: nvcastorage.ModelCacheSelectionNone, }, + { + name: "missing catalog ConfigMap resolves against the built-in catalog", + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "missing catalog ConfigMap resolves Helm against the built-in catalog", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, { name: "missing StorageClass falls Helm back to ephemeral", helm: true, @@ -326,3 +356,41 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test require.NoError(t, listErr) assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call") } + +// A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap +// did not exist, every creation message failed before the ICMSRequest was +// created, and the queue retried it forever. The agent must resolve against +// the catalog it was built with instead. +func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *testing.T) { + objects := []runtime.Object{selectionStorageClass()} + cache, _ := selectionBackendCache(objects, featureflag.CachingSupport) + cache.clients = mockKubeClients(objects...) + cache.requestsNamespace = RequestsNamespace + + msg := function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + RequestID: "missing-catalog-request", + NCAID: "test-nca", + Action: common.FunctionCreationAction, + }, + Details: function.Details{ + FunctionID: "function-id", + FunctionVersionID: "function-version-id", + }, + LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification, + } + + created, err := cache.CreateICMSCreationMessageRequest( + newTestContext(), msg, "receipt", "message-id", "queue") + require.NoError(t, err, "a missing catalog must not fail the creation message") + require.NotNil(t, created) + + requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + require.Len(t, requests.Items, 1) + selection := parseRequestStorageSelection(t, &requests.Items[0]) + assert.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode) + assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName) +} diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index ace6bc398f..79e3397ab7 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "storage", + embedsrcs = ["nvcf-storage-capabilities-v1alpha1.yaml"], srcs = [ "modelcache_selection.go", "cachebackend.go", @@ -20,6 +21,7 @@ go_library( "sharedstorage.go", "smbcsidriver.go", "storage_capabilities.go", + "storage_capabilities_builtin.go", "storage_request_api.go", "storagerequest.go", "translate_workload.go", diff --git a/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 0000000000..214ff92398 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVCA owns this catalog and installs it with the NVCA chart. Each entry is +# named by exact CSI provisioner and records the PVC access modes qualified end to end in an +# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs +# from these modes. +# +# ReadWriteMany one shared claim; readers mount it read-only +# ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own +# +# An access mode a driver merely accepts is not a qualification. A claim that +# binds is not either. An empty accessModes list means nothing is qualified +# yet, so caching stays off for that driver, and a provisioner absent from this +# file is unsupported. Enabling a backend is an edit here, backed by a +# qualification run. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + - name: nvmesh-csi.excelero.com + provider: nvmesh + # Encrypted caches are qualified on NVMesh, through a derived StorageClass + # and a per-sharing-domain Secret. + encryptionSupported: true + accessModes: + - ReadWriteOnce + - ReadOnlyMany + # The reader PV is XFS on the same filesystem as the writer, so it needs + # nouuid and norecovery or the mount fails outright. + readerMountOptions: + - ro + - norecovery + - nouuid + - name: csi.weka.io + provider: weka + # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache + # workflow was not, so nothing is qualified yet. + accessModes: [] + readerMountOptions: [] + - name: fss.csi.oraclecloud.com + provider: ociFss + # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, + # which is not evidence for a ReadOnlyMany claim. No cache workflow was + # qualified. + accessModes: [] + readerMountOptions: [] + - name: lustre.csi.oraclecloud.com + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + readerMountOptions: [] diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index fab9243c99..1471f61e38 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -83,6 +83,9 @@ const ( // live nvcf-sc object and the public capability catalog. It deliberately does // not infer behavior from a provider name or access mode. type ModelCacheStorageSelection struct { + // CatalogBuiltin is set when the ConfigMap was absent and the selection was + // resolved against the catalog compiled into NVCA. + CatalogBuiltin bool EncryptionSupported bool StorageClassName string StorageClassUID types.UID @@ -214,6 +217,10 @@ func loadStorageCapabilityCatalog( cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + catalog, _, berr := builtinStorageCapabilityCatalog() + return catalog, berr + } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } @@ -227,20 +234,33 @@ func loadStorageCapabilityCatalog( return parseStorageCapabilityCatalog(raw) } +// loadStorageCapabilityCatalogSnapshot returns the catalog, its payload +// digest, and whether the compiled-in copy was used because the ConfigMap is +// absent. A ConfigMap that exists but is empty or malformed is an error. func loadStorageCapabilityCatalogSnapshot( ctx context.Context, c client.Client, namespace string, -) (*storageCapabilityCatalog, string, error) { +) (*storageCapabilityCatalog, string, bool, error) { if namespace == "" { - return nil, "", fmt.Errorf("storage capability ConfigMap namespace is empty") + return nil, "", false, fmt.Errorf("storage capability ConfigMap namespace is empty") } cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { - return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + if apierrors.IsNotFound(err) { + catalog, digest, berr := builtinStorageCapabilityCatalog() + return catalog, digest, true, berr + } + return nil, "", false, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } + catalog, digest, err := parseStorageCapabilityConfigMap(cm, namespace) + return catalog, digest, false, err +} + +// parseStorageCapabilityConfigMap parses a present catalog ConfigMap. +func parseStorageCapabilityConfigMap(cm *corev1.ConfigMap, namespace string) (*storageCapabilityCatalog, string, error) { raw, ok := cm.Data[StorageCapabilityConfigMapKey] if !ok || raw == "" { return nil, "", fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", @@ -286,11 +306,16 @@ func ResolveModelCacheStorage( return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } - catalog, catalogDigest, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + catalog, catalogDigest, builtin, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + if err != nil { + return nil, err + } + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) + selection.CatalogBuiltin = builtin + return selection, nil } // ResolveModelCacheStorageWithClientset provides the same decision to the @@ -314,22 +339,34 @@ func ResolveModelCacheStorageWithClientset( } return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } + var ( + catalog *storageCapabilityCatalog + catalogDigest string + builtin bool + ) cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) - if err != nil { + switch { + case apierrors.IsNotFound(err): + // The chart that ships this agent installs the ConfigMap. Until it has, + // resolve against the same catalog it would install. + catalog, catalogDigest, err = builtinStorageCapabilityCatalog() + builtin = true + case err != nil: return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", catalogNamespace, StorageCapabilityConfigMapName, err) + default: + catalog, catalogDigest, err = parseStorageCapabilityConfigMap(cm, catalogNamespace) } - raw, ok := cm.Data[StorageCapabilityConfigMapKey] - if !ok || raw == "" { - return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", - catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + if err != nil { + return nil, err } - catalog, err := parseStorageCapabilityCatalog(raw) + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, digestCatalogPayload(raw), workflow) + selection.CatalogBuiltin = builtin + return selection, nil } func selectModelCacheStorageFromObjects( diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go new file mode 100644 index 0000000000..eca965453b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go @@ -0,0 +1,42 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + _ "embed" + "fmt" +) + +// builtinStorageCapabilityCatalogYAML is the catalog the NVCA chart ships as +// the nvcf-storage-capabilities ConfigMap, compiled into the agent so an agent +// whose chart has not converged resolves against the same defaults a fresh +// install would. The ConfigMap stays authoritative when it exists; this copy is +// read only when it is absent. TestBuiltinCatalogMatchesChart keeps the two +// identical. +// +//go:embed nvcf-storage-capabilities-v1alpha1.yaml +var builtinStorageCapabilityCatalogYAML string + +// builtinStorageCapabilityCatalog parses the compiled-in catalog. +func builtinStorageCapabilityCatalog() (*storageCapabilityCatalog, string, error) { + catalog, err := parseStorageCapabilityCatalog(builtinStorageCapabilityCatalogYAML) + if err != nil { + return nil, "", fmt.Errorf("built-in storage capability catalog: %w", err) + } + return catalog, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index e23fd7fad0..b4a6a659b1 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -18,7 +18,9 @@ limitations under the License. package storage import ( + "context" "encoding/json" + fakek8sclient "k8s.io/client-go/kubernetes/fake" "os" "path/filepath" "strings" @@ -93,9 +95,6 @@ func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { } func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { - missingConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: testCatalogNamespace}, - } missingData := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, } @@ -118,10 +117,6 @@ func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { name: "empty namespace", namespace: "", configMap: capabilityCatalogConfigMap(validCatalog), want: "namespace is empty", }, - { - name: "missing ConfigMap", namespace: testCatalogNamespace, - configMap: missingConfigMap, want: "get storage capability ConfigMap", - }, { name: "missing data key", namespace: testCatalogNamespace, configMap: missingData, want: "has no", @@ -363,11 +358,20 @@ func TestValidateStorageCapabilityCatalogAllowsNothingQualified(t *testing.T) { require.NoError(t, validateStorageCapabilityCatalog(catalog)) } -func TestShippedStorageCapabilityCatalog(t *testing.T) { +// shippedChartDir locates the operator chart: Bazel runs the test from the +// runfiles root with the storage-capability-catalog data dependency, go test +// runs it from the package directory. +func shippedChartDir(t *testing.T) string { + t.Helper() chartDir := filepath.Join("src", "compute-plane-services", "nvca", "deployments", "nvca-operator") if _, err := os.Stat(chartDir); os.IsNotExist(err) { chartDir = filepath.Join("..", "..", "deployments", "nvca-operator") } + return chartDir +} + +func TestShippedStorageCapabilityCatalog(t *testing.T) { + chartDir := shippedChartDir(t) raw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.yaml")) require.NoError(t, err) c := capabilityClient(t, capabilityCatalogConfigMap(string(raw))).Build() @@ -545,3 +549,55 @@ func TestStorageCapabilityCatalogEncryptionSupported(t *testing.T) { digestDriverProfile(NVMeshStorageClassProvisioner, without, ModelCacheWorkflowHelm, ModelCacheTransitionROXReadOnly), "encryption support is part of the driver profile digest") } + +func TestResolveModelCacheStorageWithClientsetMissingCatalogUsesBuiltin(t *testing.T) { + sc := testModelCacheStorageClass() + k8s := fakek8sclient.NewSimpleClientset(sc) + + selection, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err, "an absent catalog ConfigMap must resolve against the built-in catalog") + assert.True(t, selection.CatalogBuiltin) + assert.Equal(t, ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, ModelCacheTransitionROXReadOnly, selection.Transition) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), selection.CatalogRevision) + + // With the ConfigMap present the selection is identical except for the flag. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + Data: map[string]string{StorageCapabilityConfigMapKey: builtinStorageCapabilityCatalogYAML}, + }) + fromConfigMap, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err) + assert.False(t, fromConfigMap.CatalogBuiltin) + fromConfigMap.CatalogBuiltin = true + assert.Equal(t, selection, fromConfigMap) + + // A present but empty catalog is a configuration error, not absence. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + }) + _, err = ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.ErrorContains(t, err, "has no") +} + +// The built-in catalog must be the one the chart installs, or an agent ahead +// of its chart would resolve differently from a converged install. +func TestBuiltinCatalogMatchesChart(t *testing.T) { + chartCopy, err := os.ReadFile(filepath.Join(shippedChartDir(t), "files", "nvcf-storage-capabilities-v1alpha1.yaml")) + require.NoError(t, err, "the chart catalog must be readable; the built-in copy is checked against it") + assert.Equal(t, string(chartCopy), builtinStorageCapabilityCatalogYAML) + _, _, err = builtinStorageCapabilityCatalog() + require.NoError(t, err) +} + +func TestLoadStorageCapabilityCatalogMissingConfigMapUsesBuiltin(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + catalog, digest, builtin, err := loadStorageCapabilityCatalogSnapshot(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + assert.True(t, builtin) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), digest) + _, ok := catalog.Drivers[NVMeshStorageClassProvisioner] + assert.True(t, ok, "the built-in catalog must qualify NVMesh") +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go index 6a8ee98be1..09f99c42dc 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go @@ -157,13 +157,6 @@ func TestResolveModelCacheStorageErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []client.Object { return []client.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "catalog ConfigMap missing data key", namespace: testCatalogNamespace, @@ -315,13 +308,6 @@ func TestResolveModelCacheStorageWithClientsetErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []runtime.Object { return []runtime.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "empty catalog namespace", namespace: "", From 350153403c4f3de49c36eb27b30ab7209fd32f4a Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 11:15:22 -0700 Subject: [PATCH 4/6] feat(nvca): enable ReadWriteMany model caching for Weka and OCI FSS Both are shared filesystems, so the catalog entries take the ReadWriteMany shape: one shared claim per cache handle, populated once and mounted read-only by every reader, with no derived reader PV and so no reader mount options. Enabling them makes the Helm and regular cache workflows reachable on clusters whose nvcf-sc is Weka or FSS, which is what the qualification run needs. Lustre stays off. Both catalog copies, the source chart and the vendored chart, change together and remain identical. Co-Authored-By: Balaji Ganesan --- .../nvcf-storage-capabilities-v1alpha1.yaml | 21 ++++++++++++------- ...sdd-storage-agnostic-cache-architecture.md | 2 +- .../nvcf-storage-capabilities-v1alpha1.yaml | 21 ++++++++++++------- .../nvcf-storage-capabilities-v1alpha1.yaml | 21 ++++++++++++------- .../pkg/storage/storage_capabilities_test.go | 19 ++++++++++++----- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 214ff92398..dcc6f93a7e 100644 --- a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -45,16 +45,23 @@ drivers: - nouuid - name: csi.weka.io provider: weka - # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache - # workflow was not, so nothing is qualified yet. - accessModes: [] + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: fss.csi.oraclecloud.com provider: ociFss - # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, - # which is not evidence for a ReadOnlyMany claim. No cache workflow was - # qualified. - accessModes: [] + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: lustre.csi.oraclecloud.com provider: ociLustre diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index d98d80570e..c8628118d0 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -66,7 +66,7 @@ drivers: encryptionSupported: true - name: csi.weka.io provider: weka - accessModes: [] + accessModes: [ReadWriteMany] readerMountOptions: [] ``` diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 214ff92398..dcc6f93a7e 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -45,16 +45,23 @@ drivers: - nouuid - name: csi.weka.io provider: weka - # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache - # workflow was not, so nothing is qualified yet. - accessModes: [] + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: fss.csi.oraclecloud.com provider: ociFss - # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, - # which is not evidence for a ReadOnlyMany claim. No cache workflow was - # qualified. - accessModes: [] + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: lustre.csi.oraclecloud.com provider: ociLustre diff --git a/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml index 214ff92398..dcc6f93a7e 100644 --- a/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml +++ b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml @@ -45,16 +45,23 @@ drivers: - nouuid - name: csi.weka.io provider: weka - # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache - # workflow was not, so nothing is qualified yet. - accessModes: [] + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: fss.csi.oraclecloud.com provider: ociFss - # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, - # which is not evidence for a ReadOnlyMany claim. No cache workflow was - # qualified. - accessModes: [] + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: lustre.csi.oraclecloud.com provider: ociLustre diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index b4a6a659b1..f08edd16ce 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -384,18 +384,27 @@ func TestShippedStorageCapabilityCatalog(t *testing.T) { require.NotNil(t, nvmesh.ReaderMountOptions) assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) - // Weka, FSS and Lustre are recorded but not qualified for a cache workflow, - // so they carry no access modes and caching stays off for them. Enabling - // one is an edit to its accessModes, backed by a qualification run. - for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com", "lustre.csi.oraclecloud.com"} { + // Weka and OCI FSS are shared filesystems enabled on the ReadWriteMany + // shape: one shared claim, readers mount it read-only, no derived reader + // PV and therefore no reader mount options. + for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com"} { driver, ok := catalog.Drivers[provisioner] require.True(t, ok, provisioner) require.NotNil(t, driver.AccessModes, provisioner) - assert.Empty(t, *driver.AccessModes, provisioner) + assert.Equal(t, []string{"ReadWriteMany"}, *driver.AccessModes, provisioner) require.NotNil(t, driver.ReaderMountOptions, provisioner) assert.Empty(t, *driver.ReaderMountOptions, provisioner) } + // Lustre is recorded but not enabled for a cache workflow, so it carries + // no access modes and caching stays off for it. + lustre, ok := catalog.Drivers["lustre.csi.oraclecloud.com"] + require.True(t, ok) + require.NotNil(t, lustre.AccessModes) + assert.Empty(t, *lustre.AccessModes) + require.NotNil(t, lustre.ReaderMountOptions) + assert.Empty(t, *lustre.ReaderMountOptions) + schemaRaw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.schema.json")) require.NoError(t, err) assert.True(t, json.Valid(schemaRaw), "shipped JSON schema must be valid JSON") From e341ffa7b3513f93e83170b536dc210a3dbee1ac Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 13:14:28 -0700 Subject: [PATCH 5/6] fix(nvca): accept the ReadWriteMany selection for Helm functions persistModelCacheStorageSelection took the ReadWriteMany transition as durable only for the regular workflow and returned "unsupported model cache transition" for Helm, so enabling a ReadWriteMany provider would have failed every Helm creation message on that cluster. The Helm side already routes that shape to the shared-filesystem backend through HelmCacheBackendFromSelection. Both shapes are now durable for both workflows, with a test that the persisted Helm selection routes to shared-filesystem. Co-Authored-By: Balaji Ganesan --- .../pkg/nvca/modelcache_storage_selection.go | 10 +++-- .../nvca/modelcache_storage_selection_test.go | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index 2c101ba042..c0ec4ce084 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -94,10 +94,12 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( if workflow == nvcastorage.ModelCacheWorkflowHelm { mode = nvcastorage.ModelCacheSelectionEphemeral } - case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly: - mode = nvcastorage.ModelCacheSelectionDurable - case resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly && - workflow == nvcastorage.ModelCacheWorkflowRegular: + case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly, + resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly: + // Both shapes are durable for both workflows. Helm routes the + // ReadWriteMany shape to the shared-filesystem backend through + // HelmCacheBackendFromSelection; the regular workflow serves it from + // one shared claim per cache handle. mode = nvcastorage.ModelCacheSelectionDurable default: return fmt.Errorf("unsupported model cache transition %q", resolved.Transition) diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go index 324270f3b6..ce5350a4d3 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -210,6 +210,29 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantProvider: "weka", wantProvisioner: "csi.weka.io", }, + { + // Weka and OCI FSS enable this shape; a Helm request on them must + // persist a durable selection that HelmCacheBackendFromSelection routes + // to the shared-filesystem backend, not fail the creation message. + name: "Helm durable provider-neutral RWX", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{ + selectionStorageClassForProvisioner("csi.weka.io"), + selectionCatalogConfigMap(selectionCatalogRWXReadOnly), + } + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionRWXReadOnly, + wantResolvedState: true, + wantProvider: "weka", + wantProvisioner: "csi.weka.io", + }, { name: "disabled regular cache persists none", objects: func() []runtime.Object { @@ -394,3 +417,18 @@ func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *tes assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider) assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName) } + +// The persisted Helm selection on a ReadWriteMany provider must route to the +// shared-filesystem backend, the end-to-end contract this fix restores. +func TestHelmRWXSelectionRoutesToSharedFS(t *testing.T) { + cache, _ := selectionBackendCache([]runtime.Object{ + selectionStorageClassForProvisioner("csi.weka.io"), + selectionCatalogConfigMap(selectionCatalogRWXReadOnly), + }, featureflag.CachingSupport, featureflag.HelmModelCaching) + req := selectionRequest(true) + require.NoError(t, cache.persistModelCacheStorageSelection(t.Context(), req)) + selection := parseRequestStorageSelection(t, req) + backend, err := nvcastorage.HelmCacheBackendFromSelection(selection) + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, backend) +} From cb8875b9cdfa09cc669339496af1048fc4ca7dd3 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Fri, 18 Sep 2026 13:28:50 -0700 Subject: [PATCH 6/6] fix(miniservice): accept existing shared-filesystem and Samba model cache requests validatePersistedModelCacheStorageRequest rejected every existing backend except NVMesh as one that "does not create a StorageRequest", returning a terminal error on the reconcile after the request was created. makeStorageRequests creates a StorageRequest for NVMesh, shared-filesystem and Samba alike, so the validation now accepts the same set. Regression test adopts an existing shared-filesystem request from a ReadWriteMany selection. Co-Authored-By: Balaji Ganesan --- .../modelcache_storage_selection.go | 6 +- .../modelcache_storage_selection_test.go | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go index 8046c67b3c..706f5ae0b8 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go @@ -111,7 +111,11 @@ func validatePersistedModelCacheStorageRequest( existing.Spec.Type != nvcav2beta1.ModelCacheRequest { return conflict("name or type does not match") } - if backend != nvcastorage.HelmCacheBackendNVMesh { + switch backend { + case nvcastorage.HelmCacheBackendNVMesh, nvcastorage.HelmCacheBackendSharedFS, nvcastorage.HelmCacheBackendSamba: + // The backends that create a model cache StorageRequest; keep in step + // with makeStorageRequests. + default: return conflict(fmt.Sprintf("backend %q does not create a StorageRequest", backend)) } if existing.Spec.ModelCache == nil { diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go index 5ae3833ff7..6526166e54 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go @@ -99,6 +99,40 @@ func TestSelectHelmCacheBackend(t *testing.T) { assert.Equal(t, nvcastorage.HelmCacheBackendNVMesh, got) }) + t.Run("matching persisted shared-filesystem StorageRequest is adopted", func(t *testing.T) { + // Weka and OCI FSS persist a ReadWriteMany selection that routes to the + // shared-filesystem backend. The StorageRequest it creates must validate + // on the next reconcile instead of being rejected as a backend that + // creates none. + request := requestWithRWXModelCacheSelection(t) + raw := request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw, + nvcastorage.ICMSRequestUIDAnnotationKey: string(request.UID), + }, + }, + Spec: nvcav2beta1.StorageRequestSpec{ + Type: nvcav2beta1.ModelCacheRequest, + RequestName: request.Name, + RequestNamespace: request.Namespace, + ModelCache: &nvcav2beta1.ModelCacheSpec{ + Backend: string(nvcastorage.HelmCacheBackendSharedFS), + CacheHandle: helmModelCacheHandle(request), + }, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + + got, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, got) + }) + t.Run("ephemeral selection rejects a stale durable StorageRequest", func(t *testing.T) { existing := &nvcav2beta1.StorageRequest{ ObjectMeta: metav1.ObjectMeta{ @@ -368,3 +402,27 @@ func (c *getErrorClient) Get( ) error { return c.err } + +// requestWithRWXModelCacheSelection is a Helm request whose persisted selection +// is the ReadWriteMany shape a shared filesystem such as Weka resolves to. +func requestWithRWXModelCacheSelection(t *testing.T) *nvcav2beta1.ICMSRequest { + t.Helper() + resolved := &nvcastorage.ModelCacheStorageSelection{ + StorageClassName: nvcastorage.DefaultModelCacheStorageClassName, + StorageClassUID: types.UID("storage-class-uid"), + StorageClassDigest: "storage-class-digest", + ProfileDigest: "catalog-digest", + Provider: "weka", + Provisioner: "csi.weka.io", + Transition: nvcastorage.ModelCacheTransitionRWXReadOnly, + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + } + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection( + nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable, resolved) + require.NoError(t, err) + payload, err := selection.Marshal() + require.NoError(t, err) + request := requestWithModelCacheSelection(t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionEphemeral) + request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload + return request +}