diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 214ff92398..dcc6f93a7e 100644 --- a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -45,16 +45,23 @@ drivers: - nouuid - name: csi.weka.io provider: weka - # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache - # workflow was not, so nothing is qualified yet. - accessModes: [] + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: fss.csi.oraclecloud.com provider: ociFss - # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, - # which is not evidence for a ReadOnlyMany claim. No cache workflow was - # qualified. - accessModes: [] + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: lustre.csi.oraclecloud.com provider: ociLustre diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index 5f468da598..c8628118d0 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -50,8 +50,12 @@ Four pieces: ## Capability catalog -Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities`, validated -by a packaged JSON Schema and by the Go loader with the same rules. +Installed by the NVCA chart as ConfigMap `nvcf-storage-capabilities` in the +operator's namespace; the operator mirrors it into the agent's namespace, where +the agent and the storage controller read it, and re-mirrors on every edit. A +copy of the shipped catalog is compiled into NVCA and used only while the +ConfigMap is absent. Validated by a packaged JSON Schema and by the Go loader +with the same rules. ```yaml drivers: @@ -62,7 +66,7 @@ drivers: encryptionSupported: true - name: csi.weka.io provider: weka - accessModes: [] + accessModes: [ReadWriteMany] readerMountOptions: [] ``` @@ -215,6 +219,7 @@ a `Retain` class must be created for the cache. |---|---| | Catalog, class, or gate changes after the binding exists | Binding stays authoritative | | Class or catalog drifts before the binding exists | Fail before any side effect | +| Catalog ConfigMap is absent (agent ahead of its chart) | Resolve against the catalog compiled into NVCA, warn, count; the ConfigMap is authoritative once present | | Binding is `Retiring`, missing, or lacks this request's reference | Fail; never rebind | | Object has foreign or missing ownership | Never adopt or delete it | | Reader PV and claim disagree on class | Never binds; prevented by construction | diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 214ff92398..dcc6f93a7e 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -45,16 +45,23 @@ drivers: - nouuid - name: csi.weka.io provider: weka - # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache - # workflow was not, so nothing is qualified yet. - accessModes: [] + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: fss.csi.oraclecloud.com provider: ociFss - # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, - # which is not evidence for a ReadOnlyMany claim. No cache workflow was - # qualified. - accessModes: [] + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany readerMountOptions: [] - name: lustre.csi.oraclecloud.com provider: ociLustre diff --git a/src/compute-plane-services/nvca/internal/metrics/METRICS.md b/src/compute-plane-services/nvca/internal/metrics/METRICS.md index 9d7a26fc31..ae303e3167 100644 --- a/src/compute-plane-services/nvca/internal/metrics/METRICS.md +++ b/src/compute-plane-services/nvca/internal/metrics/METRICS.md @@ -583,6 +583,7 @@ rate(nvca_k8s_api_failure_total[5m]) > 0.1 | Reason | Description | |--------|-------------| | `cache_spec_invalid` | Spec validation failures (missing fields, decode errors) | +| `catalog_missing` | Storage capability catalog ConfigMap absent at request creation; the request was resolved against the catalog built into NVCA | | `pvc_setup_failed` | Primary PV/PVC setup failures | | `pvc_bind_failed` | RO PVC bind failures | | `rw_pvc_bind_failed` | RW PVC bind failures | diff --git a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go index e255904266..acb0d0b960 100644 --- a/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go +++ b/src/compute-plane-services/nvca/internal/metrics/modelcachetypes/types.go @@ -25,7 +25,10 @@ const ( // Failure reason values for model cache metrics const ( - ReasonCacheSpecInvalid = "cache_spec_invalid" + ReasonCacheSpecInvalid = "cache_spec_invalid" + // ReasonCatalogMissing: the storage capability catalog ConfigMap is absent, + // so the request was created without a durable cache. + ReasonCatalogMissing = "catalog_missing" ReasonPVCSetupFailed = "pvc_setup_failed" ReasonPVCBindFailed = "pvc_bind_failed" ReasonRWPVCBindFailed = "rw_pvc_bind_failed" @@ -49,6 +52,7 @@ const ( // Used for pre-initializing Prometheus counters to zero. var AllFailureReasons = []string{ ReasonCacheSpecInvalid, + ReasonCatalogMissing, ReasonPVCSetupFailed, ReasonPVCBindFailed, ReasonRWPVCBindFailed, diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go index 8046c67b3c..706f5ae0b8 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go @@ -111,7 +111,11 @@ func validatePersistedModelCacheStorageRequest( existing.Spec.Type != nvcav2beta1.ModelCacheRequest { return conflict("name or type does not match") } - if backend != nvcastorage.HelmCacheBackendNVMesh { + switch backend { + case nvcastorage.HelmCacheBackendNVMesh, nvcastorage.HelmCacheBackendSharedFS, nvcastorage.HelmCacheBackendSamba: + // The backends that create a model cache StorageRequest; keep in step + // with makeStorageRequests. + default: return conflict(fmt.Sprintf("backend %q does not create a StorageRequest", backend)) } if existing.Spec.ModelCache == nil { diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go index 5ae3833ff7..6526166e54 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go @@ -99,6 +99,40 @@ func TestSelectHelmCacheBackend(t *testing.T) { assert.Equal(t, nvcastorage.HelmCacheBackendNVMesh, got) }) + t.Run("matching persisted shared-filesystem StorageRequest is adopted", func(t *testing.T) { + // Weka and OCI FSS persist a ReadWriteMany selection that routes to the + // shared-filesystem backend. The StorageRequest it creates must validate + // on the next reconcile instead of being rejected as a backend that + // creates none. + request := requestWithRWXModelCacheSelection(t) + raw := request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw, + nvcastorage.ICMSRequestUIDAnnotationKey: string(request.UID), + }, + }, + Spec: nvcav2beta1.StorageRequestSpec{ + Type: nvcav2beta1.ModelCacheRequest, + RequestName: request.Name, + RequestNamespace: request.Namespace, + ModelCache: &nvcav2beta1.ModelCacheSpec{ + Backend: string(nvcastorage.HelmCacheBackendSharedFS), + CacheHandle: helmModelCacheHandle(request), + }, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + + got, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, got) + }) + t.Run("ephemeral selection rejects a stale durable StorageRequest", func(t *testing.T) { existing := &nvcav2beta1.StorageRequest{ ObjectMeta: metav1.ObjectMeta{ @@ -368,3 +402,27 @@ func (c *getErrorClient) Get( ) error { return c.err } + +// requestWithRWXModelCacheSelection is a Helm request whose persisted selection +// is the ReadWriteMany shape a shared filesystem such as Weka resolves to. +func requestWithRWXModelCacheSelection(t *testing.T) *nvcav2beta1.ICMSRequest { + t.Helper() + resolved := &nvcastorage.ModelCacheStorageSelection{ + StorageClassName: nvcastorage.DefaultModelCacheStorageClassName, + StorageClassUID: types.UID("storage-class-uid"), + StorageClassDigest: "storage-class-digest", + ProfileDigest: "catalog-digest", + Provider: "weka", + Provisioner: "csi.weka.io", + Transition: nvcastorage.ModelCacheTransitionRWXReadOnly, + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + } + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection( + nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable, resolved) + require.NoError(t, err) + payload, err := selection.Marshal() + require.NoError(t, err) + request := requestWithModelCacheSelection(t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionEphemeral) + request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload + return request +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go index b30e53f109..c0ec4ce084 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -21,6 +21,11 @@ import ( "context" "errors" "fmt" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/logging" + nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" + modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" + "github.com/sirupsen/logrus" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" @@ -75,6 +80,9 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( var err error resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset( ctx, c.clients.K8s, c.systemNamespace, workflow) + if err == nil && resolved.CatalogBuiltin { + c.noteBuiltinCatalog(ctx, req, workflow) + } switch { case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound): if workflow == nvcastorage.ModelCacheWorkflowHelm { @@ -86,10 +94,12 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( if workflow == nvcastorage.ModelCacheWorkflowHelm { mode = nvcastorage.ModelCacheSelectionEphemeral } - case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly: - mode = nvcastorage.ModelCacheSelectionDurable - case resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly && - workflow == nvcastorage.ModelCacheWorkflowRegular: + case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly, + resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly: + // Both shapes are durable for both workflows. Helm routes the + // ReadWriteMany shape to the shared-filesystem backend through + // HelmCacheBackendFromSelection; the regular workflow serves it from + // one shared claim per cache handle. mode = nvcastorage.ModelCacheSelectionDurable default: return fmt.Errorf("unsupported model cache transition %q", resolved.Transition) @@ -120,3 +130,20 @@ func (c *BackendK8sCache) persistModelCacheStorageSelection( req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload return nil } + +// noteBuiltinCatalog records that a request was resolved against the catalog +// compiled into NVCA because the nvcf-storage-capabilities ConfigMap is absent. +// The selection is the one a converged install would make; the warning and the +// counter exist so the chart rollout gap is visible. +func (c *BackendK8sCache) noteBuiltinCatalog( + ctx context.Context, req *nvcav2beta1.ICMSRequest, workflow nvcastorage.ModelCacheWorkflow, +) { + logging.NewICMSRequestFieldLogger(req, core.GetLogger(ctx)).WithFields(logrus.Fields{ + "configMap": c.systemNamespace + "/" + nvcastorage.StorageCapabilityConfigMapName, + "workflow": workflow, + }).Warn("storage capability catalog ConfigMap is missing, using the catalog built into NVCA") + if m := nvcametrics.FromContext(ctx); m != nil { + // The backend label is empty: the selection has not chosen a backend yet. + m.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonCatalogMissing, "") + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go index 0b98e2ee64..ce5350a4d3 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -210,6 +210,29 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantProvider: "weka", wantProvisioner: "csi.weka.io", }, + { + // Weka and OCI FSS enable this shape; a Helm request on them must + // persist a durable selection that HelmCacheBackendFromSelection routes + // to the shared-filesystem backend, not fail the creation message. + name: "Helm durable provider-neutral RWX", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{ + selectionStorageClassForProvisioner("csi.weka.io"), + selectionCatalogConfigMap(selectionCatalogRWXReadOnly), + } + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionRWXReadOnly, + wantResolvedState: true, + wantProvider: "weka", + wantProvisioner: "csi.weka.io", + }, { name: "disabled regular cache persists none", objects: func() []runtime.Object { @@ -249,6 +272,36 @@ func TestPersistModelCacheStorageSelection(t *testing.T) { wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, wantMode: nvcastorage.ModelCacheSelectionNone, }, + { + name: "missing catalog ConfigMap resolves against the built-in catalog", + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "missing catalog ConfigMap resolves Helm against the built-in catalog", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass()} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, { name: "missing StorageClass falls Helm back to ephemeral", helm: true, @@ -326,3 +379,56 @@ func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *test require.NoError(t, listErr) assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call") } + +// A 3.7.1 agent rolled out ahead of its chart hit this: the catalog ConfigMap +// did not exist, every creation message failed before the ICMSRequest was +// created, and the queue retried it forever. The agent must resolve against +// the catalog it was built with instead. +func TestCreateICMSCreationMessageRequestMissingCatalogUsesBuiltinCatalog(t *testing.T) { + objects := []runtime.Object{selectionStorageClass()} + cache, _ := selectionBackendCache(objects, featureflag.CachingSupport) + cache.clients = mockKubeClients(objects...) + cache.requestsNamespace = RequestsNamespace + + msg := function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + RequestID: "missing-catalog-request", + NCAID: "test-nca", + Action: common.FunctionCreationAction, + }, + Details: function.Details{ + FunctionID: "function-id", + FunctionVersionID: "function-version-id", + }, + LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification, + } + + created, err := cache.CreateICMSCreationMessageRequest( + newTestContext(), msg, "receipt", "message-id", "queue") + require.NoError(t, err, "a missing catalog must not fail the creation message") + require.NotNil(t, created) + + requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + require.Len(t, requests.Items, 1) + selection := parseRequestStorageSelection(t, &requests.Items[0]) + assert.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode) + assert.Equal(t, nvcastorage.ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName) +} + +// The persisted Helm selection on a ReadWriteMany provider must route to the +// shared-filesystem backend, the end-to-end contract this fix restores. +func TestHelmRWXSelectionRoutesToSharedFS(t *testing.T) { + cache, _ := selectionBackendCache([]runtime.Object{ + selectionStorageClassForProvisioner("csi.weka.io"), + selectionCatalogConfigMap(selectionCatalogRWXReadOnly), + }, featureflag.CachingSupport, featureflag.HelmModelCaching) + req := selectionRequest(true) + require.NoError(t, cache.persistModelCacheStorageSelection(t.Context(), req)) + selection := parseRequestStorageSelection(t, req) + backend, err := nvcastorage.HelmCacheBackendFromSelection(selection) + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendSharedFS, backend) +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel index 156962fc69..8eb3db1cb4 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel @@ -133,15 +133,16 @@ go_test( "gpu_profiling_configmap_test.go", "miniservice_restrictions_test.go", "namespace_helpers_test.go", + "nvca_config_mapper_test.go", "nvcaagent_reconcile_test.go", "nvcaagent_selfmanaged_test.go", - "nvca_config_mapper_test.go", "otel_reconcile_test.go", "reconcile_helpers_test.go", "rollout_test.go", "sanitize_test.go", "setup_auth_client_secrets_extended_test.go", "setup_auth_client_secrets_test.go", + "storage_capabilities_configmap_test.go", "transport_tls_config_test.go", "vault_config_data_test.go", "version_detection_test.go", diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go index 38c0898b6c..7e3fd220a5 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go @@ -62,6 +62,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/metrics" nvcaopotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/otel" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) const ( @@ -581,7 +582,8 @@ func configMapUpdateForcesNVCAReconcile(name string) bool { nvcfGPUProfilingConfigMapName, nvcfBackendChartDefaultsConfigMapName, agentConfigMergeConfigMapName, - nvcaOperatorConfigMapName: + nvcaOperatorConfigMapName, + nvcastorage.StorageCapabilityConfigMapName: return true default: return false diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go index 9cb793d240..108f2ce7b0 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go @@ -63,6 +63,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) var ( @@ -532,6 +533,23 @@ func mockKubeClientsForIntegrationTests() *kubeclients.KubeClients { } } +// testStorageCapabilityCatalog is the shipped catalog shape, enough for the +// sync tests to prove the mirror carries the data through unchanged. +const testStorageCapabilityCatalog = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + - name: nvmesh-csi.excelero.com + provider: nvmesh + encryptionSupported: true + accessModes: + - ReadWriteOnce + - ReadOnlyMany + readerMountOptions: + - ro + - norecovery + - nouuid +` + func mockKubeClients() *kubeclients.KubeClients { scheme := newTestScheme() k8sClient := fakek8sclient.NewSimpleClientset( @@ -543,6 +561,15 @@ func mockKubeClients() *kubeclients.KubeClients { }, Data: map[string]string{}, }, + // Rendered by the chart into the operator namespace; the sync must + // mirror it into the agent namespace, where NVCA reads it. + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: NVCAOperatorNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: testStorageCapabilityCatalog}, + }, ) discClient := k8sClient.Discovery().(*fakediscovery.FakeDiscovery) icmsCRD := makeICMSRequestCRD() @@ -720,6 +747,10 @@ func TestBackendK8sSyncMinimal(t *testing.T) { _, err = bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) require.NoError(ct, err) + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") + require.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) + _, err = bc.clients.K8s.CoreV1().Secrets(getSystemNamespace(nb)).Get(ctx, NGCServiceAPIKeySecretName, metav1.GetOptions{}) require.NoError(ct, err) @@ -925,6 +956,10 @@ func TestBackendK8sSyncMinimalExternal(t *testing.T) { _, err = bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) require.NoError(ct, err) + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") + require.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) + nbObj, err := bc.clients.NVCAOP.NvcfV1().NVCFBackends(agentOpts.SystemNamespace).Get(ctx, nb.Name, metav1.GetOptions{}) require.NoError(ct, err) require.Equal(ct, "v1.26.6", nbObj.Spec.Version) @@ -1143,6 +1178,14 @@ func TestBackendK8sSyncAllFeatures(t *testing.T) { return } + catalogCM, err := bc.clients.K8s.CoreV1().ConfigMaps(getSystemNamespace(nb)).Get(ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + if !assert.NoError(ct, err, "the storage capability catalog must be mirrored into the agent namespace") { + return + } + if !assert.Equal(ct, testStorageCapabilityCatalog, catalogCM.Data[nvcastorage.StorageCapabilityConfigMapKey]) { + return + } + dep, err = bc.clients.K8s.AppsV1().Deployments(getSystemNamespace(nb)).Get(ctx, nvcaoptypes.NVCAModuleName, metav1.GetOptions{}) if !assert.NoError(ct, err) { return diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go index 8225559478..56109e157d 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go @@ -59,6 +59,7 @@ import ( nvcaoperatorerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/internal/errors" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -577,6 +578,12 @@ func (bc *BackendK8sCache) setupNVCAAgentInfra( nb.Namespace, nb.Name, err) } + err = bc.setupStorageCapabilityCatalogConfigMap(ctx, nb) + if err != nil { + return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", nvcastorage.StorageCapabilityConfigMapName, + nb.Namespace, nb.Name, err) + } + err = bc.setupGPUProfilingConfigMap(ctx, nb) if err != nil { return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", nvcfGPUProfilingConfigMapName, @@ -1001,6 +1008,37 @@ func (bc *BackendK8sCache) mirrorConfigMap(ctx context.Context, nb *nvidiaiov1.N return bc.createOrUpdateConfigMap(ctx, &cmTemplate) } +// setupStorageCapabilityCatalogConfigMap mirrors the chart-created +// nvcf-storage-capabilities ConfigMap into the agent's system namespace, where +// the agent and the storage controller read it. The chart renders it into the +// operator's release namespace; without this copy the agent never sees it. An +// absent source is skipped with a warning rather than failing the reconcile: +// the agent falls back to the catalog compiled into it until the chart that +// ships the ConfigMap has converged. +func (bc *BackendK8sCache) setupStorageCapabilityCatalogConfigMap(ctx context.Context, nb *nvidiaiov1.NVCFBackend) error { + log := core.GetLogger(ctx) + srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + if err != nil { + if k8serr.IsNotFound(err) { + log.Warnf("%v/%v configmap not found, not mirroring the storage capability catalog into %v", + NVCAOperatorNamespace, nvcastorage.StorageCapabilityConfigMapName, getSystemNamespace(nb)) + return nil + } + return err + } + cmTemplate := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: getSystemNamespace(nb), + Annotations: getNBAnnotations(nb), + Labels: getAppLabels(), + }, + Data: srcCM.Data, + } + return bc.createOrUpdateConfigMap(ctx, &cmTemplate) +} + // setupGPUProfilingConfigMap mirrors the chart-created nvca-gpu-profiling-config ConfigMap // into the agent's system namespace, where NVCA reads it live. Unlike mirrorConfigMap it is // optional: an absent source is skipped (profiling stays off) rather than failing reconcile. diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go new file mode 100644 index 0000000000..de5a0096ca --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +// The chart renders nvcf-storage-capabilities into the operator's namespace; +// the agent reads it from its own. A 3.7.1 cluster with the chart fully synced +// still failed every cache-requesting deployment because nothing copied it. +func TestSetupStorageCapabilityCatalogConfigMap(t *testing.T) { + nb := &nvidiaiov1.NVCFBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backend", Namespace: NVCAOperatorNamespace}, + Spec: nvidiaiov1.NVCFBackendSpec{ + NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{SystemNamespace: "custom-agent-system"}, + }, + }, + } + agentNamespace := getSystemNamespace(nb) + require.NotEqual(t, NVCAOperatorNamespace, agentNamespace) + + t.Run("mirrors the chart ConfigMap into the agent namespace and follows updates", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients} + + src := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: NVCAOperatorNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "apiVersion: storage.nvcf.nvidia.com/v1alpha1"}, + } + _, err := clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Create(ctx, src, metav1.CreateOptions{}) + require.NoError(t, err) + + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + mirrored, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + + src.Data[nvcastorage.StorageCapabilityConfigMapKey] += "\ndrivers: []" + _, err = clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Update(ctx, src, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + mirrored, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data, "an edit to the chart ConfigMap must reach the agent copy") + }) + + t.Run("absent source is skipped and an existing agent copy is kept", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients} + + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb), + "a chart that has not converged must not fail the NVCFBackend reconcile") + _, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + assert.Error(t, err, "nothing to mirror, nothing created") + + existing := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName, Namespace: agentNamespace}, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "last good"}, + } + _, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Create(ctx, existing, metav1.CreateOptions{}) + require.NoError(t, err) + require.NoError(t, bc.setupStorageCapabilityCatalogConfigMap(ctx, nb)) + kept, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, nvcastorage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, existing.Data, kept.Data) + }) + + t.Run("an update to the chart ConfigMap forces an NVCA reconcile", func(t *testing.T) { + assert.True(t, configMapUpdateForcesNVCAReconcile(nvcastorage.StorageCapabilityConfigMapName)) + }) +} diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index ace6bc398f..79e3397ab7 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -5,6 +5,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "storage", + embedsrcs = ["nvcf-storage-capabilities-v1alpha1.yaml"], srcs = [ "modelcache_selection.go", "cachebackend.go", @@ -20,6 +21,7 @@ go_library( "sharedstorage.go", "smbcsidriver.go", "storage_capabilities.go", + "storage_capabilities_builtin.go", "storage_request_api.go", "storagerequest.go", "translate_workload.go", diff --git a/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 0000000000..dcc6f93a7e --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVCA owns this catalog and installs it with the NVCA chart. Each entry is +# named by exact CSI provisioner and records the PVC access modes qualified end to end in an +# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs +# from these modes. +# +# ReadWriteMany one shared claim; readers mount it read-only +# ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own +# +# An access mode a driver merely accepts is not a qualification. A claim that +# binds is not either. An empty accessModes list means nothing is qualified +# yet, so caching stays off for that driver, and a provisioner absent from this +# file is unsupported. Enabling a backend is an edit here, backed by a +# qualification run. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + - name: nvmesh-csi.excelero.com + provider: nvmesh + # Encrypted caches are qualified on NVMesh, through a derived StorageClass + # and a per-sharing-domain Secret. + encryptionSupported: true + accessModes: + - ReadWriteOnce + - ReadOnlyMany + # The reader PV is XFS on the same filesystem as the writer, so it needs + # nouuid and norecovery or the mount fails outright. + readerMountOptions: + - ro + - norecovery + - nouuid + - name: csi.weka.io + provider: weka + # Weka is a shared filesystem: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany + readerMountOptions: [] + - name: fss.csi.oraclecloud.com + provider: ociFss + # OCI File Storage is NFS: one ReadWriteMany claim per cache handle, + # populated once and mounted read-only by every reader, with no derived + # reader PV, so readerMountOptions stays empty. Enabled so the cache + # workflows can be exercised; record the qualification run in the pull + # request that flips this entry. + accessModes: + - ReadWriteMany + readerMountOptions: [] + - name: lustre.csi.oraclecloud.com + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + readerMountOptions: [] diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index fab9243c99..1471f61e38 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -83,6 +83,9 @@ const ( // live nvcf-sc object and the public capability catalog. It deliberately does // not infer behavior from a provider name or access mode. type ModelCacheStorageSelection struct { + // CatalogBuiltin is set when the ConfigMap was absent and the selection was + // resolved against the catalog compiled into NVCA. + CatalogBuiltin bool EncryptionSupported bool StorageClassName string StorageClassUID types.UID @@ -214,6 +217,10 @@ func loadStorageCapabilityCatalog( cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + if apierrors.IsNotFound(err) { + catalog, _, berr := builtinStorageCapabilityCatalog() + return catalog, berr + } return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } @@ -227,20 +234,33 @@ func loadStorageCapabilityCatalog( return parseStorageCapabilityCatalog(raw) } +// loadStorageCapabilityCatalogSnapshot returns the catalog, its payload +// digest, and whether the compiled-in copy was used because the ConfigMap is +// absent. A ConfigMap that exists but is empty or malformed is an error. func loadStorageCapabilityCatalogSnapshot( ctx context.Context, c client.Client, namespace string, -) (*storageCapabilityCatalog, string, error) { +) (*storageCapabilityCatalog, string, bool, error) { if namespace == "" { - return nil, "", fmt.Errorf("storage capability ConfigMap namespace is empty") + return nil, "", false, fmt.Errorf("storage capability ConfigMap namespace is empty") } cm := &corev1.ConfigMap{} if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { - return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + if apierrors.IsNotFound(err) { + catalog, digest, berr := builtinStorageCapabilityCatalog() + return catalog, digest, true, berr + } + return nil, "", false, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", namespace, StorageCapabilityConfigMapName, err) } + catalog, digest, err := parseStorageCapabilityConfigMap(cm, namespace) + return catalog, digest, false, err +} + +// parseStorageCapabilityConfigMap parses a present catalog ConfigMap. +func parseStorageCapabilityConfigMap(cm *corev1.ConfigMap, namespace string) (*storageCapabilityCatalog, string, error) { raw, ok := cm.Data[StorageCapabilityConfigMapKey] if !ok || raw == "" { return nil, "", fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", @@ -286,11 +306,16 @@ func ResolveModelCacheStorage( return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } - catalog, catalogDigest, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + catalog, catalogDigest, builtin, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + if err != nil { + return nil, err + } + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) + selection.CatalogBuiltin = builtin + return selection, nil } // ResolveModelCacheStorageWithClientset provides the same decision to the @@ -314,22 +339,34 @@ func ResolveModelCacheStorageWithClientset( } return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) } + var ( + catalog *storageCapabilityCatalog + catalogDigest string + builtin bool + ) cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) - if err != nil { + switch { + case apierrors.IsNotFound(err): + // The chart that ships this agent installs the ConfigMap. Until it has, + // resolve against the same catalog it would install. + catalog, catalogDigest, err = builtinStorageCapabilityCatalog() + builtin = true + case err != nil: return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", catalogNamespace, StorageCapabilityConfigMapName, err) + default: + catalog, catalogDigest, err = parseStorageCapabilityConfigMap(cm, catalogNamespace) } - raw, ok := cm.Data[StorageCapabilityConfigMapKey] - if !ok || raw == "" { - return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", - catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + if err != nil { + return nil, err } - catalog, err := parseStorageCapabilityCatalog(raw) + selection, err := selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) if err != nil { return nil, err } - return selectModelCacheStorageFromObjects(sc, catalog, digestCatalogPayload(raw), workflow) + selection.CatalogBuiltin = builtin + return selection, nil } func selectModelCacheStorageFromObjects( diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go new file mode 100644 index 0000000000..eca965453b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_builtin.go @@ -0,0 +1,42 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + _ "embed" + "fmt" +) + +// builtinStorageCapabilityCatalogYAML is the catalog the NVCA chart ships as +// the nvcf-storage-capabilities ConfigMap, compiled into the agent so an agent +// whose chart has not converged resolves against the same defaults a fresh +// install would. The ConfigMap stays authoritative when it exists; this copy is +// read only when it is absent. TestBuiltinCatalogMatchesChart keeps the two +// identical. +// +//go:embed nvcf-storage-capabilities-v1alpha1.yaml +var builtinStorageCapabilityCatalogYAML string + +// builtinStorageCapabilityCatalog parses the compiled-in catalog. +func builtinStorageCapabilityCatalog() (*storageCapabilityCatalog, string, error) { + catalog, err := parseStorageCapabilityCatalog(builtinStorageCapabilityCatalogYAML) + if err != nil { + return nil, "", fmt.Errorf("built-in storage capability catalog: %w", err) + } + return catalog, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index e23fd7fad0..f08edd16ce 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -18,7 +18,9 @@ limitations under the License. package storage import ( + "context" "encoding/json" + fakek8sclient "k8s.io/client-go/kubernetes/fake" "os" "path/filepath" "strings" @@ -93,9 +95,6 @@ func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { } func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { - missingConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: testCatalogNamespace}, - } missingData := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, } @@ -118,10 +117,6 @@ func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { name: "empty namespace", namespace: "", configMap: capabilityCatalogConfigMap(validCatalog), want: "namespace is empty", }, - { - name: "missing ConfigMap", namespace: testCatalogNamespace, - configMap: missingConfigMap, want: "get storage capability ConfigMap", - }, { name: "missing data key", namespace: testCatalogNamespace, configMap: missingData, want: "has no", @@ -363,11 +358,20 @@ func TestValidateStorageCapabilityCatalogAllowsNothingQualified(t *testing.T) { require.NoError(t, validateStorageCapabilityCatalog(catalog)) } -func TestShippedStorageCapabilityCatalog(t *testing.T) { +// shippedChartDir locates the operator chart: Bazel runs the test from the +// runfiles root with the storage-capability-catalog data dependency, go test +// runs it from the package directory. +func shippedChartDir(t *testing.T) string { + t.Helper() chartDir := filepath.Join("src", "compute-plane-services", "nvca", "deployments", "nvca-operator") if _, err := os.Stat(chartDir); os.IsNotExist(err) { chartDir = filepath.Join("..", "..", "deployments", "nvca-operator") } + return chartDir +} + +func TestShippedStorageCapabilityCatalog(t *testing.T) { + chartDir := shippedChartDir(t) raw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.yaml")) require.NoError(t, err) c := capabilityClient(t, capabilityCatalogConfigMap(string(raw))).Build() @@ -380,18 +384,27 @@ func TestShippedStorageCapabilityCatalog(t *testing.T) { require.NotNil(t, nvmesh.ReaderMountOptions) assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) - // Weka, FSS and Lustre are recorded but not qualified for a cache workflow, - // so they carry no access modes and caching stays off for them. Enabling - // one is an edit to its accessModes, backed by a qualification run. - for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com", "lustre.csi.oraclecloud.com"} { + // Weka and OCI FSS are shared filesystems enabled on the ReadWriteMany + // shape: one shared claim, readers mount it read-only, no derived reader + // PV and therefore no reader mount options. + for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com"} { driver, ok := catalog.Drivers[provisioner] require.True(t, ok, provisioner) require.NotNil(t, driver.AccessModes, provisioner) - assert.Empty(t, *driver.AccessModes, provisioner) + assert.Equal(t, []string{"ReadWriteMany"}, *driver.AccessModes, provisioner) require.NotNil(t, driver.ReaderMountOptions, provisioner) assert.Empty(t, *driver.ReaderMountOptions, provisioner) } + // Lustre is recorded but not enabled for a cache workflow, so it carries + // no access modes and caching stays off for it. + lustre, ok := catalog.Drivers["lustre.csi.oraclecloud.com"] + require.True(t, ok) + require.NotNil(t, lustre.AccessModes) + assert.Empty(t, *lustre.AccessModes) + require.NotNil(t, lustre.ReaderMountOptions) + assert.Empty(t, *lustre.ReaderMountOptions) + schemaRaw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.schema.json")) require.NoError(t, err) assert.True(t, json.Valid(schemaRaw), "shipped JSON schema must be valid JSON") @@ -545,3 +558,55 @@ func TestStorageCapabilityCatalogEncryptionSupported(t *testing.T) { digestDriverProfile(NVMeshStorageClassProvisioner, without, ModelCacheWorkflowHelm, ModelCacheTransitionROXReadOnly), "encryption support is part of the driver profile digest") } + +func TestResolveModelCacheStorageWithClientsetMissingCatalogUsesBuiltin(t *testing.T) { + sc := testModelCacheStorageClass() + k8s := fakek8sclient.NewSimpleClientset(sc) + + selection, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err, "an absent catalog ConfigMap must resolve against the built-in catalog") + assert.True(t, selection.CatalogBuiltin) + assert.Equal(t, ModelCacheProviderNVMesh, selection.Provider) + assert.Equal(t, ModelCacheTransitionROXReadOnly, selection.Transition) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), selection.CatalogRevision) + + // With the ConfigMap present the selection is identical except for the flag. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + Data: map[string]string{StorageCapabilityConfigMapKey: builtinStorageCapabilityCatalogYAML}, + }) + fromConfigMap, err := ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.NoError(t, err) + assert.False(t, fromConfigMap.CatalogBuiltin) + fromConfigMap.CatalogBuiltin = true + assert.Equal(t, selection, fromConfigMap) + + // A present but empty catalog is a configuration error, not absence. + k8s = fakek8sclient.NewSimpleClientset(sc, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: "nvca-system"}, + }) + _, err = ResolveModelCacheStorageWithClientset(context.Background(), k8s, "nvca-system", ModelCacheWorkflowRegular) + require.ErrorContains(t, err, "has no") +} + +// The built-in catalog must be the one the chart installs, or an agent ahead +// of its chart would resolve differently from a converged install. +func TestBuiltinCatalogMatchesChart(t *testing.T) { + chartCopy, err := os.ReadFile(filepath.Join(shippedChartDir(t), "files", "nvcf-storage-capabilities-v1alpha1.yaml")) + require.NoError(t, err, "the chart catalog must be readable; the built-in copy is checked against it") + assert.Equal(t, string(chartCopy), builtinStorageCapabilityCatalogYAML) + _, _, err = builtinStorageCapabilityCatalog() + require.NoError(t, err) +} + +func TestLoadStorageCapabilityCatalogMissingConfigMapUsesBuiltin(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + catalog, digest, builtin, err := loadStorageCapabilityCatalogSnapshot(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + assert.True(t, builtin) + assert.Equal(t, digestCatalogPayload(builtinStorageCapabilityCatalogYAML), digest) + _, ok := catalog.Drivers[NVMeshStorageClassProvisioner] + assert.True(t, ok, "the built-in catalog must qualify NVMesh") +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go index 6a8ee98be1..09f99c42dc 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go @@ -157,13 +157,6 @@ func TestResolveModelCacheStorageErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []client.Object { return []client.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "catalog ConfigMap missing data key", namespace: testCatalogNamespace, @@ -315,13 +308,6 @@ func TestResolveModelCacheStorageWithClientsetErrors(t *testing.T) { want: "model cache StorageClass not found", notFound: true, }, - { - name: "missing catalog ConfigMap", - namespace: testCatalogNamespace, - workflow: ModelCacheWorkflowRegular, - objects: func() []runtime.Object { return []runtime.Object{testModelCacheStorageClass()} }, - want: "get storage capability ConfigMap", - }, { name: "empty catalog namespace", namespace: "",